update

plots/df_tr_rnn_cnn_dnn_runs_te_prec.csv

@@ -0,0 +1,34 @@
+indices	tran_prec	rnn_prec	cnn_prec	dnn_prec	transformer_runs_te_prec_low	rnn_runs_te_prec_low	cnn_runs_te_prec_low	dnn_runs_te_prec_low
+10	0.1916	0.0493	0.1825	0.1686	0.0093	0.0036	0.0088	0.0118
+1000	0.6538	0.2212	0.6698	0.4421	0.1727	0.017	0.2014	0.0648
+2000	0.8686	0.225	0.8609	0.5861	0.573	0.0171	0.6278	0.1926
+5000	0.9624	0.5627	0.9498	0.7569	0.7937	0.0679	0.8159	0.5146
+10000	0.9739	0.9078	0.9735	0.8392	0.8112	0.7961	0.833	0.6089
+12000	0.9677	0.9359	0.9698	0.8555	0.8235	0.8633	0.8421	0.6424
+15000	0.981	0.9531	0.969	0.8669	0.8267	0.8814	0.8347	0.648
+20000	0.98	0.966	0.9839	0.8998	0.8425	0.9043	0.8441	0.6959
+25000	0.9875	0.9695	0.9695	0.9196	0.8401	0.9144	0.8547	0.7159
+30000	0.9824	0.9816	0.979	0.9207	0.8489	0.9177	0.8596	0.7303
+35000	0.9826	0.9837	0.9856	0.9212	0.8489	0.9186	0.8606	0.7689
+10	0.1715	0.0528	0.1923	0.1546	0.0105	0.001	0.0032	0.0166
+1000	0.6728	0.2365	0.6848	0.4302	0.1882	0.0104	0.2036	0.0554
+2000	0.8675	0.2197	0.8354	0.5639	0.5612	0.0102	0.5919	0.2002
+5000	0.9644	0.5478	0.9362	0.7695	0.749	0.0567	0.7802	0.4823
+10000	0.9732	0.8762	0.9639	0.8514	0.7946	0.6573	0.8112	0.646
+12000	0.9743	0.9328	0.9706	0.861	0.7987	0.7871	0.8339	0.6588
+15000	0.9682	0.9608	0.9743	0.8508	0.8058	0.8474	0.8484	0.662
+20000	0.9717	0.9783	0.97	0.8802	0.8099	0.8776	0.87	0.7015
+25000	0.9692	0.9769	0.9708	0.9028	0.8026	0.8809	0.8637	0.709
+30000	0.967	0.9792	0.9819	0.9071	0.8093	0.8752	0.8516	0.7264
+35000	0.9597	0.9799	0.9783	0.9104	0.8014	0.8752	0.865	0.7338
+10	0.1665	0.0696	0.1767	0.1263	0.0124	0.0047	0.0079	0.0137
+1000	0.6444	0.2397	0.6969	0.4206	0.1554	0.0137	0.2398	0.0501
+2000	0.858	0.2375	0.8904	0.5938	0.5058	0.0231	0.6409	0.2025
+5000	0.9772	0.5743	0.955	0.7505	0.8144	0.0562	0.807	0.4627
+10000	0.9783	0.9005	0.9591	0.8409	0.8549	0.7203	0.8323	0.5862
+12000	0.9756	0.939	0.9868	0.8721	0.8544	0.8306	0.8387	0.6181
+15000	0.9718	0.9676	0.9696	0.8717	0.8615	0.8855	0.8546	0.6745
+20000	0.9663	0.9825	0.9789	0.8847	0.8596	0.9119	0.856	0.7003
+25000	0.9708	0.9798	0.9786	0.9167	0.8711	0.9103	0.8619	0.7322
+30000	0.9769	0.9773	0.9772	0.8903	0.8711	0.9042	0.8668	0.7406
+35000	0.9731	0.9765	0.9728	0.9047	0.8662	0.9103	0.8641	0.744

scripts/analyze_workflows.py

@@ -47,7 +47,6 @@ def find_wf_statistics(unique_paths):
     utils.write_dictionary("data/aug_22/freq_tool.txt", s_freq_tool)
 
     df_freq_tools.to_csv("data/aug_22/df_freq_tools.csv", sep=",", index=None)
-
 
 
 if __name__ == "__main__":

scripts/evaluate_model.py

@@ -66,7 +66,7 @@
     base_path = "log_cnn/"
 
 elif model_type == "transformer":
-    base_path = "log_transformer/"
+    base_path = "log/"
 
 elif model_type == "dnn":
     base_path = "log_dnn/"
@@ -104,7 +104,7 @@
 
 #tr_pos_plot = [1000, 5000, 10000, 20000, 30000, 40000]
 
-model_number = 30
+model_number = 40
 model_path = base_path + "saved_model/" + str(model_number) + "/tf_model/"
 model_path_h5 = base_path + "saved_model/" + str(model_number) + "/tf_model_h5/"
 
@@ -564,7 +564,7 @@ def plot_TSNE(embed, labels):
     print("Plotting embedding...")
     print(labels)
 
-    #perplexity = 50
+    #perplexity = 500.14460978835978835
     n_colors = 10
     figsize = (8, 8)
 

scripts/train_cnn.py

@@ -5,7 +5,7 @@
 import tensorflow as tf
 from tensorflow.keras.layers import Dropout, Embedding, Input, Dense, GRU
 from tensorflow.keras.models import Model, Sequential
-from tensorflow.keras.layers import Conv1D, Conv2D, MaxPooling2D, Flatten, Lambda, GlobalMaxPooling1D
+from tensorflow.keras.layers import Conv2D, MaxPooling2D, Flatten, Lambda
 from tensorflow.keras.models import Model
 
 import utils

scripts/train_rnn.py

@@ -64,6 +64,9 @@ def create_rnn_architecture(train_data, train_labels, test_data, test_labels, f_
     enc_optimizer = tf.keras.optimizers.Adam(learning_rate=config["learning_rate"])
 
     model = create_model(vocab_size, config)
+
+    model.summary()
+
     prev_model_number = config["restart_step"]
     if prev_model_number > 0:
         model_path_h5 = model_path + str(prev_model_number) + "/tf_model_h5/model.h5"

scripts/transformer_paper_plots.py

@@ -20,8 +20,8 @@
 from tensorflow.keras.layers import Dense, GRU, Dropout, Embedding, SpatialDropout1D
 from tensorflow.keras.layers import MultiHeadAttention, LayerNormalization, Dropout, Layer
 
-from tensorflow.keras.layers import Embedding, Input, GlobalAveragePooling1D, Dense
-from tensorflow.keras.models import Model
+from tensorflow.keras.layers import Embedding, Input, GlobalAveragePooling1D, Dense, Lambda, Conv2D, MaxPooling2D, Flatten
+from tensorflow.keras.models import Model, Sequential
 
 import transformer_network
 
@@ -38,14 +38,22 @@
 embed_dim = 128 # Embedding size for each token d_model
 num_heads = 4 # Number of attention heads
 ff_dim = 128 # Hidden layer size in feed forward network inside transformer # dff
-dropout = 0.1
+dropout = 0.2
 seq_len = 25
 
 
-base_path = "/media/anupkumar/b1ea0d39-97af-4ba5-983f-cd3ff76cf7a6/tool_prediction_datasets/computed_results/aug_22 data/"
-n_runs = 5
+base_path = "/media/anupkumar/b1ea0d39-97af-4ba5-983f-cd3ff76cf7a6/backup_tool_pred_transformer_computed_results/aug_22_data/"
+#"/media/anupkumar/b1ea0d39-97af-4ba5-983f-cd3ff76cf7a6/tool_prediction_datasets/computed_results/aug_22 data/"
+#"/media/anupkumar/b1ea0d39-97af-4ba5-983f-cd3ff76cf7a6/backup_tool_pred_transformer_computed_results/aug_22_data/"
+#"/media/anupkumar/b1ea0d39-97af-4ba5-983f-cd3ff76cf7a6/tool_prediction_datasets/computed_results/aug_22 data/"
+n_runs = 3
 
 
+def read_file_cnn_dnn(file_path):
+    with open(file_path, "r") as json_file:
+        file_content = json_file.read()
+    return file_content
+
 def read_file(file_path):
     with open(file_path, "r") as json_file:
         file_content = json.loads(json_file.read())
@@ -55,9 +63,10 @@ def read_file(file_path):
 def collect_loss_prec_data(m_type):
     model_type = m_type[0]
     m_path = base_path + model_type + "/run"
+    print(m_path)
     runs_indices = list()
     runs_te_loss = list()
-    model_numbers = [1, 100, 200, 500, 1000, 1500, 2000, 2500, 3000, 3500]
+    model_numbers = [1, 100, 200, 500, 1000, 1200, 1500, 2000, 2500, 3000, 3500] #, 1500, 2000, 2500, 3000, 3500
     fig = plt.figure(figsize=fig_size)
     ## Transformer: For test loss
     for i in range(n_runs):
@@ -83,48 +92,102 @@ def collect_loss_prec_data(m_type):
     fig = plt.figure(figsize=fig_size)
     transformer_runs_te_prec = list()
     transformer_runs_te_prec_low = list()
+    print(m_path)
     #model_numbers = [100, 200, 500, 1000, 2000, 3000, 3500]
     for i in range(n_runs):
        x_path = "{}{}/".format(m_path, str(i+1))
        epo_te_batch_prec = read_file(x_path + "data/epo_te_precision.txt").split(",")
        epo_te_batch_prec = np.array([np.round(float(item), 4) for item in epo_te_batch_prec])
        epo_te_batch_prec = epo_te_batch_prec[model_numbers]
        transformer_runs_te_prec.extend(epo_te_batch_prec)
-       #epo_low_te_precision = read_file(x_path + "data/epo_low_te_precision.txt").split(",")
-       #epo_low_te_precision = [np.round(float(item), 4) for item in epo_low_te_precision]
-       #transformer_run3500s_te_prec_low.extend(epo_low_te_precision)
+
+       epo_low_te_precision = read_file(x_path + "data/epo_low_te_precision.txt").split(",")
+       print(i, epo_low_te_precision)
+       epo_low_te_precision = np.array([np.round(float(item), 4) for item in epo_low_te_precision])
+       epo_low_te_precision = epo_low_te_precision[model_numbers]
+       transformer_runs_te_prec_low.extend(epo_low_te_precision)
 
     ## RNN: For test precision
     rnn_runs_te_prec = list()
     rnn_runs_te_prec_low = list()
     m_path = base_path + m_type[1] + "/run"
+    print(m_path)
+    for i in range(n_runs):
+
+       x_path = "{}{}/".format(m_path, str(i+1))
+       epo_te_batch_prec = read_file(x_path + "data/epo_te_precision.txt").split(",")
+       epo_te_batch_prec = np.array([np.round(float(item), 4) for item in epo_te_batch_prec])
+       epo_te_batch_prec = epo_te_batch_prec[model_numbers]
+       rnn_runs_te_prec.extend(epo_te_batch_prec)
+
+       rnn_low_te_precision = read_file(x_path + "data/epo_low_te_precision.txt").split(",")
+       rnn_low_te_precision = np.array([np.round(float(item), 4) for item in rnn_low_te_precision])
+       rnn_low_te_precision = rnn_low_te_precision[model_numbers]
+       rnn_runs_te_prec_low.extend(rnn_low_te_precision)
+
+    ## CNN: For test precision
+    cnn_runs_te_prec = list()
+    cnn_runs_te_prec_low = list()
+    m_path = base_path + m_type[2] + "/run"
+    print(m_path)
     for i in range(n_runs):
 
        x_path = "{}{}/".format(m_path, str(i+1))
-       rnn_te_batch_prec = read_file(x_path + "data/epo_te_precision.txt").split(",")
-       rnn_te_batch_prec = np.array([np.round(float(item), 4) for item in rnn_te_batch_prec])
-       rnn_te_batch_prec = rnn_te_batch_prec[model_numbers]
-       rnn_runs_te_prec.extend(rnn_te_batch_prec)
+       print(x_path)
+       epo_te_batch_prec = read_file_cnn_dnn(x_path + "data/epo_te_precision.txt").split(",")
+       print(epo_te_batch_prec)
+       epo_te_batch_prec = np.array([np.round(float(item), 4) for item in epo_te_batch_prec])
+       epo_te_batch_prec = epo_te_batch_prec[model_numbers]
+       cnn_runs_te_prec.extend(epo_te_batch_prec)
+
+       cnn_low_te_precision = read_file_cnn_dnn(x_path + "data/epo_low_te_precision.txt").split(",")
+       cnn_low_te_precision = np.array([np.round(float(item), 4) for item in cnn_low_te_precision])
+       cnn_low_te_precision = cnn_low_te_precision[model_numbers]
+       cnn_runs_te_prec_low.extend(cnn_low_te_precision)
+
+
+    ## DNN: For test precision
+    dnn_runs_te_prec = list()
+    dnn_runs_te_prec_low = list()
+    m_path = base_path + m_type[3] + "/run"
+    print(m_path)
+    for i in range(n_runs):
+
+       x_path = "{}{}/".format(m_path, str(i+1))
+       epo_te_batch_prec = read_file_cnn_dnn(x_path + "data/epo_te_precision.txt").split(",")
+       epo_te_batch_prec = np.array([np.round(float(item), 4) for item in epo_te_batch_prec])
+       epo_te_batch_prec = epo_te_batch_prec[model_numbers]
+       dnn_runs_te_prec.extend(epo_te_batch_prec)
 
-       #rnn_low_te_precision = read_file(x_path + "data/epo_low_te_precision.txt").split(",")
-       #rnn_low_te_precision = [np.round(float(item), 4) for item in rnn_low_te_precision]
-       #rnn_runs_te_prec_low.extend(rnn_low_te_precision)
+       dnn_low_te_precision = read_file_cnn_dnn(x_path + "data/epo_low_te_precision.txt").split(",")
+       dnn_low_te_precision = np.array([np.round(float(item), 4) for item in dnn_low_te_precision])
+       dnn_low_te_precision = dnn_low_te_precision[model_numbers]
+       dnn_runs_te_prec_low.extend(dnn_low_te_precision)
 
-    #df_tr_rnn_runs_te_prec = pd.DataFrame(zip(runs_indices, transformer_runs_te_prec, transformer_runs_te_prec_low, rnn_runs_te_prec, rnn_runs_te_prec_low), columns=["indices", "tran_prec", "tran_low_prec", "rnn_prec", "rnn_low_prec"])
 
-    df_tr_rnn_runs_te_prec = pd.DataFrame(zip(runs_indices, transformer_runs_te_prec, rnn_runs_te_prec), columns=["indices", "tran_prec", "rnn_prec"])
-    print(df_tr_rnn_runs_te_prec)
+    df_tr_rnn_cnn_dnn_runs_te_prec = pd.DataFrame(zip(runs_indices, transformer_runs_te_prec, rnn_runs_te_prec, cnn_runs_te_prec, dnn_runs_te_prec, transformer_runs_te_prec_low, rnn_runs_te_prec_low, cnn_runs_te_prec_low, dnn_runs_te_prec_low), columns=["indices", "tran_prec", "rnn_prec", "cnn_prec", "dnn_prec", "transformer_runs_te_prec_low", "rnn_runs_te_prec_low", "cnn_runs_te_prec_low", "dnn_runs_te_prec_low"])
 
-    print(df_tr_rnn_runs_te_prec)
-    sns.lineplot(data=df_tr_rnn_runs_te_prec, x="indices", y="tran_prec", label="Transformer: test tools", linestyle="-", color="green")
-    #sns.lineplot(data=df_tr_rnn_runs_te_prec, x="indices", y="tran_low_prec", label="Transformer: lowest 25% test tools", color="green", linestyle=":")
-    sns.lineplot(data=df_tr_rnn_runs_te_prec, x="indices", y="rnn_prec", label="RNN (GRU): test tools", color="red", linestyle="-")
-    #sns.lineplot(data=df_tr_rnn_runs_te_prec, x="indices", y="rnn_low_prec", label="RNN (GRU): lowest 25% test tools", color="red", linestyle=":")
+    print(df_tr_rnn_cnn_dnn_runs_te_prec)
+
+    df_tr_rnn_cnn_dnn_runs_te_prec.to_csv("plots/df_tr_rnn_cnn_dnn_runs_te_prec.csv", index=None, sep="\t")
+
+    sns.lineplot(data=df_tr_rnn_cnn_dnn_runs_te_prec, x="indices", y="tran_prec", label="Transformer: test tools", color="green", linestyle="-")
+    sns.lineplot(data=df_tr_rnn_cnn_dnn_runs_te_prec, x="indices", y="rnn_prec", label="RNN (GRU): test tools", color="red", linestyle="-")
+    sns.lineplot(data=df_tr_rnn_cnn_dnn_runs_te_prec, x="indices", y="cnn_prec", label="CNN: test tools", color="blue", linestyle="-")
+    sns.lineplot(data=df_tr_rnn_cnn_dnn_runs_te_prec, x="indices", y="dnn_prec", label="DNN: test tools", color="black", linestyle="-")
+
+    sns.lineplot(data=df_tr_rnn_cnn_dnn_runs_te_prec, x="indices", y="transformer_runs_te_prec_low", label="Transformer: lowest 25% test tools", color="green", linestyle=":")
+    sns.lineplot(data=df_tr_rnn_cnn_dnn_runs_te_prec, x="indices", y="rnn_runs_te_prec_low", label="RNN (GRU): lowest 25% test tools", color="red", linestyle=":")
+    sns.lineplot(data=df_tr_rnn_cnn_dnn_runs_te_prec, x="indices", y="cnn_runs_te_prec_low", label="CNN: lowest 25% test tools", color="blue", linestyle=":")
+    sns.lineplot(data=df_tr_rnn_cnn_dnn_runs_te_prec, x="indices", y="dnn_runs_te_prec_low", label="DNN: lowest 25% test tools", color="black", linestyle=":")
+
     plt.grid(True)
     plt.xlabel("Training iteration")
     plt.ylabel("Precision@k")
     plt.title("Test: precision@k")
-    plt.savefig("plots/transformer_rnn_runs_te_prec.pdf", dpi=150)
+
+    plt.savefig("plots/transformer_rnn_cnn_dnn_runs_te_low_prec.pdf", dpi=150)
+    plt.show()
 
 
 ##################### Model vs load time ###############################
@@ -147,8 +210,14 @@ def read_h5_model(run, m_type, m_num):
     m_load_s_time = time.time()
     if m_type == "transformer":
         tf_loaded_model = create_transformer_model(seq_len, len(r_dict) + 1)
-    else:
+    elif m_type == "rnn":
         tf_loaded_model = create_rnn_model(seq_len, len(r_dict) + 1)
+    elif m_type == "cnn":
+        print("reading cnn model")
+        tf_loaded_model = create_cnn_model(seq_len, len(r_dict) + 1)
+    elif m_type == "dnn":
+        tf_loaded_model = create_dnn_model(seq_len, len(r_dict) + 1)
+
     tf_loaded_model.load_weights(h5_path)
     m_load_e_time = time.time()
     model_loading_time = m_load_e_time - m_load_s_time
@@ -194,15 +263,48 @@ def create_transformer_model(maxlen, vocab_size):
     return Model(inputs=inputs, outputs=[outputs, weights])
 
 
+def create_dnn_model(maxlen, vocab_size):
+
+    model = Sequential()
+    model.add(Embedding(vocab_size+1, embed_dim, input_length=maxlen))
+    model.add(SpatialDropout1D(dropout))
+    model.add(Flatten())
+    model.add(Dense(embed_dim, input_shape=(seq_len,), activation="elu"))
+    model.add(Dropout(dropout))
+    model.add(Dense(embed_dim, activation="elu"))
+    model.add(Dropout(dropout))
+    model.add(Dense(vocab_size, activation="sigmoid"))
+
+    return model
+
+def create_cnn_model(maxlen, vocab_size): 
+
+    model = Sequential()
+    model.add(Embedding(vocab_size+1, ff_dim, input_length=maxlen))
+    model.add(Lambda(lambda x: tf.expand_dims(x, 3)))
+    model.add(Conv2D(ff_dim, kernel_size=(16, 3), activation = 'relu', kernel_initializer='he_normal', padding = 'VALID'))
+    model.add(Dropout(dropout))
+    model.add(MaxPooling2D(pool_size=(2, 2)))
+    model.add(Flatten())
+    model.add(Dense(ff_dim, activation='relu', kernel_initializer='he_normal'))
+    model.add(Dropout(dropout))
+    model.add(Dense(vocab_size, activation='sigmoid'))
+
+    return model
+
+
 def plot_model_vs_load_time():
-    model_types = ["transformer", "rnn"]
-    model_numbers = ["1000", "2000", "3000", "4000", "5000", "10000", "20000", "25000", "30000", "35000"]
+    model_types = ["transformer", "rnn", "cnn", "dnn"]
+    model_numbers = ["1000", "2000", "3000", "4000", "5000", "10000", "20000", "25000", "30000", "35000"] 
+    #, "3000", "4000", "5000", "10000", "20000", "25000", "30000", "35000"
     input_seq_lengths = [1, 5, 10, 15, 20]
     top_k = [1, 5, 10, 15, 20]
     transformer_model_num = list()
     rnn_model_num = list()
     transformer_load_time = list()
     rnn_load_time = list()
+    cnn_load_time = list()
+    dnn_load_time = list()
     for m_type in model_types:
         for run in range(n_runs):
             for m_num in model_numbers:
@@ -211,28 +313,39 @@ def plot_model_vs_load_time():
                 print()
                 if m_type == "transformer":
                    transformer_load_time.append(model_loading_time)
-                else:
+                elif m_type == "rnn":
                    rnn_load_time.append(model_loading_time)
+                elif m_type == "cnn":
+                   cnn_load_time.append(model_loading_time)
+                elif m_type == "dnn":
+                   dnn_load_time.append(model_loading_time)
             if m_type == "transformer":
                 transformer_model_num.extend(model_numbers)
             else:
                 rnn_model_num.extend(model_numbers)
             print("Model number ends")
         print("Run ends")
-
     print(transformer_load_time, len(transformer_load_time))
     print()
     print(rnn_load_time, len(rnn_load_time))
+    print()
+    print(cnn_load_time, len(cnn_load_time))
+    print()
+    print(dnn_load_time, len(dnn_load_time))
 
-    df_tran_rnn_model_load_time = pd.DataFrame(zip(rnn_model_num, transformer_load_time, rnn_load_time), columns=["model_nums", "tran_load_time", "rnn_load_time"])
+
+    df_tran_rnn_dnn_cnn_model_load_time = pd.DataFrame(zip(rnn_model_num, transformer_load_time, rnn_load_time, cnn_load_time, dnn_load_time,), columns=["model_nums", "tran_load_time", "rnn_load_time", "cnn_load_time", "dnn_load_time"])
     fig = plt.figure(figsize=fig_size)
-    sns.lineplot(data=df_tran_rnn_model_load_time, x="model_nums", y="tran_load_time", label="Transformer: model load time", linestyle="-", color="green")
-    sns.lineplot(data=df_tran_rnn_model_load_time, x="model_nums", y="rnn_load_time", label="RNN (GRU): model load time", color="red", linestyle="-")
+    sns.lineplot(data=df_tran_rnn_dnn_cnn_model_load_time, x="model_nums", y="tran_load_time", label="Transformer: model load time", linestyle="-", color="green")
+    sns.lineplot(data=df_tran_rnn_dnn_cnn_model_load_time, x="model_nums", y="rnn_load_time", label="RNN (GRU): model load time", color="red", linestyle="-")
+    sns.lineplot(data=df_tran_rnn_dnn_cnn_model_load_time, x="model_nums", y="cnn_load_time", label="CNN: model load time", linestyle="-", color="blue")
+    sns.lineplot(data=df_tran_rnn_dnn_cnn_model_load_time, x="model_nums", y="dnn_load_time", label="DNN: model load time", color="black", linestyle="-")
     plt.grid(True)
     plt.xlabel("Training step")
     plt.ylabel("Model load time (seconds)")
-    plt.title("Transformer vs RNN (GRU) model loading time")
+    plt.title("Transformer, RNN (GRU), CNN and DNN models loading time")
     plt.savefig("plots/transformer_rnn_runs_model_load_time.pdf", dpi=150)
+    plt.show()
 
 
 def predict_tools_topk(tf_loaded_model, test_input, k, m_type):
@@ -384,10 +497,10 @@ def plot_usage_time_vs_seq_len():
 
 ############ Call methods ###########################
 
-collect_loss_prec_data(["transformer", "rnn"])
+collect_loss_prec_data(["transformer", "rnn", "cnn", "dnn"])
 plot_model_vs_load_time()
-plot_usage_time_vs_topk()
-plot_usage_time_vs_seq_len()
+#plot_usage_time_vs_topk()
+#plot_usage_time_vs_seq_len()