Code quality

Author: vineeths96

src/get_data.py

@@ -3,10 +3,10 @@
 import requests
 import pandas as pd
 from path import Path
-from src.parameters import *
+from parameters import *
 
 
-def downloadData(data_path='/input/speech_commands/'):
+def downloadData(data_path="/input/speech_commands/"):
     """
     Downloads Google Speech Commands dataset (version0.01)
     :param data_path: Path to download dataset
@@ -15,10 +15,10 @@ def downloadData(data_path='/input/speech_commands/'):
 
     dataset_path = Path(os.path.abspath(__file__)).parent.parent + data_path
 
-    datasets = ['train', 'test']
+    datasets = ["train", "test"]
     urls = [
-        'http://download.tensorflow.org/data/speech_commands_v0.01.tar.gz',
-        'http://download.tensorflow.org/data/speech_commands_test_set_v0.01.tar.gz'
+        "http://download.tensorflow.org/data/speech_commands_v0.01.tar.gz",
+        "http://download.tensorflow.org/data/speech_commands_test_set_v0.01.tar.gz",
     ]
 
     for dataset, url in zip(datasets, urls):
@@ -27,7 +27,7 @@ def downloadData(data_path='/input/speech_commands/'):
         # Check if we need to extract the dataset
         if not os.path.isdir(dataset_directory):
             os.makedirs(dataset_directory)
-            file_name = dataset_path + dataset + '.tar.gz'
+            file_name = dataset_path + dataset + ".tar.gz"
 
             # Check if the dataset has been downloaded, else download it
             if os.path.isfile(file_name):
@@ -36,7 +36,7 @@ def downloadData(data_path='/input/speech_commands/'):
                 print("Downloading '{}' into '{}' file".format(url, file_name))
 
                 data_request = requests.get(url)
-                with open(file_name, 'wb') as file:
+                with open(file_name, "wb") as file:
                     file.write(data_request.content)
 
             # Extract downloaded file
@@ -54,7 +54,7 @@ def downloadData(data_path='/input/speech_commands/'):
     print("Input data setup successful.")
 
 
-def getDataDict(data_path='/input/speech_commands/'):
+def getDataDict(data_path="/input/speech_commands/"):
     """
     Creates a dictionary with train, test, validate and test file names and labels.
     :param data_path: Path to the downloaded dataset
@@ -64,24 +64,24 @@ def getDataDict(data_path='/input/speech_commands/'):
     data_path = Path(os.path.abspath(__file__)).parent.parent + data_path
 
     # Get the validation files
-    validation_files = open(data_path + 'train/validation_list.txt').read().splitlines()
-    validation_files = [data_path + 'train/' + file_name for file_name in validation_files]
+    validation_files = open(data_path + "train/validation_list.txt").read().splitlines()
+    validation_files = [data_path + "train/" + file_name for file_name in validation_files]
 
     # Get the dev files
-    dev_files = open(data_path + 'train/testing_list.txt').read().splitlines()
-    dev_files = [data_path + 'train/' + file_name for file_name in dev_files]
+    dev_files = open(data_path + "train/testing_list.txt").read().splitlines()
+    dev_files = [data_path + "train/" + file_name for file_name in dev_files]
 
     # Find train_files as allFiles - {validation_files, dev_files}
     all_files = []
-    for root, dirs, files in os.walk(data_path + 'train/'):
-        all_files += [root + '/' + file_name for file_name in files if file_name.endswith('.wav')]
+    for root, dirs, files in os.walk(data_path + "train/"):
+        all_files += [root + "/" + file_name for file_name in files if file_name.endswith(".wav")]
 
     train_files = list(set(all_files) - set(validation_files) - set(dev_files))
 
     # Get the test files
     test_files = list()
-    for root, dirs, files in os.walk(data_path + 'test/'):
-        test_files += [root + '/' + file_name for file_name in files if file_name.endswith('.wav')]
+    for root, dirs, files in os.walk(data_path + "test/"):
+        test_files += [root + "/" + file_name for file_name in files if file_name.endswith(".wav")]
 
     # Get labels
     validation_file_labels = [getLabel(wav) for wav in validation_files]
@@ -90,17 +90,12 @@ def getDataDict(data_path='/input/speech_commands/'):
     test_file_labels = [getLabel(wav) for wav in test_files]
 
     # Create dictionaries containing (file, labels)
-    trainData = {'files': train_files, 'labels': train_file_labels}
-    valData = {'files': validation_files, 'labels': validation_file_labels}
-    devData = {'files': dev_files, 'labels': dev_file_labels}
-    testData = {'files': test_files, 'labels': test_file_labels}
-
-    dataDict = {
-        'train': trainData,
-        'val': valData,
-        'dev': devData,
-        'test': testData
-    }
+    trainData = {"files": train_files, "labels": train_file_labels}
+    valData = {"files": validation_files, "labels": validation_file_labels}
+    devData = {"files": dev_files, "labels": dev_file_labels}
+    testData = {"files": test_files, "labels": test_file_labels}
+
+    dataDict = {"train": trainData, "val": valData, "dev": devData, "test": testData}
 
     return dataDict
 
@@ -112,8 +107,8 @@ def getLabel(file_name):
     :return: Class label
     """
 
-    category = file_name.split('/')[-2]
-    label = categories.get(category, categories['_background_noise_'])
+    category = file_name.split("/")[-2]
+    label = categories.get(category, categories["_background_noise_"])
 
     return label
 
@@ -127,9 +122,9 @@ def getDataframe(data, include_unknown=False):
     """
 
     df = pd.DataFrame(data)
-    df['category'] = df.apply(lambda row: inv_categories[row['labels']], axis=1)
+    df["category"] = df.apply(lambda row: inv_categories[row["labels"]], axis=1)
 
     if not include_unknown:
-        df = df.loc[df['category'] != '_background_noise_', :]
+        df = df.loc[df["category"] != "_background_noise_", :]
 
     return df

src/main.py

@@ -1,11 +1,10 @@
 import os
-from src.model_train import model_train, marvin_kws_model
-from src.model_test import marvin_model_test
+from model_train import model_train, marvin_kws_model
+from model_test import marvin_model_test
 
 
 def main():
-    trained = os.path.isfile('../models/marvin_kws_svm.pickle') \
-              and os.path.isfile('../models/marvin_kws_pca.pickle')
+    trained = os.path.isfile("../models/marvin_kws_svm.pickle") and os.path.isfile("../models/marvin_kws_pca.pickle")
 
     if not trained:
         print("Training model")
@@ -16,5 +15,5 @@ def main():
         marvin_model_test()
 
 
-if __name__ == '__main__':
+if __name__ == "__main__":
     main()

src/model_test.py

@@ -3,10 +3,10 @@
 import tensorflow as tf
 from tensorflow.keras.models import Model, load_model
 
-from src.parameters import *
-from src.utils import OC_Statistics
-from src.utils import getDataset
-from src.get_data import downloadData, getDataDict, getDataframe
+from parameters import *
+from utils import OC_Statistics
+from utils import getDataset
+from get_data import downloadData, getDataDict, getDataframe
 
 
 def marvin_model_test():
@@ -16,43 +16,38 @@ def marvin_model_test():
     """
 
     # Download data
-    downloadData(data_path='/input/speech_commands/')
+    downloadData(data_path="/input/speech_commands/")
 
     # Get dictionary with files and labels
-    dataDict = getDataDict(data_path='/input/speech_commands/')
+    dataDict = getDataDict(data_path="/input/speech_commands/")
 
     # Obtain dataframe by merging dev and test dataset
-    devDF = getDataframe(dataDict['dev'], include_unknown=True)
-    testDF = getDataframe(dataDict['test'], include_unknown=True)
+    devDF = getDataframe(dataDict["dev"], include_unknown=True)
+    testDF = getDataframe(dataDict["test"], include_unknown=True)
 
     evalDF = pd.concat([devDF, testDF], ignore_index=True)
 
     print("Test files: {}".format(evalDF.shape[0]))
 
     # Obtain Marvin - Other separated data
-    evalDF['class'] = evalDF.apply(lambda row: 1 if row['category'] == 'marvin' else -1, axis=1)
-    evalDF.drop('category', axis=1)
-    test_true_labels = evalDF['class'].tolist()
+    evalDF["class"] = evalDF.apply(lambda row: 1 if row["category"] == "marvin" else -1, axis=1)
+    evalDF.drop("category", axis=1)
+    test_true_labels = evalDF["class"].tolist()
 
-    eval_data, _ = getDataset(
-        df=evalDF,
-        batch_size=BATCH_SIZE,
-        cache_file='kws_val_cache',
-        shuffle=False
-    )
+    eval_data, _ = getDataset(df=evalDF, batch_size=BATCH_SIZE, cache_file="kws_val_cache", shuffle=False)
 
     # Load trained model
-    model = load_model('../models/marvin_kws.h5')
+    model = load_model("../models/marvin_kws.h5")
 
-    layer_name = 'features256'
+    layer_name = "features256"
     feature_extractor = Model(inputs=model.input, outputs=model.get_layer(layer_name).output)
 
     # Load trained PCA object
-    with open('../models/marvin_kws_pca.pickle', "rb") as file:
+    with open("../models/marvin_kws_pca.pickle", "rb") as file:
         pca = pickle.load(file)
 
     # Load trained SVM
-    with open('../models/marvin_kws_svm.pickle', "rb") as file:
+    with open("../models/marvin_kws_svm.pickle", "rb") as file:
         marvin_svm = pickle.load(file)
 
     # Extract the feature embeddings and evaluate using SVM
@@ -61,4 +56,4 @@ def marvin_model_test():
     X_test_scaled = pca.transform(X_test)
     test_pred_labels = marvin_svm.predict(X_test_scaled)
 
-    OC_Statistics(test_pred_labels, test_true_labels, 'marvin_cm_without_noise')
+    OC_Statistics(test_pred_labels, test_true_labels, "marvin_cm_without_noise")