Neural networks: The zen of Testing: Three sets strategy

Author: DevOpsThinh

ml/supervised_learning/classifications/our_own_mnist_lib.py

@@ -10,7 +10,7 @@
 import matplotlib.pyplot as plt
 import numpy as np
 
-from ml.util import prepend_bias, one_hot_encoding
+from ml.util import prepend_bias, one_hot_encoding, load_images, load_labels
 
 # TRAIN_IMAGE = "../../../fundamentals/datasets/mnist/train-images-idx3-ubyte.gz"
 # TRAIN_LABEL = "../../../fundamentals/datasets/mnist/train-labels-idx1-ubyte.gz"
@@ -23,14 +23,6 @@
 TEST_LABEL = "../../../../fundamentals/datasets/mnist/t10k-labels-idx1-ubyte.gz"
 
 
-def load_images(filename):
-    """Decodes images from MNIST's library files"""
-    with gzip.open(filename, "rb") as f:
-        _ignored, n_images, columns, rows = struct.unpack('>IIII', f.read(16))
-        all_pixels = np.frombuffer(f.read(), dtype=np.uint8)
-        return all_pixels.reshape(n_images, columns * rows)
-
-
 # 60000 images, each 785 (1 bias + 28 * 28) elements
 X_train_data = prepend_bias(load_images(TRAIN_IMAGE))
 print(X_train_data.shape)
@@ -43,16 +35,6 @@ def load_images(filename):
 X_test = load_images(TEST_IMAGE)  # no bias - 784 elements
 
 
-def load_labels(filename):
-    """Loads MNIST labels into a Numpy array, then molds that array into a
-        one-column matrix.
-    """
-    with gzip.open(filename, "rb") as f:
-        f.read(8)
-        all_labels = f.read()
-        return np.frombuffer(all_labels, dtype=np.uint8).reshape(-1, 1)
-
-
 def encode_fives(y):
     """Convert all 5s to 1, and everything else to 0"""
     return (y == 5).astype(int)

ml/supervised_learning/neural_networks/the_zen_of_testing.py

@@ -0,0 +1,84 @@
+# Learner: Nguyen Truong Thinh
+# Contact me: [email protected] || +84393280504
+#
+# Topic: Supervised Learning: The zen of Testing
+#           A neural network implementation
+
+# An MNIST data (pre-shuffled) loader that splits data into training, validation & test sets.
+import matplotlib.pyplot as plt
+import numpy as np
+from ml.supervised_learning.neural_networks import training_the_network as tn
+from ml.util import load_images, load_labels, one_hot_encoding
+
+TRAIN_IMAGE = "../../../fundamentals/datasets/mnist/train-images-idx3-ubyte.gz"
+TRAIN_LABEL = "../../../fundamentals/datasets/mnist/train-labels-idx1-ubyte.gz"
+TEST_IMAGE = "../../../fundamentals/datasets/mnist/t10k-images-idx3-ubyte.gz"
+TEST_LABEL = "../../../fundamentals/datasets/mnist/t10k-labels-idx1-ubyte.gz"
+
+# X_train/ X_validation/ X_test: 60k/ 5k/ 5k images
+# Each image has 784 elements (28 * 28 pixels)
+X_train = load_images(TRAIN_IMAGE)
+X_test_all = load_images(TEST_IMAGE)  # To ensure best practice: np.random.shuffle(X_test_all)
+X_validation, X_test = np.split(X_test_all, 2)
+
+# 60K labels, each a single digit from 0 to 9
+Y_train_unencoded = load_labels(TRAIN_LABEL)
+#  Y_train: 60k labels, each consisting of 10 one-hot-encoded elements
+Y_train = one_hot_encoding(Y_train_unencoded, 10)
+# Y_validation/ Y_test: 5k/ 5k labels, each a single digit from 0 to 9
+Y_test_all = load_labels(TEST_LABEL)  # To ensure best practice: np.random.shuffle(Y_test_all)
+Y_validation, Y_test = np.split(Y_test_all, 2)
+
+
+# This loss() takes different parameters than the ones in other source files
+def loss(_x, _y, _w1, _w2):
+    _y_hat, _ = tn.forward(_x, _w1, _w2)
+    return -np.sum(_y * np.log(_y_hat)) / _y.shape[0]
+
+
+def train(x_train, y_train, x_test, y_test, _n_hidden_nodes, iterations, lr):
+    n_input_variables = x_train.shape[1]
+    n_classes = y_train.shape[1]
+    # Initialize all the weights at zero
+    # _w1 = np.zeros((n_input_variables + 1, _n_hidden_nodes))
+    # _w2 = np.zeros((_n_hidden_nodes + 1, n_classes))
+    # Initialize all the weights with good initialization
+    _w1, _w2 = tn.initialize_weights(n_input_variables, _n_hidden_nodes, n_classes)
+    _training_losses = []
+    _test_losses = []
+
+    for i in range(iterations):
+        y_hat_train, h = tn.forward(x_train, _w1, _w2)
+        y_hat_test, _ = tn.forward(x_test, _w1, _w2)
+        w1_gradient, w2_gradient = tn.back(x_train, y_train, y_hat_train, _w2, h)
+        _w1 = _w1 - (w1_gradient * lr)
+        _w2 = _w2 - (w2_gradient * lr)
+
+        training_loss = -np.sum(y_train * np.log(y_hat_train)) / y_train.shape[0]
+        _training_losses.append(training_loss)
+        test_loss = -np.sum(y_test * np.log(y_hat_test)) / y_test.shape[0]
+        _test_losses.append(test_loss)
+
+        print("%5d > Training loss: %.5f - Test loss: %.5f" % (i, training_loss, test_loss))
+
+    return _training_losses, _test_losses, _w1, _w2
+
+
+training_losses, test_losses, w1, w2 = train(X_train, Y_train,
+                                             X_test,
+                                             one_hot_encoding(Y_test, 10),
+                                             _n_hidden_nodes=200,
+                                             iterations=10000,
+                                             lr=0.01)
+training_accuracy = tn.accuracy(X_train, Y_train, w1, w2)
+test_accuracy = tn.accuracy(X_test, Y_test, w1, w2)
+print("Training accuracy: %.2f%%, Test accuracy: %.2f%%" % (training_accuracy, test_accuracy))
+
+plt.plot(training_losses, label="Training set", color='blue', linestyle='-')
+plt.plot(test_losses, label="Test set", color='green', linestyle='--')
+plt.xlabel("Iterations", fontsize=30)
+plt.ylabel("Loss", fontsize=30)
+plt.xticks(fontsize=15)
+plt.yticks(fontsize=15)
+plt.legend(fontsize=30)
+plt.show()

ml/supervised_learning/neural_networks/training_the_network.py

@@ -37,8 +37,8 @@ def report(iteration, x_train, y_train, x_test, y_test, _w1, _w2):
     y_hat, _ = forward(x_train, _w1, _w2)
     training_loss = loss(y_train, y_hat)
     classifications = classify(x_test, _w1, _w2)
-    accuracy = np.average(classifications == y_test) * 100.0
-    print("Iteration: %5d, Loss: %.8f, Accuracy: %.2f%%" % (iteration, training_loss, accuracy))
+    _accuracy = np.average(classifications == y_test) * 100.0
+    print("Iteration: %5d, Loss: %.8f, Accuracy: %.2f%%" % (iteration, training_loss, _accuracy))
 
 
 def initialize_weights(_n_input_variables, _n_hidden_nodes, _n_classes):
@@ -58,6 +58,10 @@ def initialize_weights(_n_input_variables, _n_hidden_nodes, _n_classes):
     return _w1, _w2
 
 
+def accuracy(_x, _y_unencoded, _w1, _w2):
+    return np.average(classify(_x, _w1, _w2) == _y_unencoded) * 100.0
+
+
 def classify(_x, _w1, _w2):
     """Predict the value of an unlabeled image"""
     y_hat, _ = forward(_x, _w1, _w2)
@@ -108,5 +112,4 @@ def softmax(logits):
     exponentials = np.exp(logits)
     return exponentials / np.sum(exponentials, axis=1).reshape(-1, 1)
 
-
 # w1, w2 = train(mnist.X_train, mnist.Y_train, mnist.X_test, mnist.Y_test, _n_hidden_nodes=200, iterations=10000, lr=0.01)

ml/supervised_learning/plotting_graphs/graphs/2d_non_linearly_separable_neural_net.png

@@ -2,6 +2,9 @@
 # Contact me: [email protected] || +84393280504
 #
 # The helper functions
+import gzip
+import struct
+
 import matplotlib.pyplot as plt
 import numpy as np
 import seaborn as sns
@@ -124,6 +127,32 @@ def loss(x, y, w, b):
     return np.average((predict(x, w, b) - y) ** 2)
 
 
+def load_labels(fileName):
+    """ Loads MNIST labels into a Numpy array, then molds
+    that array into a one-column matrix.
+    """
+    # Open and unzip the file of images:
+    with gzip.open(fileName, 'rb') as f:
+        # Skip the header bytes:
+        f.read(8)
+        # Read all the labels into a list:
+        all_labels = f.read()
+        # Reshape the list of labels into a one-column matrix
+        return np.frombuffer(all_labels, dtype=np.uint8).reshape(-1, 1)
+
+
+def load_images(fileName):
+    """ Decodes images from MNIST's library files """
+    # Open and unzip the file of images
+    with gzip.open(fileName, 'rb') as f:
+        # Read the header information into a bunch of variables
+        _ignored, n_images, columns, rows = struct.unpack('>IIII', f.read(16))
+        # Read all the pixels into a NumPy array of bytes
+        all_pixels = np.frombuffer(f.read(), dtype=np.uint8)
+        # Reshape the pixels into a matrix where each line is an image
+        return all_pixels.reshape(n_images, columns * rows)
+
+
 def load_text_dataset(text_dataset):
     return np.loadtxt(text_dataset, skiprows=1, unpack=True)