Early stopping (#13)

* added test loss and proper early stopping in

* added preprocessing of the test data

* adding test_label normalisation

* bug fixing

* fixing early stop reporting

* adding test loss reporting

* updating tests

* bug fixing

* updating python version after 3.6 end of life

* bumping version number

* some flake8 on tests

* pep8 check on globalmeu codes

* flake8 on eval.py

* increasing epochs in test so that early_stop is hit

README.rst

@@ -7,7 +7,7 @@ Introduction
 
 :globalemu: Robust Global 21-cm Signal Emulation
 :Author: Harry Thomas Jones Bevins
-:Version: 1.5.2
+:Version: 1.6.0
 :Homepage: https://github.com/htjb/globalemu
 :Documentation: https://globalemu.readthedocs.io/
 

globalemu/eval.py

@@ -246,5 +246,5 @@ def __call__(self, parameters):
 
         if type(evaluation) is not np.ndarray:
             evaluation = np.array(evaluation)
-        
+
         return evaluation, self.z

globalemu/network.py

@@ -74,18 +74,8 @@ class nn():
 
         early_stop: **Bool / default: False**
             | If ``early_stop`` is set too ``True`` then the network will stop
-                learning if the loss has not changed up to an accuracy given
-                by ``early_stop_lim`` within the last ten epochs.
-
-        early_stop_lim: **float / default: 1e-4**
-            | The precision with which to assess the change in loss over the
-                last ten epochs when ``early_stop=True``. The value of this
-                parameter is strongly dependent on the magnitude of the
-                evaluated loss at each epoch and the default may be to high or
-                too low for the desired outcome. For example if our loss value
-                is initially 0.01 and decreases with each epoch then a
-                ``epoch_stop_lim`` of 0.1 will cause training to stop after
-                10 epochs and give poor results.
+                learning if the loss has not changed within
+                the last twenty epochs.
 
         xHI: **Bool / default: False**
             | If True then ``globalemu`` will act as if it is training a
@@ -153,7 +143,7 @@ def __init__(self, **kwargs):
                     ['batch_size', 'activation', 'epochs',
                         'lr', 'dropout', 'input_shape',
                         'output_shape', 'layer_sizes', 'base_dir',
-                        'early_stop', 'early_stop_lim', 'xHI', 'resume',
+                        'early_stop', 'xHI', 'resume',
                         'random_seed', 'output_activation',
                         'loss_function']):
                 raise KeyError("Unexpected keyword argument in nn()")
@@ -184,7 +174,6 @@ def __init__(self, **kwargs):
             'layer_sizes', [self.input_shape, self.input_shape])
         if type(self.layer_sizes) is not list:
             raise TypeError("'layer_sizes' must be a list.")
-        self.early_stop_lim = kwargs.pop('early_stop_lim', 1e-4)
         self.early_stop = kwargs.pop('early_stop', False)
         self.xHI = kwargs.pop('xHI', False)
         self.random_seed = kwargs.pop('random_seed', None)
@@ -203,9 +192,9 @@ def __init__(self, **kwargs):
             if type(int_kwargs[i]) is not int:
                 raise TypeError("'" + int_strings[i] + "' must be a int.")
 
-        float_kwargs = [self.lr, self.early_stop_lim, self.drop_val,
+        float_kwargs = [self.lr, self.drop_val,
                         self.random_seed]
-        float_strings = ['lr', 'early_stop_lim', 'dropout', 'random_seed']
+        float_strings = ['lr', 'dropout', 'random_seed']
         for i in range(len(float_kwargs)):
             if float_kwargs[i] is not None:
                 if type(float_kwargs[i]) not in set([float, int]):
@@ -238,6 +227,9 @@ def __init__(self, **kwargs):
             label_name=label_names,
             num_epochs=1)
 
+        test_data = np.loadtxt(self.base_dir + 'test_data.txt')
+        test_labels = np.loadtxt(self.base_dir + 'test_label.txt')
+
         def pack_features_vector(features, labels):
             return tf.stack(list(features.values()), axis=1), labels
 
@@ -276,8 +268,11 @@ def grad(model, inputs, targets):
         if self.resume is True:
             train_loss_results = list(
                 np.loadtxt(self.base_dir + 'loss_history.txt'))
+            test_loss_results = list(
+                np.loadtxt(self.base_dir + 'test_loss_history.txt'))
         else:
             train_loss_results = []
+            test_loss_results = []
         train_rmse_results = []
         num_epochs = self.epochs
         for epoch in range(num_epochs):
@@ -296,24 +291,35 @@ def grad(model, inputs, targets):
             train_rmse_results.append(epoch_rmse_avg.result())
             e = time.time()
 
+            test_loss, _ = loss(model, test_data, test_labels, training=False)
+            test_loss_results.append(test_loss)
+
             print(
-                'Epoch: {:03d}, Loss: {:.5f}, RMSE: {:.5f}, Time: {:.3f}'
-                .format(
-                    epoch, epoch_loss_avg.result(),
-                    epoch_rmse_avg.result(), e-s))
-
-            if self.early_stop is True:
-                if len(train_loss_results) > 10:
-                    if np.isclose(
-                            train_loss_results[-10], train_loss_results[-1],
-                            self.early_stop_lim, self.early_stop_lim):
-                        print('Early Stop')
-                        model.save(self.base_dir + 'model.h5')
+                'Epoch: {:03d}, Loss: {:.5f}, Test Loss: {:.5f},'
+                .format(epoch, epoch_loss_avg.result(), test_loss_results[-1])
+                + 'RMSE: {:.5f}, Time: {:.3f}'
+                .format(epoch_rmse_avg.result(), e-s))
+
+            if self.early_stop:
+                if len(test_loss_results) > 20:
+                    delta = 2*np.abs(test_loss_results[-21] -
+                                     test_loss_results[-1]) / \
+                                     (test_loss_results[-21] +
+                                      test_loss_results[-1])
+
+                    if delta*100 < 1e-2:
+                        print('Early Stopped: {:.5f}'.format(delta.numpy()*100)
+                              + ' < 1e-2')
+                        print('Epochs used = ' + str(len(test_loss_results)))
                         break
+
             if (epoch + 1) % 10 == 0:
                 model.save(self.base_dir + 'model.h5')
                 np.savetxt(
                     self.base_dir + 'loss_history.txt', train_loss_results)
+                np.savetxt(
+                    self.base_dir + 'test_loss_history.txt', test_loss_results)
 
         model.save(self.base_dir + 'model.h5')
         np.savetxt(self.base_dir + 'loss_history.txt', train_loss_results)
+        np.savetxt(self.base_dir + 'test_loss_history.txt', test_loss_results)

globalemu/preprocess.py

@@ -100,7 +100,8 @@ def __init__(self, num, z, **kwargs):
         if type(z) not in set([np.ndarray, list]):
             raise TypeError("'z' should be a numpy array or list.")
 
-        # Convert to numpy array, and cast to float to avoid NaN errors in AFB later
+        # Convert to numpy array, and cast to float to
+        # avoid NaN errors in AFB later
         z = np.array(z, dtype=float)
         self.z = z
 
@@ -149,12 +150,15 @@ def __init__(self, num, z, **kwargs):
 
         np.savetxt(self.base_dir + 'z.txt', self.z)
 
-        full_train_data = pd.read_csv(
-            self.data_location + 'train_data.txt',
-            delim_whitespace=True, header=None).values
-        full_train_labels = pd.read_csv(
-            self.data_location + 'train_labels.txt',
-            delim_whitespace=True, header=None).values
+        def load_data(file):
+            return pd.read_csv(
+                self.data_location + file,
+                delim_whitespace=True, header=None).values
+
+        full_train_data = load_data('train_data.txt')
+        full_train_labels = load_data('train_labels.txt')
+        full_test_data = load_data('test_data.txt')
+        test_labels = load_data('test_labels.txt')
 
         if self.preprocess_settings['AFB'] is True:
             np.save(
@@ -166,6 +170,7 @@ def __init__(self, num, z, **kwargs):
             train_data = full_train_data.copy()
             if self.preprocess_settings['AFB'] is True:
                 train_labels = full_train_labels.copy() - res.deltaT
+                test_labels -= res.deltaT
             else:
                 train_labels = full_train_labels.copy()
         else:
@@ -189,16 +194,27 @@ def __init__(self, num, z, **kwargs):
             train_data, train_labels = np.array(train_data), \
                 np.array(train_labels)
 
-        log_td = []
+        log_train_data = []
         for i in range(train_data.shape[1]):
             if i in set(self.logs):
                 for j in range(train_data.shape[0]):
                     if train_data[j, i] == 0:
                         train_data[j, i] = 1e-6
-                log_td.append(np.log10(train_data[:, i]))
+                log_train_data.append(np.log10(train_data[:, i]))
+            else:
+                log_train_data.append(train_data[:, i])
+        train_data = np.array(log_train_data).T
+
+        log_test_data = []
+        for i in range(full_test_data.shape[1]):
+            if i in set(self.logs):
+                for j in range(full_test_data.shape[0]):
+                    if full_test_data[j, i] == 0:
+                        full_test_data[j, i] = 1e-6
+                log_test_data.append(np.log10(full_test_data[:, i]))
             else:
-                log_td.append(train_data[:, i])
-        train_data = np.array(log_td).T
+                log_test_data.append(full_test_data[:, i])
+        test_data = np.array(log_test_data).T
 
         if self.preprocess_settings['resampling'] is True:
             sampling_call = sampling(
@@ -213,18 +229,35 @@ def __init__(self, num, z, **kwargs):
             train_labels = np.array(resampled_labels)
 
             norm_s = np.interp(samples, self.z, cdf)
+
+            resampled_test_labels = []
+            for i in range(len(test_labels)):
+                resampled_test_labels.append(
+                    np.interp(samples, self.z, test_labels[i]))
+            test_labels = np.array(resampled_test_labels)
         else:
             norm_s = (self.z - self.z.min())/(self.z.max() - self.z.min())
 
-        data_mins = train_data.min(axis=0)
-        data_maxs = train_data.max(axis=0)
+        train_data_mins = train_data.min(axis=0)
+        train_data_maxs = train_data.max(axis=0)
+
+        test_data_mins = test_data.min(axis=0)
+        test_data_maxs = test_data.max(axis=0)
 
         norm_train_data = []
         for i in range(train_data.shape[1]):
             norm_train_data.append(
-                (train_data[:, i] - data_mins[i])/(data_maxs[i]-data_mins[i]))
+                                   (train_data[:, i] - train_data_mins[i]) /
+                                   (train_data_maxs[i] - train_data_mins[i]))
         norm_train_data = np.array(norm_train_data).T
 
+        norm_test_data = []
+        for i in range(test_data.shape[1]):
+            norm_test_data.append(
+                                  (test_data[:, i] - test_data_mins[i]) /
+                                  (test_data_maxs[i] - test_data_mins[i]))
+        norm_test_data = np.array(norm_test_data).T
+
         if self.preprocess_settings['std_division'] is True:
             labels_stds = train_labels.std()
             norm_train_labels = [
@@ -234,13 +267,23 @@ def __init__(self, num, z, **kwargs):
 
             norm_train_labels = norm_train_labels.flatten()
             np.save(self.base_dir + 'labels_stds.npy', labels_stds)
+
+            test_labels_stds = test_labels.std()
+            norm_test_labels = [
+                test_labels[i, :]/test_labels_stds
+                for i in range(test_labels.shape[0])]
+            norm_test_labels = np.array(norm_test_labels)
+
+            norm_test_labels = norm_test_labels.flatten()
+
         else:
             norm_train_labels = train_labels.flatten()
+            norm_test_labels = test_labels.flatten()
 
         if self.num != 'full':
             np.savetxt(self.base_dir + 'indices.txt', ind)
-        np.savetxt(self.base_dir + 'data_mins.txt', data_mins)
-        np.savetxt(self.base_dir + 'data_maxs.txt', data_maxs)
+        np.savetxt(self.base_dir + 'data_mins.txt', train_data_mins)
+        np.savetxt(self.base_dir + 'data_maxs.txt', train_data_maxs)
 
         flattened_train_data = []
         for i in range(len(norm_train_data)):
@@ -249,12 +292,21 @@ def __init__(self, num, z, **kwargs):
                     np.hstack([norm_train_data[i, :], norm_s[j]]))
         flattened_train_data = np.array(flattened_train_data)
 
-        train_data, train_label = flattened_train_data, norm_train_labels
-        train_dataset = np.hstack([train_data, train_label[:, np.newaxis]])
+        flattened_test_data = []
+        for i in range(len(norm_test_data)):
+            for j in range(len(norm_s)):
+                flattened_test_data.append(
+                    np.hstack([norm_test_data[i, :], norm_s[j]]))
+        flattened_test_data = np.array(flattened_test_data)
+
+        train_dataset = np.hstack([flattened_train_data,
+                                   norm_train_labels[:, np.newaxis]])
 
         np.savetxt(
             self.base_dir + 'train_dataset.csv', train_dataset, delimiter=',')
-        np.savetxt(self.base_dir + 'train_data.txt', train_data)
-        np.savetxt(self.base_dir + 'train_label.txt', train_label)
+        np.savetxt(self.base_dir + 'train_data.txt', flattened_train_data)
+        np.savetxt(self.base_dir + 'train_label.txt', norm_train_labels)
+        np.savetxt(self.base_dir + 'test_data.txt', flattened_test_data)
+        np.savetxt(self.base_dir + 'test_label.txt', norm_test_labels)
 
         print('...preprocessing done.')

setup.py

@@ -9,7 +9,7 @@ def readme(short=False):
 
 setup(
     name='globalemu',
-    version='1.5.2',
+    version='1.6.0',
     description='globalemu: Robust and Fast Global 21-cm Signal Emulation',
     long_description=readme(),
     author='Harry T. J. Bevins',

tests/test_download.py

@@ -5,6 +5,7 @@
 import pandas as pd
 import numpy as np
 
+
 def download_21cmGEM_data():
     data_dir = '21cmGEM_data/'
     if not os.path.exists(data_dir):
@@ -34,6 +35,7 @@ def download_21cmGEM_data():
     np.savetxt(data_dir + 'train_data.txt', td[:500, :])
     np.savetxt(data_dir + 'train_labels.txt', tl[:500, :])
 
+
 def test_existing_dir():
     if os.path.exists('kappa_HH.txt'):
         os.remove('kappa_HH.txt')

tests/test_gui_config.py

@@ -1,7 +1,6 @@
 import pandas as pd
 from globalemu.gui_config import config
 from globalemu.downloads import download
-import requests
 import shutil
 import os
 import pytest

tests/test_network.py

@@ -2,7 +2,6 @@
 from globalemu.preprocess import process
 from globalemu.network import nn
 from tensorflow.keras import backend as K
-import requests
 import os
 import shutil
 import pytest
@@ -56,8 +55,6 @@ def custom_loss(y, y_, x):
         nn(base_dir='dir')
     with pytest.raises(TypeError):
         nn(early_stop='foo')
-    with pytest.raises(TypeError):
-        nn(early_stop_lim=False)
     with pytest.raises(TypeError):
         nn(xHI='false')
     with pytest.raises(TypeError):
@@ -68,8 +65,8 @@ def custom_loss(y, y_, x):
         nn(loss_function='foobar')
 
     process(10, z, data_location='21cmGEM_data/', base_dir='base_dir/')
-    nn(batch_size=451, layer_sizes=[], random_seed=10,
-       base_dir='base_dir/')
+    nn(batch_size=451, layer_sizes=[], random_seed=10, epochs=30,
+       base_dir='base_dir/', early_stop=True)
 
     dir = ['model_dir/', 'base_dir/']
     for i in range(len(dir)):

tests/test_plotter.py

@@ -5,11 +5,11 @@
 import os
 import shutil
 import pytest
-import requests
 
 params = [0.25, 30, 2, 0.056, 1.3, 2, 30]
 z = np.arange(10, 20, 100)
 
+
 def test_existing_dir():
     if os.path.exists('T_release/'):
         shutil.rmtree('T_release/')

tests/test_preprocess.py

@@ -1,6 +1,5 @@
 import numpy as np
 from globalemu.preprocess import process
-import requests
 import os
 import pytest
 import pandas as pd