Increase test coverage

demo.ipynb

@@ -25,7 +25,7 @@
   },
   {
    "cell_type": "code",
-   "execution_count": 2,
+   "execution_count": 21,
    "metadata": {},
    "outputs": [],
    "source": [
@@ -50,7 +50,7 @@
   },
   {
    "cell_type": "code",
-   "execution_count": 3,
+   "execution_count": 27,
    "metadata": {},
    "outputs": [],
    "source": [
@@ -70,16 +70,16 @@
   },
   {
    "cell_type": "code",
-   "execution_count": 4,
+   "execution_count": 10,
    "metadata": {},
    "outputs": [
     {
      "data": {
       "text/plain": [
-       "Experiment(id=2, name=example)"
+       "Experiment(id=9, name=example)"
       ]
      },
-     "execution_count": 4,
+     "execution_count": 10,
      "metadata": {},
      "output_type": "execute_result"
     }
@@ -770,6 +770,51 @@
     "exp.metrics['training_loss'].plot()\n",
     "plt.legend()"
    ]
+  },
+  {
+   "cell_type": "markdown",
+   "metadata": {},
+   "source": [
+    "## Utils\n",
+    "The `utils` module contains realted functionality, that might be useful during the manual interpretation of experiments."
+   ]
+  },
+  {
+   "cell_type": "code",
+   "execution_count": 15,
+   "metadata": {},
+   "outputs": [],
+   "source": [
+    "from incense import utils"
+   ]
+  },
+  {
+   "cell_type": "markdown",
+   "metadata": {},
+   "source": [
+    "The `find_differing_config_keys` function returns the set of config values that differ in a set of experiments."
+   ]
+  },
+  {
+   "cell_type": "code",
+   "execution_count": 39,
+   "metadata": {},
+   "outputs": [
+    {
+     "data": {
+      "text/plain": [
+       "{'epochs'}"
+      ]
+     },
+     "execution_count": 39,
+     "metadata": {},
+     "output_type": "execute_result"
+    }
+   ],
+   "source": [
+    "exps = loader.find_by_ids([1, 2])\n",
+    "utils.find_differing_config_keys(exps)"
+   ]
   }
  ],
  "metadata": {

example_experiment/conduct.py

@@ -79,6 +79,16 @@ def plot_accuracy_development(history, _run):
     _run.add_artifact(filename=filename, name='accuracy_movie')
 
 
+def write_csv_as_text(history, _run):
+    filename = 'history.txt'
+    with open(filename, 'w') as handle:
+        handle.write('acc, loss\n')
+        for acc, loss in zip(history.history['acc'], history.history['loss']):
+            handle.write(f'{acc}, {loss}\n')
+
+    _run.add_artifact(filename=filename, name='history')
+
+
 ex = Experiment('example')
 ex.captured_out_filter = apply_backspaces_and_linefeeds
 
@@ -101,6 +111,7 @@ def plot_accuracy_development(history, _run):
 def hyperparameters():
     optimizer = 'sgd'  # lgtm [py/unused-local-variable]
     epochs = 1  # lgtm [py/unused-local-variable]
+    seed = 0
 
 
 def make_model():
@@ -151,7 +162,7 @@ def conduct(epochs, optimizer, _run):
     _run.add_artifact('confusion_matrix.png', name='confusion_matrix')
 
     plot_accuracy_development(history, _run)
-
+    write_csv_as_text(history, _run)
     scalar_results = model.evaluate(x_test, y_test, verbose=0)
 
     results = dict(zip(model.metrics_names, scalar_results))

incense/utils.py

@@ -4,8 +4,8 @@
 from typing import *
 
 
-def find_differing_config_keys(experiments: Iterable[Experiment]):
-    """Find the config keys that were assigned to different values in a cohort of experiments.."""
+def find_differing_config_keys(experiments: Iterable[Experiment]) -> set:
+    """Find the config keys that were assigned to different values in a cohort of experiments."""
     config_values = []
     configs = []
     for experiment in experiments:
@@ -22,5 +22,5 @@ def find_differing_config_keys(experiments: Iterable[Experiment]):
     return differing_config_keys
 
 
-def format_config(config, *config_keys):
-    return ' | '.join(f'{key}={config[key]}' for key in config_keys)
+def format_config(exp: Experiment, *config_keys) -> str:
+    return ' | '.join(f'{key}={exp.config[key]}' for key in config_keys)

tests/test_artifact.py

@@ -2,13 +2,22 @@
 import imghdr
 import pickle
 
+from pytest import raises
 import pandas as pd
 import matplotlib
 from IPython.display import HTML
 
 from incense import artifact
 
 
+def test_repr(loader):
+    exp = loader.find_by_id(3)
+    csv_artifact = exp.artifacts['predictions']
+    assert repr(csv_artifact) == 'CSVArtifact(name=predictions)'
+    mp4_artifact = exp.artifacts['accuracy_movie']
+    assert repr(mp4_artifact) == 'MP4Artifact(name=accuracy_movie)'
+
+
 def test_png_artifact_show(loader):
     exp = loader.find_by_id(3)
     png_artifact = exp.artifacts['confusion_matrix']
@@ -83,3 +92,23 @@ def test_as_type(loader):
     pickle_artifact1 = exp.artifacts['predictions_df'].as_type(artifact.PickleArtifact)
     pickle_artifact2 = exp.artifacts['predictions_df'].as_type(artifact.PickleArtifact)
     assert isinstance(pickle_artifact2.show(), pd.DataFrame)
+
+
+def test_as_content_type(loader):
+    exp = loader.find_by_id(2)
+    assert isinstance(exp.artifacts['history'], artifact.Artifact)
+    text_artifact_as_csv = exp.artifacts['history'].as_content_type('text/csv')
+    assert isinstance(text_artifact_as_csv.show(), pd.DataFrame)
+
+
+def test_as_content_type_with_unkwown_content_type(loader):
+    exp = loader.find_by_id(2)
+    assert isinstance(exp.artifacts['history'], artifact.Artifact)
+    with raises(ValueError):
+        text_artifact_as_something_strange = exp.artifacts['history'].as_content_type('something/strange')
+
+
+def test_artifact_show(loader):
+    exp = loader.find_by_id(3)
+    with raises(NotImplementedError):
+        exp.artifacts['predictions_df'].show()

tests/test_experiment.py

@@ -2,6 +2,11 @@
 import pandas as pd
 
 
+def test_repr(loader):
+    exp = loader.find_by_id(3)
+    assert repr(exp) == 'Experiment(id=3, name=example)'
+
+
 def test_metrics(loader):
     exp = loader.find_by_id(3)
     metric_names = [

tests/test_experiment_loader.py

@@ -7,6 +7,13 @@ def test_find_by_id(loader):
     assert exp.id == 1
 
 
+def test_find_by_ids(loader):
+    exps = loader.find_by_ids([1, 2])
+    for exp in exps:
+        assert isinstance(exp, Experiment)
+    assert len(exps) == 2
+
+
 def test_find_by_name(loader):
     exps = loader.find_by_name('example')
     assert len(exps) == 3

tests/test_utils.py

@@ -0,0 +1,13 @@
+from incense import utils
+
+
+def test_find_differing_config_keys(loader):
+    assert utils.find_differing_config_keys(loader.find_by_ids([1, 2])) == {'epochs'}
+    assert utils.find_differing_config_keys(loader.find_by_ids([1, 3])) == {'optimizer'}
+    assert utils.find_differing_config_keys(loader.find_by_ids([2, 3])) == {'epochs', 'optimizer'}
+
+
+def test_format_config(loader):
+    exp = loader.find_by_id(2)
+    assert utils.format_config(exp, 'epochs', 'optimizer') == 'epochs=3 | optimizer=sgd'
+    assert utils.format_config(exp, 'epochs') == 'epochs=3'