From fd54a9d7835ac8a2444a70131791708f77cc8440 Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?R=C3=BCdiger=20Busche?= Date: Thu, 27 Jun 2019 10:03:14 +0200 Subject: [PATCH] Use pyristent to store internal data (#42) * Use pyristent to store internal data * Pin tf version * Add toknize requirement * Update tf version * Add tokenize to dev reqs * Fix test for py35 --- demo.ipynb | 84 +++++++++++++++++++--------------------- incense/experiment.py | 7 ++-- requirements-dev.txt | 5 ++- setup.py | 3 +- tests/test_experiment.py | 16 +++++++- 5 files changed, 63 insertions(+), 52 deletions(-) diff --git a/demo.ipynb b/demo.ipynb index 379a263..24db4b7 100755 --- a/demo.ipynb +++ b/demo.ipynb @@ -227,10 +227,10 @@ { "data": { "text/plain": [ - "{'_id': 2,\n", - " 'experiment': {'name': 'example',\n", + "{'stop_time': datetime.datetime(2019, 4, 11, 19, 23, 43, 146000),\n", + " 'experiment': {'repositories': [],\n", " 'base_dir': '/home/jarno/projects/incense/example_experiment',\n", - " 'sources': [['conduct.py', ObjectId('5caf94233bd29849e342a7e2')]],\n", + " 'name': 'example',\n", " 'dependencies': ['matplotlib==3.0.2',\n", " 'numpy==1.16.2',\n", " 'pandas==0.24.1',\n", @@ -238,37 +238,43 @@ " 'scikit-learn==0.20.3',\n", " 'seaborn==0.9.0',\n", " 'tensorflow==1.13.1'],\n", - " 'repositories': [],\n", - " 'mainfile': 'conduct.py'},\n", + " 'mainfile': 'conduct.py',\n", + " 'sources': [['conduct.py', ObjectId('5caf94233bd29849e342a7e2')]]},\n", " 'format': 'MongoObserver-0.7.0',\n", - " 'command': 'conduct',\n", - " 'host': {'hostname': 'work',\n", + " 'host': {'cpu': 'Intel(R) Core(TM) i7-4800MQ CPU @ 2.70GHz',\n", + " 'ENV': {},\n", + " 'hostname': 'work',\n", " 'os': ['Linux', 'Linux-4.18.0-17-generic-x86_64-with-debian-buster-sid'],\n", - " 'python_version': '3.6.8',\n", - " 'cpu': 'Intel(R) Core(TM) i7-4800MQ CPU @ 2.70GHz',\n", - " 'ENV': {}},\n", + " 'python_version': '3.6.8'},\n", + " 'status': 'COMPLETED',\n", + " 'captured_out': 'INFO - example - Running command \\'conduct\\'\\nINFO - example - Started run with ID \"2\"\\nFailed to detect content-type automatically for artifact /home/jarno/projects/incense/predictions_df.pickle.\\nAdded text/csv as content-type of artifact /home/jarno/projects/incense/predictions.csv.\\nAdded image/png as content-type of artifact /home/jarno/projects/incense/confusion_matrix.png.\\nAdded application/pdf as content-type of artifact /home/jarno/projects/incense/confusion_matrix.pdf.\\nINFO - matplotlib.animation - MovieWriter.run: running command: [\\'ffmpeg\\', \\'-f\\', \\'rawvideo\\', \\'-vcodec\\', \\'rawvideo\\', \\'-s\\', \\'3840x2880\\', \\'-pix_fmt\\', \\'rgba\\', \\'-r\\', \\'1\\', \\'-loglevel\\', \\'quiet\\', \\'-i\\', \\'pipe:\\', \\'-vcodec\\', \\'h264\\', \\'-pix_fmt\\', \\'yuv420p\\', \\'-y\\', \\'accuracy_movie.mp4\\']\\nAdded video/mp4 as content-type of artifact /home/jarno/projects/incense/accuracy_movie.mp4.\\nAdded text/plain as content-type of artifact /home/jarno/projects/incense/history.txt.\\nFailed to detect content-type automatically for artifact /home/jarno/projects/incense/model.hdf5.\\nINFO - example - Result: 0.9315000176429749\\nINFO - example - Completed after 0:00:19\\n',\n", + " 'config': {'seed': 0, 'optimizer': 'sgd', 'epochs': 3},\n", + " 'info': {'metrics': [{'name': 'training_loss',\n", + " 'id': '5caf9435eeb8baa519c5a8af'},\n", + " {'name': 'training_acc', 'id': '5caf9435eeb8baa519c5a8b1'},\n", + " {'name': 'test_loss', 'id': '5caf943feeb8baa519c5a8e8'},\n", + " {'name': 'test_acc', 'id': '5caf943feeb8baa519c5a8ea'}]},\n", " 'start_time': datetime.datetime(2019, 4, 11, 19, 23, 23, 890000),\n", - " 'config': {'epochs': 3, 'optimizer': 'sgd', 'seed': 0},\n", + " '_id': 2,\n", " 'meta': {'command': 'conduct',\n", - " 'options': {'--sql': None,\n", + " 'options': {'--capture': None,\n", " '--mongo_db': None,\n", - " '--name': None,\n", - " '--file_storage': None,\n", - " '--capture': None,\n", - " '--loglevel': None,\n", - " '--queue': False,\n", - " '--enforce_clean': False,\n", - " '--pdb': False,\n", " '--beat_interval': None,\n", + " '--help': False,\n", + " '--file_storage': None,\n", " '--comment': None,\n", - " '--print_config': False,\n", - " '--priority': None,\n", - " '--tiny_db': None,\n", " '--force': False,\n", " '--unobserved': False,\n", + " '--tiny_db': None,\n", + " '--queue': False,\n", + " '--loglevel': None,\n", + " '--sql': None,\n", + " '--pdb': False,\n", + " '--print_config': False,\n", " '--debug': False,\n", - " '--help': False}},\n", - " 'status': 'COMPLETED',\n", + " '--priority': None,\n", + " '--name': None,\n", + " '--enforce_clean': False}},\n", " 'resources': [],\n", " 'artifacts': [{'name': 'predictions_df',\n", " 'file_id': ObjectId('5caf943d3bd29849e342a7fe')},\n", @@ -280,15 +286,9 @@ " {'name': 'accuracy_movie', 'file_id': ObjectId('5caf943e3bd29849e342a806')},\n", " {'name': 'history', 'file_id': ObjectId('5caf943e3bd29849e342a808')},\n", " {'name': 'model.hdf5', 'file_id': ObjectId('5caf943e3bd29849e342a80a')}],\n", - " 'captured_out': 'INFO - example - Running command \\'conduct\\'\\nINFO - example - Started run with ID \"2\"\\nFailed to detect content-type automatically for artifact /home/jarno/projects/incense/predictions_df.pickle.\\nAdded text/csv as content-type of artifact /home/jarno/projects/incense/predictions.csv.\\nAdded image/png as content-type of artifact /home/jarno/projects/incense/confusion_matrix.png.\\nAdded application/pdf as content-type of artifact /home/jarno/projects/incense/confusion_matrix.pdf.\\nINFO - matplotlib.animation - MovieWriter.run: running command: [\\'ffmpeg\\', \\'-f\\', \\'rawvideo\\', \\'-vcodec\\', \\'rawvideo\\', \\'-s\\', \\'3840x2880\\', \\'-pix_fmt\\', \\'rgba\\', \\'-r\\', \\'1\\', \\'-loglevel\\', \\'quiet\\', \\'-i\\', \\'pipe:\\', \\'-vcodec\\', \\'h264\\', \\'-pix_fmt\\', \\'yuv420p\\', \\'-y\\', \\'accuracy_movie.mp4\\']\\nAdded video/mp4 as content-type of artifact /home/jarno/projects/incense/accuracy_movie.mp4.\\nAdded text/plain as content-type of artifact /home/jarno/projects/incense/history.txt.\\nFailed to detect content-type automatically for artifact /home/jarno/projects/incense/model.hdf5.\\nINFO - example - Result: 0.9315000176429749\\nINFO - example - Completed after 0:00:19\\n',\n", - " 'info': {'metrics': [{'id': '5caf9435eeb8baa519c5a8af',\n", - " 'name': 'training_loss'},\n", - " {'id': '5caf9435eeb8baa519c5a8b1', 'name': 'training_acc'},\n", - " {'id': '5caf943feeb8baa519c5a8e8', 'name': 'test_loss'},\n", - " {'id': '5caf943feeb8baa519c5a8ea', 'name': 'test_acc'}]},\n", " 'heartbeat': datetime.datetime(2019, 4, 11, 19, 23, 43, 148000),\n", - " 'result': 0.9315000176429749,\n", - " 'stop_time': datetime.datetime(2019, 4, 11, 19, 23, 43, 146000)}" + " 'command': 'conduct',\n", + " 'result': 0.9315000176429749}" ] }, "execution_count": 10, @@ -404,7 +404,7 @@ { "data": { "text/plain": [ - "{'epochs': 3, 'optimizer': 'sgd', 'seed': 0}" + "pmap({'seed': 0, 'optimizer': 'sgd', 'epochs': 3})" ] }, "execution_count": 15, @@ -880,7 +880,7 @@ { "data": { "text/plain": [ - "" + "" ] }, "execution_count": 26, @@ -912,7 +912,7 @@ { "data": { "text/plain": [ - "" + "" ] }, "execution_count": 27, @@ -1204,16 +1204,10 @@ "metadata": {}, "outputs": [ { - "ename": "StdinNotImplementedError", - "evalue": "raw_input was called, but this frontend does not support input requests.", - "output_type": "error", - "traceback": [ - "\u001b[0;31m---------------------------------------------------------------------------\u001b[0m", - "\u001b[0;31mStdinNotImplementedError\u001b[0m Traceback (most recent call last)", - "\u001b[0;32m\u001b[0m in \u001b[0;36m\u001b[0;34m\u001b[0m\n\u001b[1;32m 1\u001b[0m \u001b[0mexp\u001b[0m \u001b[0;34m=\u001b[0m \u001b[0mloader\u001b[0m\u001b[0;34m.\u001b[0m\u001b[0mfind_by_id\u001b[0m\u001b[0;34m(\u001b[0m\u001b[0;36m2\u001b[0m\u001b[0;34m)\u001b[0m\u001b[0;34m\u001b[0m\u001b[0;34m\u001b[0m\u001b[0m\n\u001b[0;32m----> 2\u001b[0;31m \u001b[0mexp\u001b[0m\u001b[0;34m.\u001b[0m\u001b[0mdelete\u001b[0m\u001b[0;34m(\u001b[0m\u001b[0;34m)\u001b[0m\u001b[0;34m\u001b[0m\u001b[0;34m\u001b[0m\u001b[0m\n\u001b[0m", - "\u001b[0;32m~/projects/incense/incense/experiment.py\u001b[0m in \u001b[0;36mdelete\u001b[0;34m(self, confirmed)\u001b[0m\n\u001b[1;32m 76\u001b[0m \"\"\"\n\u001b[1;32m 77\u001b[0m \u001b[0;32mif\u001b[0m \u001b[0;32mnot\u001b[0m \u001b[0mconfirmed\u001b[0m\u001b[0;34m:\u001b[0m\u001b[0;34m\u001b[0m\u001b[0;34m\u001b[0m\u001b[0m\n\u001b[0;32m---> 78\u001b[0;31m \u001b[0mconfirmed\u001b[0m \u001b[0;34m=\u001b[0m \u001b[0minput\u001b[0m\u001b[0;34m(\u001b[0m\u001b[0;34mf\"Are you sure you want to delete {self}? [y/N]\"\u001b[0m\u001b[0;34m)\u001b[0m \u001b[0;34m==\u001b[0m \u001b[0;34m\"y\"\u001b[0m\u001b[0;34m\u001b[0m\u001b[0;34m\u001b[0m\u001b[0m\n\u001b[0m\u001b[1;32m 79\u001b[0m \u001b[0;32mif\u001b[0m \u001b[0mconfirmed\u001b[0m\u001b[0;34m:\u001b[0m\u001b[0;34m\u001b[0m\u001b[0;34m\u001b[0m\u001b[0m\n\u001b[1;32m 80\u001b[0m \u001b[0mself\u001b[0m\u001b[0;34m.\u001b[0m\u001b[0m_delete\u001b[0m\u001b[0;34m(\u001b[0m\u001b[0;34m)\u001b[0m\u001b[0;34m\u001b[0m\u001b[0;34m\u001b[0m\u001b[0m\n", - "\u001b[0;32m~/.miniconda/envs/incense-dev/lib/python3.6/site-packages/ipykernel/kernelbase.py\u001b[0m in \u001b[0;36mraw_input\u001b[0;34m(self, prompt)\u001b[0m\n\u001b[1;32m 846\u001b[0m \u001b[0;32mif\u001b[0m \u001b[0;32mnot\u001b[0m \u001b[0mself\u001b[0m\u001b[0;34m.\u001b[0m\u001b[0m_allow_stdin\u001b[0m\u001b[0;34m:\u001b[0m\u001b[0;34m\u001b[0m\u001b[0;34m\u001b[0m\u001b[0m\n\u001b[1;32m 847\u001b[0m raise StdinNotImplementedError(\n\u001b[0;32m--> 848\u001b[0;31m \u001b[0;34m\"raw_input was called, but this frontend does not support input requests.\"\u001b[0m\u001b[0;34m\u001b[0m\u001b[0;34m\u001b[0m\u001b[0m\n\u001b[0m\u001b[1;32m 849\u001b[0m )\n\u001b[1;32m 850\u001b[0m return self._input_request(str(prompt),\n", - "\u001b[0;31mStdinNotImplementedError\u001b[0m: raw_input was called, but this frontend does not support input requests." + "name": "stdin", + "output_type": "stream", + "text": [ + "Are you sure you want to delete Experiment(id=2, name=example)? [y/N] N\n" ] } ], diff --git a/incense/experiment.py b/incense/experiment.py index 8522304..fc13a58 100755 --- a/incense/experiment.py +++ b/incense/experiment.py @@ -2,7 +2,7 @@ from typing import * import pandas as pd -from easydict import EasyDict +from pyrsistent import freeze, thaw from incense.artifact import Artifact, content_type_to_artifact_cls @@ -27,7 +27,8 @@ def __getattr__(self, item): @classmethod def from_db_object(cls, database, grid_filesystem, experiment_data: dict, loader): - data = EasyDict(experiment_data) + data = freeze(experiment_data) + artifacts_links = experiment_data["artifacts"] id_ = experiment_data["_id"] return cls(id_, database, grid_filesystem, data, artifacts_links, loader) @@ -66,7 +67,7 @@ def to_dict(self) -> dict: Returns: A dict with all data from the sacred data model. """ - return dict(zip(self.keys(), self.values())) + return thaw(self._data) def delete(self, confirmed: bool = False): """Delete run together with its artifacts and metrics. diff --git a/requirements-dev.txt b/requirements-dev.txt index 645a5db..0148836 100755 --- a/requirements-dev.txt +++ b/requirements-dev.txt @@ -2,9 +2,10 @@ pytest pytest-cov codecov -tensorflow +tensorflow==1.13.1 python-dotenv scikit-learn jupyterlab seaborn -pre-commit \ No newline at end of file +pre-commit +tokenize-rt==2.2.0 \ No newline at end of file diff --git a/setup.py b/setup.py index ad0f794..5aa6b96 100644 --- a/setup.py +++ b/setup.py @@ -27,8 +27,9 @@ "pandas>=0.23", "jupyterlab>=0.35", "pymongo>=3.7", - "easydict>=1.9", + "pyrsistent>=0.15.2", "future-fstrings==1.0.0", + "tokenize-rt==2.2.0", ], include_package_data=True, classifiers=[ diff --git a/tests/test_experiment.py b/tests/test_experiment.py index 6772495..bb65353 100644 --- a/tests/test_experiment.py +++ b/tests/test_experiment.py @@ -50,7 +50,9 @@ def test_to_dict(loader): exp = loader.find_by_id(2) exp_dict = exp.to_dict() assert isinstance(exp_dict, dict) - assert exp.keys() == exp_dict.keys() + # Sort for py35 compatibility. + for x, y in zip(sorted(exp.keys()), sorted(exp_dict.keys())): + assert x == y def test_delete(delete_db_loader, mongo_observer): @@ -83,3 +85,15 @@ def test_delete_prompt(loader, monkeypatch): loader.find_by_id.cache_clear() exp = loader.find_by_id(1) assert exp.id == 1 + + +def test_immutability__should_raise_type_error_on_item_access(loader): + exp = loader.find_by_id(2) + with raises(TypeError): + exp.meta["command"] = "mutate" + + +def test_immutability__should_raise_attribute_error_on_attribute_access(loader): + exp = loader.find_by_id(2) + with raises(AttributeError): + exp.meta.command = "mutate"