Skip to content

Commit

Permalink
Use pyristent to store internal data (#42)
Browse files Browse the repository at this point in the history 
* Use pyristent to store internal data

* Pin tf version

* Add toknize requirement

* Update tf version

* Add tokenize to dev reqs

* Fix test for py35
  • Loading branch information
@JarnoRFB
JarnoRFB authored Jun 27, 2019
1 parent 2581127 commit fd54a9d
Show file tree
Hide file tree
Showing 5 changed files with 63 additions and 52 deletions.
84 changes: 39 additions & 45 deletions demo.ipynb
View file
Original file line number Diff line number Diff line change
Expand Up @@ -227,48 +227,54 @@
{
"data": {
"text/plain": [
"{'_id': 2,\n",
" 'experiment': {'name': 'example',\n",
"{'stop_time': datetime.datetime(2019, 4, 11, 19, 23, 43, 146000),\n",
" 'experiment': {'repositories': [],\n",
" 'base_dir': '/home/jarno/projects/incense/example_experiment',\n",
" 'sources': [['conduct.py', ObjectId('5caf94233bd29849e342a7e2')]],\n",
" 'name': 'example',\n",
" 'dependencies': ['matplotlib==3.0.2',\n",
" 'numpy==1.16.2',\n",
" 'pandas==0.24.1',\n",
" 'sacred==0.7.4-onurgu',\n",
" 'scikit-learn==0.20.3',\n",
" 'seaborn==0.9.0',\n",
" 'tensorflow==1.13.1'],\n",
" 'repositories': [],\n",
" 'mainfile': 'conduct.py'},\n",
" 'mainfile': 'conduct.py',\n",
" 'sources': [['conduct.py', ObjectId('5caf94233bd29849e342a7e2')]]},\n",
" 'format': 'MongoObserver-0.7.0',\n",
" 'command': 'conduct',\n",
" 'host': {'hostname': 'work',\n",
" 'host': {'cpu': 'Intel(R) Core(TM) i7-4800MQ CPU @ 2.70GHz',\n",
" 'ENV': {},\n",
" 'hostname': 'work',\n",
" 'os': ['Linux', 'Linux-4.18.0-17-generic-x86_64-with-debian-buster-sid'],\n",
" 'python_version': '3.6.8',\n",
" 'cpu': 'Intel(R) Core(TM) i7-4800MQ CPU @ 2.70GHz',\n",
" 'ENV': {}},\n",
" 'python_version': '3.6.8'},\n",
" 'status': 'COMPLETED',\n",
" 'captured_out': 'INFO - example - Running command \\'conduct\\'\\nINFO - example - Started run with ID \"2\"\\nFailed to detect content-type automatically for artifact /home/jarno/projects/incense/predictions_df.pickle.\\nAdded text/csv as content-type of artifact /home/jarno/projects/incense/predictions.csv.\\nAdded image/png as content-type of artifact /home/jarno/projects/incense/confusion_matrix.png.\\nAdded application/pdf as content-type of artifact /home/jarno/projects/incense/confusion_matrix.pdf.\\nINFO - matplotlib.animation - MovieWriter.run: running command: [\\'ffmpeg\\', \\'-f\\', \\'rawvideo\\', \\'-vcodec\\', \\'rawvideo\\', \\'-s\\', \\'3840x2880\\', \\'-pix_fmt\\', \\'rgba\\', \\'-r\\', \\'1\\', \\'-loglevel\\', \\'quiet\\', \\'-i\\', \\'pipe:\\', \\'-vcodec\\', \\'h264\\', \\'-pix_fmt\\', \\'yuv420p\\', \\'-y\\', \\'accuracy_movie.mp4\\']\\nAdded video/mp4 as content-type of artifact /home/jarno/projects/incense/accuracy_movie.mp4.\\nAdded text/plain as content-type of artifact /home/jarno/projects/incense/history.txt.\\nFailed to detect content-type automatically for artifact /home/jarno/projects/incense/model.hdf5.\\nINFO - example - Result: 0.9315000176429749\\nINFO - example - Completed after 0:00:19\\n',\n",
" 'config': {'seed': 0, 'optimizer': 'sgd', 'epochs': 3},\n",
" 'info': {'metrics': [{'name': 'training_loss',\n",
" 'id': '5caf9435eeb8baa519c5a8af'},\n",
" {'name': 'training_acc', 'id': '5caf9435eeb8baa519c5a8b1'},\n",
" {'name': 'test_loss', 'id': '5caf943feeb8baa519c5a8e8'},\n",
" {'name': 'test_acc', 'id': '5caf943feeb8baa519c5a8ea'}]},\n",
" 'start_time': datetime.datetime(2019, 4, 11, 19, 23, 23, 890000),\n",
" 'config': {'epochs': 3, 'optimizer': 'sgd', 'seed': 0},\n",
" '_id': 2,\n",
" 'meta': {'command': 'conduct',\n",
" 'options': {'--sql': None,\n",
" 'options': {'--capture': None,\n",
" '--mongo_db': None,\n",
" '--name': None,\n",
" '--file_storage': None,\n",
" '--capture': None,\n",
" '--loglevel': None,\n",
" '--queue': False,\n",
" '--enforce_clean': False,\n",
" '--pdb': False,\n",
" '--beat_interval': None,\n",
" '--help': False,\n",
" '--file_storage': None,\n",
" '--comment': None,\n",
" '--print_config': False,\n",
" '--priority': None,\n",
" '--tiny_db': None,\n",
" '--force': False,\n",
" '--unobserved': False,\n",
" '--tiny_db': None,\n",
" '--queue': False,\n",
" '--loglevel': None,\n",
" '--sql': None,\n",
" '--pdb': False,\n",
" '--print_config': False,\n",
" '--debug': False,\n",
" '--help': False}},\n",
" 'status': 'COMPLETED',\n",
" '--priority': None,\n",
" '--name': None,\n",
" '--enforce_clean': False}},\n",
" 'resources': [],\n",
" 'artifacts': [{'name': 'predictions_df',\n",
" 'file_id': ObjectId('5caf943d3bd29849e342a7fe')},\n",
Expand All @@ -280,15 +286,9 @@
" {'name': 'accuracy_movie', 'file_id': ObjectId('5caf943e3bd29849e342a806')},\n",
" {'name': 'history', 'file_id': ObjectId('5caf943e3bd29849e342a808')},\n",
" {'name': 'model.hdf5', 'file_id': ObjectId('5caf943e3bd29849e342a80a')}],\n",
" 'captured_out': 'INFO - example - Running command \\'conduct\\'\\nINFO - example - Started run with ID \"2\"\\nFailed to detect content-type automatically for artifact /home/jarno/projects/incense/predictions_df.pickle.\\nAdded text/csv as content-type of artifact /home/jarno/projects/incense/predictions.csv.\\nAdded image/png as content-type of artifact /home/jarno/projects/incense/confusion_matrix.png.\\nAdded application/pdf as content-type of artifact /home/jarno/projects/incense/confusion_matrix.pdf.\\nINFO - matplotlib.animation - MovieWriter.run: running command: [\\'ffmpeg\\', \\'-f\\', \\'rawvideo\\', \\'-vcodec\\', \\'rawvideo\\', \\'-s\\', \\'3840x2880\\', \\'-pix_fmt\\', \\'rgba\\', \\'-r\\', \\'1\\', \\'-loglevel\\', \\'quiet\\', \\'-i\\', \\'pipe:\\', \\'-vcodec\\', \\'h264\\', \\'-pix_fmt\\', \\'yuv420p\\', \\'-y\\', \\'accuracy_movie.mp4\\']\\nAdded video/mp4 as content-type of artifact /home/jarno/projects/incense/accuracy_movie.mp4.\\nAdded text/plain as content-type of artifact /home/jarno/projects/incense/history.txt.\\nFailed to detect content-type automatically for artifact /home/jarno/projects/incense/model.hdf5.\\nINFO - example - Result: 0.9315000176429749\\nINFO - example - Completed after 0:00:19\\n',\n",
" 'info': {'metrics': [{'id': '5caf9435eeb8baa519c5a8af',\n",
" 'name': 'training_loss'},\n",
" {'id': '5caf9435eeb8baa519c5a8b1', 'name': 'training_acc'},\n",
" {'id': '5caf943feeb8baa519c5a8e8', 'name': 'test_loss'},\n",
" {'id': '5caf943feeb8baa519c5a8ea', 'name': 'test_acc'}]},\n",
" 'heartbeat': datetime.datetime(2019, 4, 11, 19, 23, 43, 148000),\n",
" 'result': 0.9315000176429749,\n",
" 'stop_time': datetime.datetime(2019, 4, 11, 19, 23, 43, 146000)}"
" 'command': 'conduct',\n",
" 'result': 0.9315000176429749}"
]
},
"execution_count": 10,
Expand Down Expand Up @@ -404,7 +404,7 @@
{
"data": {
"text/plain": [
"{'epochs': 3, 'optimizer': 'sgd', 'seed': 0}"
"pmap({'seed': 0, 'optimizer': 'sgd', 'epochs': 3})"
]
},
"execution_count": 15,
Expand Down Expand Up @@ -880,7 +880,7 @@
{
"data": {
"text/plain": [
"<matplotlib.axes._subplots.AxesSubplot at 0x7f93c14c6748>"
"<matplotlib.axes._subplots.AxesSubplot at 0x7f3b31795b70>"
]
},
"execution_count": 26,
Expand Down Expand Up @@ -912,7 +912,7 @@
{
"data": {
"text/plain": [
"<matplotlib.legend.Legend at 0x7f93c0418668>"
"<matplotlib.legend.Legend at 0x7f3b30671e80>"
]
},
"execution_count": 27,
Expand Down Expand Up @@ -1204,16 +1204,10 @@
"metadata": {},
"outputs": [
{
"ename": "StdinNotImplementedError",
"evalue": "raw_input was called, but this frontend does not support input requests.",
"output_type": "error",
"traceback": [
"\u001b[0;31m---------------------------------------------------------------------------\u001b[0m",
"\u001b[0;31mStdinNotImplementedError\u001b[0m Traceback (most recent call last)",
"\u001b[0;32m<ipython-input-34-1005ee59ea02>\u001b[0m in \u001b[0;36m<module>\u001b[0;34m\u001b[0m\n\u001b[1;32m 1\u001b[0m \u001b[0mexp\u001b[0m \u001b[0;34m=\u001b[0m \u001b[0mloader\u001b[0m\u001b[0;34m.\u001b[0m\u001b[0mfind_by_id\u001b[0m\u001b[0;34m(\u001b[0m\u001b[0;36m2\u001b[0m\u001b[0;34m)\u001b[0m\u001b[0;34m\u001b[0m\u001b[0;34m\u001b[0m\u001b[0m\n\u001b[0;32m----> 2\u001b[0;31m \u001b[0mexp\u001b[0m\u001b[0;34m.\u001b[0m\u001b[0mdelete\u001b[0m\u001b[0;34m(\u001b[0m\u001b[0;34m)\u001b[0m\u001b[0;34m\u001b[0m\u001b[0;34m\u001b[0m\u001b[0m\n\u001b[0m",
"\u001b[0;32m~/projects/incense/incense/experiment.py\u001b[0m in \u001b[0;36mdelete\u001b[0;34m(self, confirmed)\u001b[0m\n\u001b[1;32m 76\u001b[0m \"\"\"\n\u001b[1;32m 77\u001b[0m \u001b[0;32mif\u001b[0m \u001b[0;32mnot\u001b[0m \u001b[0mconfirmed\u001b[0m\u001b[0;34m:\u001b[0m\u001b[0;34m\u001b[0m\u001b[0;34m\u001b[0m\u001b[0m\n\u001b[0;32m---> 78\u001b[0;31m \u001b[0mconfirmed\u001b[0m \u001b[0;34m=\u001b[0m \u001b[0minput\u001b[0m\u001b[0;34m(\u001b[0m\u001b[0;34mf\"Are you sure you want to delete {self}? [y/N]\"\u001b[0m\u001b[0;34m)\u001b[0m \u001b[0;34m==\u001b[0m \u001b[0;34m\"y\"\u001b[0m\u001b[0;34m\u001b[0m\u001b[0;34m\u001b[0m\u001b[0m\n\u001b[0m\u001b[1;32m 79\u001b[0m \u001b[0;32mif\u001b[0m \u001b[0mconfirmed\u001b[0m\u001b[0;34m:\u001b[0m\u001b[0;34m\u001b[0m\u001b[0;34m\u001b[0m\u001b[0m\n\u001b[1;32m 80\u001b[0m \u001b[0mself\u001b[0m\u001b[0;34m.\u001b[0m\u001b[0m_delete\u001b[0m\u001b[0;34m(\u001b[0m\u001b[0;34m)\u001b[0m\u001b[0;34m\u001b[0m\u001b[0;34m\u001b[0m\u001b[0m\n",
"\u001b[0;32m~/.miniconda/envs/incense-dev/lib/python3.6/site-packages/ipykernel/kernelbase.py\u001b[0m in \u001b[0;36mraw_input\u001b[0;34m(self, prompt)\u001b[0m\n\u001b[1;32m 846\u001b[0m \u001b[0;32mif\u001b[0m \u001b[0;32mnot\u001b[0m \u001b[0mself\u001b[0m\u001b[0;34m.\u001b[0m\u001b[0m_allow_stdin\u001b[0m\u001b[0;34m:\u001b[0m\u001b[0;34m\u001b[0m\u001b[0;34m\u001b[0m\u001b[0m\n\u001b[1;32m 847\u001b[0m raise StdinNotImplementedError(\n\u001b[0;32m--> 848\u001b[0;31m \u001b[0;34m\"raw_input was called, but this frontend does not support input requests.\"\u001b[0m\u001b[0;34m\u001b[0m\u001b[0;34m\u001b[0m\u001b[0m\n\u001b[0m\u001b[1;32m 849\u001b[0m )\n\u001b[1;32m 850\u001b[0m return self._input_request(str(prompt),\n",
"\u001b[0;31mStdinNotImplementedError\u001b[0m: raw_input was called, but this frontend does not support input requests."
"name": "stdin",
"output_type": "stream",
"text": [
"Are you sure you want to delete Experiment(id=2, name=example)? [y/N] N\n"
]
}
],
Expand Down
7 changes: 4 additions & 3 deletions incense/experiment.py
View file
Original file line number Diff line number Diff line change
Expand Up @@ -2,7 +2,7 @@
from typing import *

import pandas as pd
from easydict import EasyDict
from pyrsistent import freeze, thaw

from incense.artifact import Artifact, content_type_to_artifact_cls

Expand All @@ -27,7 +27,8 @@ def __getattr__(self, item):

@classmethod
def from_db_object(cls, database, grid_filesystem, experiment_data: dict, loader):
data = EasyDict(experiment_data)
data = freeze(experiment_data)

artifacts_links = experiment_data["artifacts"]
id_ = experiment_data["_id"]
return cls(id_, database, grid_filesystem, data, artifacts_links, loader)
Expand Down Expand Up @@ -66,7 +67,7 @@ def to_dict(self) -> dict:
Returns:
A dict with all data from the sacred data model.
"""
return dict(zip(self.keys(), self.values()))
return thaw(self._data)

def delete(self, confirmed: bool = False):
"""Delete run together with its artifacts and metrics.
Expand Down
5 changes: 3 additions & 2 deletions requirements-dev.txt
View file
Original file line number Diff line number Diff line change
Expand Up @@ -2,9 +2,10 @@
pytest
pytest-cov
codecov
tensorflow
tensorflow==1.13.1
python-dotenv
scikit-learn
jupyterlab
seaborn
pre-commit
pre-commit
tokenize-rt==2.2.0
3 changes: 2 additions & 1 deletion setup.py
View file
Original file line number Diff line number Diff line change
Expand Up @@ -27,8 +27,9 @@
"pandas>=0.23",
"jupyterlab>=0.35",
"pymongo>=3.7",
"easydict>=1.9",
"pyrsistent>=0.15.2",
"future-fstrings==1.0.0",
"tokenize-rt==2.2.0",
],
include_package_data=True,
classifiers=[
Expand Down
16 changes: 15 additions & 1 deletion tests/test_experiment.py
View file
Original file line number Diff line number Diff line change
Expand Up @@ -50,7 +50,9 @@ def test_to_dict(loader):
exp = loader.find_by_id(2)
exp_dict = exp.to_dict()
assert isinstance(exp_dict, dict)
assert exp.keys() == exp_dict.keys()
# Sort for py35 compatibility.
for x, y in zip(sorted(exp.keys()), sorted(exp_dict.keys())):
assert x == y


def test_delete(delete_db_loader, mongo_observer):
Expand Down Expand Up @@ -83,3 +85,15 @@ def test_delete_prompt(loader, monkeypatch):
loader.find_by_id.cache_clear()
exp = loader.find_by_id(1)
assert exp.id == 1


def test_immutability__should_raise_type_error_on_item_access(loader):
exp = loader.find_by_id(2)
with raises(TypeError):
exp.meta["command"] = "mutate"


def test_immutability__should_raise_attribute_error_on_attribute_access(loader):
exp = loader.find_by_id(2)
with raises(AttributeError):
exp.meta.command = "mutate"

0 comments on commit fd54a9d

Please sign in to comment.