diff --git a/CHANGELOG.md b/CHANGELOG.md index 04104e0..f6425ee 100644 --- a/CHANGELOG.md +++ b/CHANGELOG.md @@ -3,9 +3,9 @@ Observes [Semantic Versioning](https://semver.org/spec/v2.0.0.html) standard and [Keep a Changelog](https://keepachangelog.com/en/1.0.0/) convention. -## [0.1.3] - 2022-09-20 +## [0.1.4] - 2022-09-20 -+ Add - Notebook for Allen Institute workshop ++ Add - Add (0.1.3) and revise (0.1.4) notebook for Allen Institute workshop ## [0.1.2] - 2022-09-19 @@ -43,6 +43,7 @@ Observes [Semantic Versioning](https://semver.org/spec/v2.0.0.html) standard and + Add - Containerization for pytests + Comment - Phase previously designated 0.1.0 -> 0.0.0 +[0.1.4]: https://github.com/datajoint/workflow-calcium-imaging/releases/tag/0.1.4 [0.1.3]: https://github.com/datajoint/workflow-calcium-imaging/releases/tag/0.1.3 [0.1.2]: https://github.com/datajoint/workflow-calcium-imaging/releases/tag/0.1.2 [0.1.1]: https://github.com/datajoint/workflow-calcium-imaging/releases/tag/0.1.1 diff --git a/notebooks/2022-allen-institute-workshop.ipynb b/notebooks/2022-allen-institute-workshop.ipynb index 259b742..9d607ba 100644 --- a/notebooks/2022-allen-institute-workshop.ipynb +++ b/notebooks/2022-allen-institute-workshop.ipynb @@ -13,7 +13,21 @@ "\n", "+ This notebook is meant to be run on CodeBook (`https://codebook.datajoint.io`) which contains example data.\n", "\n", - "+ First run the `01-configure` and `04-automate` notebooks to set up your environment and load example data into the database, respectively." + "First, some packages we'll use in this notebook..." + ] + }, + { + "cell_type": "code", + "execution_count": null, + "id": "e1e0295c-57bf-49d2-9a1e-931d0e53264b", + "metadata": {}, + "outputs": [], + "source": [ + "import datajoint as dj \n", + "import numpy as np\n", + "from matplotlib import pyplot\n", + "import os\n", + "import getpass" ] }, { @@ -21,22 +35,15 @@ "id": "14a6ba1d", "metadata": {}, "source": [ - "## Configuration\n", - "\n", - "Import the relevant packages." + "## Configuration" ] }, { - "cell_type": "code", - "execution_count": null, - "id": "e1e0295c-57bf-49d2-9a1e-931d0e53264b", + "cell_type": "markdown", + "id": "cb03797f", "metadata": {}, - "outputs": [], "source": [ - "import datajoint as dj\n", - "import numpy as np\n", - "from matplotlib import pyplot\n", - "import os" + "These steps are taken from [01-configure](01-configure.ipynb). If you've already saved a config file, you can skip to the next section." ] }, { @@ -56,16 +63,253 @@ }, "outputs": [], "source": [ - "dj.config['custom'] = {'database.prefix': '_allen_ophys_',\n", - " 'imaging_root_data_dir': '/home/inbox/0_1_0a2/'}" + "username_as_prefix = dj.config[\"database.user\"] + \"_img_\"\n", + "dj.config['custom'] = {\n", + " 'database.prefix': username_as_prefix,\n", + " 'imaging_root_data_dir': '/home/'\n", + "}" ] }, { "cell_type": "markdown", - "id": "7c545770", + "id": "21a69c02", "metadata": {}, "source": [ - "Import the workflow. The current workflow is composed of multiple database schemas, each of them corresponding to a module within the `workflow_calcium_imaging.pipeline` file." + "Next, we'll use a prompt to securely save your password." + ] + }, + { + "cell_type": "code", + "execution_count": null, + "id": "66bea7cb", + "metadata": {}, + "outputs": [], + "source": [ + "dj.config[\"database.password\"] = getpass.getpass()" + ] + }, + { + "cell_type": "markdown", + "id": "c45a3aad", + "metadata": {}, + "source": [ + "Now to save these credentials." + ] + }, + { + "cell_type": "code", + "execution_count": null, + "id": "4e4f0448", + "metadata": {}, + "outputs": [], + "source": [ + "dj.config.save_global()" + ] + }, + { + "cell_type": "markdown", + "id": "a570822d", + "metadata": {}, + "source": [ + "## Populating the database" + ] + }, + { + "cell_type": "markdown", + "id": "8ee61c1d", + "metadata": {}, + "source": [ + "Next, we'll populate these schema using some steps from [04-automate](04-automate-optional.ipynb). If your schema are already populated, you can skip this step. For more details on each of these steps, please visit [that notebook](04-automate-optional.ipynb). Additional steps ensure write permissions on output directories." + ] + }, + { + "cell_type": "code", + "execution_count": 1, + "id": "bc01b55c", + "metadata": {}, + "outputs": [ + { + "name": "stderr", + "output_type": "stream", + "text": [ + "[2022-09-21 14:04:13,920][INFO]: Connecting cbroz@dss-db.datajoint.io:3306\n", + "[2022-09-21 14:04:14,701][INFO]: Connected cbroz@dss-db.datajoint.io:3306\n" + ] + }, + { + "ename": "KeyboardInterrupt", + "evalue": "", + "output_type": "error", + "traceback": [ + "\u001b[0;31m---------------------------------------------------------------------------\u001b[0m", + "\u001b[0;31mKeyboardInterrupt\u001b[0m Traceback (most recent call last)", + "Cell \u001b[0;32mIn [1], line 1\u001b[0m\n\u001b[0;32m----> 1\u001b[0m \u001b[38;5;28;01mfrom\u001b[39;00m \u001b[38;5;21;01mworkflow_calcium_imaging\u001b[39;00m\u001b[38;5;21;01m.\u001b[39;00m\u001b[38;5;21;01mpipeline\u001b[39;00m \u001b[38;5;28;01mimport\u001b[39;00m session, imaging \u001b[38;5;66;03m# import schemas\u001b[39;00m\n\u001b[1;32m 2\u001b[0m \u001b[38;5;28;01mfrom\u001b[39;00m \u001b[38;5;21;01mworkflow_calcium_imaging\u001b[39;00m\u001b[38;5;21;01m.\u001b[39;00m\u001b[38;5;21;01mingest\u001b[39;00m \u001b[38;5;28;01mimport\u001b[39;00m ingest_subjects, ingest_sessions \u001b[38;5;66;03m#csv loaders\u001b[39;00m\n\u001b[1;32m 4\u001b[0m \u001b[38;5;28;01mimport\u001b[39;00m \u001b[38;5;21;01mcsv\u001b[39;00m\n", + "File \u001b[0;32m~/Documents/dev/workflow-calcium-imaging/workflow_calcium_imaging/pipeline.py:74\u001b[0m\n\u001b[1;32m 67\u001b[0m definition \u001b[39m=\u001b[39m \u001b[39m\"\"\"\u001b[39m\n\u001b[1;32m 68\u001b[0m \u001b[39m scanner: varchar(32)\u001b[39m\n\u001b[1;32m 69\u001b[0m \u001b[39m \u001b[39m\u001b[39m\"\"\"\u001b[39m\n\u001b[1;32m 72\u001b[0m \u001b[39m# ------------- Activate \"imaging\" schema -------------\u001b[39;00m\n\u001b[0;32m---> 74\u001b[0m imaging\u001b[39m.\u001b[39;49mactivate(db_prefix \u001b[39m+\u001b[39;49m \u001b[39m\"\u001b[39;49m\u001b[39mimaging\u001b[39;49m\u001b[39m\"\u001b[39;49m, db_prefix \u001b[39m+\u001b[39;49m \u001b[39m\"\u001b[39;49m\u001b[39mscan\u001b[39;49m\u001b[39m\"\u001b[39;49m, linking_module\u001b[39m=\u001b[39;49m\u001b[39m__name__\u001b[39;49m)\n\u001b[1;32m 76\u001b[0m \u001b[39m# ------------- Activate \"analysis\" schema ------------\u001b[39;00m\n\u001b[1;32m 78\u001b[0m analysis\u001b[39m.\u001b[39mactivate(db_prefix \u001b[39m+\u001b[39m \u001b[39m\"\u001b[39m\u001b[39manalysis\u001b[39m\u001b[39m\"\u001b[39m, linking_module\u001b[39m=\u001b[39m\u001b[39m__name__\u001b[39m)\n", + "File \u001b[0;32m~/Documents/dev/element-calcium-imaging/element_calcium_imaging/imaging.py:58\u001b[0m, in \u001b[0;36mactivate\u001b[0;34m(imaging_schema_name, scan_schema_name, create_schema, create_tables, linking_module)\u001b[0m\n\u001b[1;32m 50\u001b[0m _linking_module \u001b[39m=\u001b[39m linking_module\n\u001b[1;32m 52\u001b[0m scan\u001b[39m.\u001b[39mactivate(\n\u001b[1;32m 53\u001b[0m scan_schema_name,\n\u001b[1;32m 54\u001b[0m create_schema\u001b[39m=\u001b[39mcreate_schema,\n\u001b[1;32m 55\u001b[0m create_tables\u001b[39m=\u001b[39mcreate_tables,\n\u001b[1;32m 56\u001b[0m linking_module\u001b[39m=\u001b[39mlinking_module,\n\u001b[1;32m 57\u001b[0m )\n\u001b[0;32m---> 58\u001b[0m schema\u001b[39m.\u001b[39;49mactivate(\n\u001b[1;32m 59\u001b[0m imaging_schema_name,\n\u001b[1;32m 60\u001b[0m create_schema\u001b[39m=\u001b[39;49mcreate_schema,\n\u001b[1;32m 61\u001b[0m create_tables\u001b[39m=\u001b[39;49mcreate_tables,\n\u001b[1;32m 62\u001b[0m add_objects\u001b[39m=\u001b[39;49m_linking_module\u001b[39m.\u001b[39;49m\u001b[39m__dict__\u001b[39;49m,\n\u001b[1;32m 63\u001b[0m )\n", + "File \u001b[0;32m~/miniforge3/envs/ele/lib/python3.9/site-packages/datajoint/schemas.py:155\u001b[0m, in \u001b[0;36mSchema.activate\u001b[0;34m(self, schema_name, connection, create_schema, create_tables, add_objects)\u001b[0m\n\u001b[1;32m 153\u001b[0m \u001b[39mif\u001b[39;00m \u001b[39mself\u001b[39m\u001b[39m.\u001b[39madd_objects:\n\u001b[1;32m 154\u001b[0m context \u001b[39m=\u001b[39m \u001b[39mdict\u001b[39m(context, \u001b[39m*\u001b[39m\u001b[39m*\u001b[39m\u001b[39mself\u001b[39m\u001b[39m.\u001b[39madd_objects)\n\u001b[0;32m--> 155\u001b[0m \u001b[39mself\u001b[39;49m\u001b[39m.\u001b[39;49m_decorate_master(\u001b[39mcls\u001b[39;49m, context)\n", + "File \u001b[0;32m~/miniforge3/envs/ele/lib/python3.9/site-packages/datajoint/schemas.py:198\u001b[0m, in \u001b[0;36mSchema._decorate_master\u001b[0;34m(self, cls, context)\u001b[0m\n\u001b[1;32m 196\u001b[0m part\u001b[39m.\u001b[39m_master \u001b[39m=\u001b[39m \u001b[39mcls\u001b[39m\n\u001b[1;32m 197\u001b[0m \u001b[39m# allow addressing master by name or keyword 'master'\u001b[39;00m\n\u001b[0;32m--> 198\u001b[0m \u001b[39mself\u001b[39;49m\u001b[39m.\u001b[39;49m_decorate_table(\n\u001b[1;32m 199\u001b[0m part,\n\u001b[1;32m 200\u001b[0m context\u001b[39m=\u001b[39;49m\u001b[39mdict\u001b[39;49m(\n\u001b[1;32m 201\u001b[0m context, master\u001b[39m=\u001b[39;49m\u001b[39mcls\u001b[39;49m, \u001b[39mself\u001b[39;49m\u001b[39m=\u001b[39;49mpart, \u001b[39m*\u001b[39;49m\u001b[39m*\u001b[39;49m{\u001b[39mcls\u001b[39;49m\u001b[39m.\u001b[39;49m\u001b[39m__name__\u001b[39;49m: \u001b[39mcls\u001b[39;49m}\n\u001b[1;32m 202\u001b[0m ),\n\u001b[1;32m 203\u001b[0m )\n", + "File \u001b[0;32m~/miniforge3/envs/ele/lib/python3.9/site-packages/datajoint/schemas.py:224\u001b[0m, in \u001b[0;36mSchema._decorate_table\u001b[0;34m(self, table_class, context, assert_declared)\u001b[0m\n\u001b[1;32m 222\u001b[0m \u001b[39m# instantiate the class, declare the table if not already\u001b[39;00m\n\u001b[1;32m 223\u001b[0m instance \u001b[39m=\u001b[39m table_class()\n\u001b[0;32m--> 224\u001b[0m is_declared \u001b[39m=\u001b[39m instance\u001b[39m.\u001b[39;49mis_declared\n\u001b[1;32m 225\u001b[0m \u001b[39mif\u001b[39;00m \u001b[39mnot\u001b[39;00m is_declared:\n\u001b[1;32m 226\u001b[0m \u001b[39mif\u001b[39;00m \u001b[39mnot\u001b[39;00m \u001b[39mself\u001b[39m\u001b[39m.\u001b[39mcreate_tables \u001b[39mor\u001b[39;00m assert_declared:\n", + "File \u001b[0;32m~/miniforge3/envs/ele/lib/python3.9/site-packages/datajoint/table.py:247\u001b[0m, in \u001b[0;36mTable.is_declared\u001b[0;34m(self)\u001b[0m\n\u001b[1;32m 241\u001b[0m \u001b[39m@property\u001b[39m\n\u001b[1;32m 242\u001b[0m \u001b[39mdef\u001b[39;00m \u001b[39mis_declared\u001b[39m(\u001b[39mself\u001b[39m):\n\u001b[1;32m 243\u001b[0m \u001b[39m\"\"\"\u001b[39;00m\n\u001b[1;32m 244\u001b[0m \u001b[39m :return: True is the table is declared in the schema.\u001b[39;00m\n\u001b[1;32m 245\u001b[0m \u001b[39m \"\"\"\u001b[39;00m\n\u001b[1;32m 246\u001b[0m \u001b[39mreturn\u001b[39;00m (\n\u001b[0;32m--> 247\u001b[0m \u001b[39mself\u001b[39;49m\u001b[39m.\u001b[39;49mconnection\u001b[39m.\u001b[39;49mquery(\n\u001b[1;32m 248\u001b[0m \u001b[39m'\u001b[39;49m\u001b[39mSHOW TABLES in `\u001b[39;49m\u001b[39m{database}\u001b[39;49;00m\u001b[39m` LIKE \u001b[39;49m\u001b[39m\"\u001b[39;49m\u001b[39m{table_name}\u001b[39;49;00m\u001b[39m\"\u001b[39;49m\u001b[39m'\u001b[39;49m\u001b[39m.\u001b[39;49mformat(\n\u001b[1;32m 249\u001b[0m database\u001b[39m=\u001b[39;49m\u001b[39mself\u001b[39;49m\u001b[39m.\u001b[39;49mdatabase, table_name\u001b[39m=\u001b[39;49m\u001b[39mself\u001b[39;49m\u001b[39m.\u001b[39;49mtable_name\n\u001b[1;32m 250\u001b[0m )\n\u001b[1;32m 251\u001b[0m )\u001b[39m.\u001b[39mrowcount\n\u001b[1;32m 252\u001b[0m \u001b[39m>\u001b[39m \u001b[39m0\u001b[39m\n\u001b[1;32m 253\u001b[0m )\n", + "File \u001b[0;32m~/miniforge3/envs/ele/lib/python3.9/site-packages/datajoint/connection.py:340\u001b[0m, in \u001b[0;36mConnection.query\u001b[0;34m(self, query, args, as_dict, suppress_warnings, reconnect)\u001b[0m\n\u001b[1;32m 338\u001b[0m cursor \u001b[39m=\u001b[39m \u001b[39mself\u001b[39m\u001b[39m.\u001b[39m_conn\u001b[39m.\u001b[39mcursor(cursor\u001b[39m=\u001b[39mcursor_class)\n\u001b[1;32m 339\u001b[0m \u001b[39mtry\u001b[39;00m:\n\u001b[0;32m--> 340\u001b[0m \u001b[39mself\u001b[39;49m\u001b[39m.\u001b[39;49m_execute_query(cursor, query, args, suppress_warnings)\n\u001b[1;32m 341\u001b[0m \u001b[39mexcept\u001b[39;00m errors\u001b[39m.\u001b[39mLostConnectionError:\n\u001b[1;32m 342\u001b[0m \u001b[39mif\u001b[39;00m \u001b[39mnot\u001b[39;00m reconnect:\n", + "File \u001b[0;32m~/miniforge3/envs/ele/lib/python3.9/site-packages/datajoint/connection.py:294\u001b[0m, in \u001b[0;36mConnection._execute_query\u001b[0;34m(cursor, query, args, suppress_warnings)\u001b[0m\n\u001b[1;32m 291\u001b[0m \u001b[39mif\u001b[39;00m suppress_warnings:\n\u001b[1;32m 292\u001b[0m \u001b[39m# suppress all warnings arising from underlying SQL library\u001b[39;00m\n\u001b[1;32m 293\u001b[0m warnings\u001b[39m.\u001b[39msimplefilter(\u001b[39m\"\u001b[39m\u001b[39mignore\u001b[39m\u001b[39m\"\u001b[39m)\n\u001b[0;32m--> 294\u001b[0m cursor\u001b[39m.\u001b[39;49mexecute(query, args)\n\u001b[1;32m 295\u001b[0m \u001b[39mexcept\u001b[39;00m client\u001b[39m.\u001b[39merr\u001b[39m.\u001b[39mError \u001b[39mas\u001b[39;00m err:\n\u001b[1;32m 296\u001b[0m \u001b[39mraise\u001b[39;00m translate_query_error(err, query)\n", + "File \u001b[0;32m~/miniforge3/envs/ele/lib/python3.9/site-packages/pymysql/cursors.py:148\u001b[0m, in \u001b[0;36mCursor.execute\u001b[0;34m(self, query, args)\u001b[0m\n\u001b[1;32m 144\u001b[0m \u001b[39mpass\u001b[39;00m\n\u001b[1;32m 146\u001b[0m query \u001b[39m=\u001b[39m \u001b[39mself\u001b[39m\u001b[39m.\u001b[39mmogrify(query, args)\n\u001b[0;32m--> 148\u001b[0m result \u001b[39m=\u001b[39m \u001b[39mself\u001b[39;49m\u001b[39m.\u001b[39;49m_query(query)\n\u001b[1;32m 149\u001b[0m \u001b[39mself\u001b[39m\u001b[39m.\u001b[39m_executed \u001b[39m=\u001b[39m query\n\u001b[1;32m 150\u001b[0m \u001b[39mreturn\u001b[39;00m result\n", + "File \u001b[0;32m~/miniforge3/envs/ele/lib/python3.9/site-packages/pymysql/cursors.py:310\u001b[0m, in \u001b[0;36mCursor._query\u001b[0;34m(self, q)\u001b[0m\n\u001b[1;32m 308\u001b[0m \u001b[39mself\u001b[39m\u001b[39m.\u001b[39m_last_executed \u001b[39m=\u001b[39m q\n\u001b[1;32m 309\u001b[0m \u001b[39mself\u001b[39m\u001b[39m.\u001b[39m_clear_result()\n\u001b[0;32m--> 310\u001b[0m conn\u001b[39m.\u001b[39;49mquery(q)\n\u001b[1;32m 311\u001b[0m \u001b[39mself\u001b[39m\u001b[39m.\u001b[39m_do_get_result()\n\u001b[1;32m 312\u001b[0m \u001b[39mreturn\u001b[39;00m \u001b[39mself\u001b[39m\u001b[39m.\u001b[39mrowcount\n", + "File \u001b[0;32m~/miniforge3/envs/ele/lib/python3.9/site-packages/pymysql/connections.py:548\u001b[0m, in \u001b[0;36mConnection.query\u001b[0;34m(self, sql, unbuffered)\u001b[0m\n\u001b[1;32m 546\u001b[0m sql \u001b[39m=\u001b[39m sql\u001b[39m.\u001b[39mencode(\u001b[39mself\u001b[39m\u001b[39m.\u001b[39mencoding, \u001b[39m\"\u001b[39m\u001b[39msurrogateescape\u001b[39m\u001b[39m\"\u001b[39m)\n\u001b[1;32m 547\u001b[0m \u001b[39mself\u001b[39m\u001b[39m.\u001b[39m_execute_command(COMMAND\u001b[39m.\u001b[39mCOM_QUERY, sql)\n\u001b[0;32m--> 548\u001b[0m \u001b[39mself\u001b[39m\u001b[39m.\u001b[39m_affected_rows \u001b[39m=\u001b[39m \u001b[39mself\u001b[39;49m\u001b[39m.\u001b[39;49m_read_query_result(unbuffered\u001b[39m=\u001b[39;49munbuffered)\n\u001b[1;32m 549\u001b[0m \u001b[39mreturn\u001b[39;00m \u001b[39mself\u001b[39m\u001b[39m.\u001b[39m_affected_rows\n", + "File \u001b[0;32m~/miniforge3/envs/ele/lib/python3.9/site-packages/pymysql/connections.py:775\u001b[0m, in \u001b[0;36mConnection._read_query_result\u001b[0;34m(self, unbuffered)\u001b[0m\n\u001b[1;32m 773\u001b[0m \u001b[39melse\u001b[39;00m:\n\u001b[1;32m 774\u001b[0m result \u001b[39m=\u001b[39m MySQLResult(\u001b[39mself\u001b[39m)\n\u001b[0;32m--> 775\u001b[0m result\u001b[39m.\u001b[39;49mread()\n\u001b[1;32m 776\u001b[0m \u001b[39mself\u001b[39m\u001b[39m.\u001b[39m_result \u001b[39m=\u001b[39m result\n\u001b[1;32m 777\u001b[0m \u001b[39mif\u001b[39;00m result\u001b[39m.\u001b[39mserver_status \u001b[39mis\u001b[39;00m \u001b[39mnot\u001b[39;00m \u001b[39mNone\u001b[39;00m:\n", + "File \u001b[0;32m~/miniforge3/envs/ele/lib/python3.9/site-packages/pymysql/connections.py:1156\u001b[0m, in \u001b[0;36mMySQLResult.read\u001b[0;34m(self)\u001b[0m\n\u001b[1;32m 1154\u001b[0m \u001b[39mdef\u001b[39;00m \u001b[39mread\u001b[39m(\u001b[39mself\u001b[39m):\n\u001b[1;32m 1155\u001b[0m \u001b[39mtry\u001b[39;00m:\n\u001b[0;32m-> 1156\u001b[0m first_packet \u001b[39m=\u001b[39m \u001b[39mself\u001b[39;49m\u001b[39m.\u001b[39;49mconnection\u001b[39m.\u001b[39;49m_read_packet()\n\u001b[1;32m 1158\u001b[0m \u001b[39mif\u001b[39;00m first_packet\u001b[39m.\u001b[39mis_ok_packet():\n\u001b[1;32m 1159\u001b[0m \u001b[39mself\u001b[39m\u001b[39m.\u001b[39m_read_ok_packet(first_packet)\n", + "File \u001b[0;32m~/miniforge3/envs/ele/lib/python3.9/site-packages/pymysql/connections.py:692\u001b[0m, in \u001b[0;36mConnection._read_packet\u001b[0;34m(self, packet_type)\u001b[0m\n\u001b[1;32m 690\u001b[0m buff \u001b[39m=\u001b[39m \u001b[39mbytearray\u001b[39m()\n\u001b[1;32m 691\u001b[0m \u001b[39mwhile\u001b[39;00m \u001b[39mTrue\u001b[39;00m:\n\u001b[0;32m--> 692\u001b[0m packet_header \u001b[39m=\u001b[39m \u001b[39mself\u001b[39;49m\u001b[39m.\u001b[39;49m_read_bytes(\u001b[39m4\u001b[39;49m)\n\u001b[1;32m 693\u001b[0m \u001b[39m# if DEBUG: dump_packet(packet_header)\u001b[39;00m\n\u001b[1;32m 695\u001b[0m btrl, btrh, packet_number \u001b[39m=\u001b[39m struct\u001b[39m.\u001b[39munpack(\u001b[39m\"\u001b[39m\u001b[39m 732\u001b[0m data \u001b[39m=\u001b[39m \u001b[39mself\u001b[39;49m\u001b[39m.\u001b[39;49m_rfile\u001b[39m.\u001b[39;49mread(num_bytes)\n\u001b[1;32m 733\u001b[0m \u001b[39mbreak\u001b[39;00m\n\u001b[1;32m 734\u001b[0m \u001b[39mexcept\u001b[39;00m (\u001b[39mIOError\u001b[39;00m, \u001b[39mOSError\u001b[39;00m) \u001b[39mas\u001b[39;00m e:\n", + "File \u001b[0;32m~/miniforge3/envs/ele/lib/python3.9/socket.py:704\u001b[0m, in \u001b[0;36mSocketIO.readinto\u001b[0;34m(self, b)\u001b[0m\n\u001b[1;32m 702\u001b[0m \u001b[39mwhile\u001b[39;00m \u001b[39mTrue\u001b[39;00m:\n\u001b[1;32m 703\u001b[0m \u001b[39mtry\u001b[39;00m:\n\u001b[0;32m--> 704\u001b[0m \u001b[39mreturn\u001b[39;00m \u001b[39mself\u001b[39;49m\u001b[39m.\u001b[39;49m_sock\u001b[39m.\u001b[39;49mrecv_into(b)\n\u001b[1;32m 705\u001b[0m \u001b[39mexcept\u001b[39;00m timeout:\n\u001b[1;32m 706\u001b[0m \u001b[39mself\u001b[39m\u001b[39m.\u001b[39m_timeout_occurred \u001b[39m=\u001b[39m \u001b[39mTrue\u001b[39;00m\n", + "\u001b[0;31mKeyboardInterrupt\u001b[0m: " + ] + } + ], + "source": [ + "\n", + "from workflow_calcium_imaging.pipeline import session, imaging # import schemas\n", + "from workflow_calcium_imaging.ingest import ingest_subjects, ingest_sessions #csv loaders\n", + "\n", + "import csv\n", + "\n", + "sessions_csv_path = f\"/home/{dj.config['database.user']}/sessions.csv\"\n", + "with open(sessions_csv_path, 'w', newline='') as f:\n", + " csv_writer = csv.writer(f)\n", + " csv_writer.writerow([\"subject\",\"session_dir\"])\n", + " csv_writer.writerow([\"subject3\",\"inbox/0_1_0a2/subject3/210107_run00_orientation_8dir/\"])\n", + "\n", + "ingest_subjects(subject_csv_path=\"/home/user_data/subjects.csv\")\n", + "ingest_sessions(session_csv_path=sessions_csv_path)\n", + "\n", + "params_suite2p = {'look_one_level_down': 0.0,\n", + " 'fast_disk': [],\n", + " 'delete_bin': False,\n", + " 'mesoscan': False,\n", + " 'h5py': [],\n", + " 'h5py_key': 'data',\n", + " 'save_path0': [],\n", + " 'subfolders': [],\n", + " 'nplanes': 1,\n", + " 'nchannels': 1,\n", + " 'functional_chan': 1,\n", + " 'tau': 1.0,\n", + " 'fs': 10.0,\n", + " 'force_sktiff': False,\n", + " 'preclassify': 0.0,\n", + " 'save_mat': False,\n", + " 'combined': True,\n", + " 'aspect': 1.0,\n", + " 'do_bidiphase': False,\n", + " 'bidiphase': 0.0,\n", + " 'do_registration': True,\n", + " 'keep_movie_raw': False,\n", + " 'nimg_init': 300,\n", + " 'batch_size': 500,\n", + " 'maxregshift': 0.1,\n", + " 'align_by_chan': 1,\n", + " 'reg_tif': False,\n", + " 'reg_tif_chan2': False,\n", + " 'subpixel': 10,\n", + " 'smooth_sigma': 1.15,\n", + " 'th_badframes': 1.0,\n", + " 'pad_fft': False,\n", + " 'nonrigid': True,\n", + " 'block_size': [128, 128],\n", + " 'snr_thresh': 1.2,\n", + " 'maxregshiftNR': 5.0,\n", + " '1Preg': False,\n", + " 'spatial_hp': 50.0,\n", + " 'pre_smooth': 2.0,\n", + " 'spatial_taper': 50.0,\n", + " 'roidetect': True,\n", + " 'sparse_mode': False,\n", + " 'diameter': 12,\n", + " 'spatial_scale': 0,\n", + " 'connected': True,\n", + " 'nbinned': 5000,\n", + " 'max_iterations': 20,\n", + " 'threshold_scaling': 1.0,\n", + " 'max_overlap': 0.75,\n", + " 'high_pass': 100.0,\n", + " 'inner_neuropil_radius': 2,\n", + " 'min_neuropil_pixels': 350,\n", + " 'allow_overlap': False,\n", + " 'chan2_thres': 0.65,\n", + " 'baseline': 'maximin',\n", + " 'win_baseline': 60.0,\n", + " 'sig_baseline': 10.0,\n", + " 'prctile_baseline': 8.0,\n", + " 'neucoeff': 0.7,\n", + " 'xrange': np.array([0, 0]),\n", + " 'yrange': np.array([0, 0])}\n", + "\n", + "imaging.ProcessingParamSet.insert_new_params(\n", + " processing_method='suite2p', \n", + " paramset_idx=0, \n", + " params=params_suite2p,\n", + " paramset_desc='Calcium imaging analysis with Suite2p using default Suite2p parameters')" + ] + }, + { + "cell_type": "markdown", + "id": "97019839", + "metadata": {}, + "source": [ + "Next, we'll trigger the relevant `populate` commands." + ] + }, + { + "cell_type": "code", + "execution_count": null, + "id": "4b790374", + "metadata": {}, + "outputs": [], + "source": [ + "from workflow_calcium_imaging import process\n", + "\n", + "process.run()\n", + "session_key = (session.Session & 'subject=\"subject3\"').fetch('KEY')[0]\n", + "imaging.ProcessingTask.insert1(\n", + " dict(\n", + " session_key, \n", + " scan_id=0,\n", + " paramset_idx=0,\n", + " processing_output_dir='inbox/0_1_0a2/subject3/210107_run00_orientation_8dir/suite2p'\n", + " ),\n", + " skip_duplicates=True\n", + ")\n", + "process.run()" + ] + }, + { + "cell_type": "markdown", + "id": "6d72a724", + "metadata": {}, + "source": [ + "And then, we'll insert new Curation to trigger ingestion of curated results, followed by the same `process.run` automation.\n" + ] + }, + { + "cell_type": "code", + "execution_count": null, + "id": "07d485e0", + "metadata": {}, + "outputs": [], + "source": [ + "key = (imaging.ProcessingTask & session_key).fetch1('KEY')\n", + "imaging.Curation().create1_from_processing_task(key)\n", + "process.run()" + ] + }, + { + "cell_type": "markdown", + "id": "a6cddd3c", + "metadata": {}, + "source": [ + "## Exploring the workflow\n", + "\n", + "### Import the workflow\n", + "\n", + "The current workflow is composed of multiple database schemas, each of them corresponding to a module within the `workflow_calcium_imaging.pipeline` file." ] }, { @@ -83,9 +327,9 @@ "id": "2bd6d86b", "metadata": {}, "source": [ - "## Workflow diagram\n", + "### Diagrams and table design\n", "\n", - "Plot the workflow diagram. In relational databases, the entities (i.e. rows) in different tables are connected to each other. Visualization of this relationship helps one to write accurate queries. For the calcium imaging workflow, this connection is as follows:" + "We can plot the workflow diagram. In relational databases, the entities (i.e. rows) in different tables are connected to each other. Visualization of this relationship helps one to write accurate queries. For the calcium imaging workflow, this connection is as follows:" ] }, { @@ -136,9 +380,9 @@ "id": "5ca53cb7", "metadata": {}, "source": [ - "## Fetch data from the database\n", + "### Fetch data\n", "\n", - "Fetch a fluorescence trace for a single mask and plot these values." + "Here, we fetch a fluorescence trace for a single mask and plot these values." ] }, { @@ -168,7 +412,7 @@ "source": [ "query_trace = imaging.Fluorescence.Trace & 'subject=\"subject3\"' \\\n", " & 'session_datetime=\"2022-09-01 19:16:44\"' \\\n", - " & 'mask_id=120'\n", + " & 'mask=120'\n", "query_trace" ] }, @@ -187,7 +431,7 @@ "metadata": {}, "outputs": [], "source": [ - "trace = (query_trace).fetch('fluorescence')" + "trace = (query_trace).fetch('fluorescence')[0]" ] }, { @@ -209,10 +453,10 @@ "\n", "pyplot.plot(np.r_[:trace.size] * 1/sampling_rate, trace, 'k')\n", "\n", - "pyplot.title('Fluorescence trace for mask 120',labelsize=14)\n", + "pyplot.title('Fluorescence trace for mask 120', fontsize=14)\n", "pyplot.tick_params(labelsize=14)\n", - "pyplot.set_xlabel('Time (s)')\n", - "pyplot.set_ylabel('Activity (a.u.)')" + "pyplot.xlabel('Time (s)')\n", + "pyplot.ylabel('Activity (a.u.)')" ] }, { @@ -220,9 +464,9 @@ "id": "68559c95-3c4e-4c6a-acbb-c06796a8399c", "metadata": {}, "source": [ - "## Run analysis\n", + "## Running an analysis\n", "\n", - "The workflow has already been run for with a parameter set (paramset_idx=1). Let's re-run Suite2p with a different parameter set, changing the cell diameter to 10 microns." + "The workflow has already been run for with a parameter set (`paramset_idx=1`). Let's re-run Suite2p with a different parameter set, changing the cell diameter to 10 microns." ] }, { @@ -270,7 +514,7 @@ " 'h5py_key': 'data',\n", " 'save_path0': [],\n", " 'subfolders': [],\n", - " 'nplanes': 1,\n", + " 'nplanes': 4,\n", " 'nchannels': 1,\n", " 'functional_chan': 1,\n", " 'tau': 1.0,\n", @@ -355,13 +599,20 @@ "metadata": {}, "outputs": [], "source": [ - "os.makedirs('/home/inbox/0_1_0a2/subject3/210107_run00_orientation_8dir/suite2p_1', exist_ok=True)\n", + "output_dir = f\"{dj.config['database.user']}\"\n", + "print(output_dir)\n", + "os.makedirs(output_dir, exist_ok=True)\n", "\n", - "imaging.ProcessingTask.insert1(dict(subject='subject3', \n", - " session_datetime='2022-09-01 19:16:44', \n", - " scan_id=0,\n", - " paramset_idx=1,\n", - " processing_output_dir='subject3/210107_run00_orientation_8dir/suite2p_1'))" + "imaging.ProcessingTask.insert1(\n", + " dict(\n", + " subject='subject3', \n", + " session_datetime='2022-09-01 19:16:44', \n", + " scan_id=0,\n", + " paramset_idx=1,\n", + " processing_output_dir=output_dir,\n", + " task_mode='trigger'\n", + " )\n", + ")" ] }, { @@ -379,7 +630,7 @@ "id": "637b9ec1", "metadata": {}, "source": [ - "Run Suite2p for the new parameter set and save the results to the respective tables." + "You can then run Suite2p for the new parameter set and save the results to the respective tables. For this dataset (4 channels, 4 depths, 7.5k frames), this may take several hours." ] }, { @@ -422,7 +673,7 @@ "formats": "ipynb,py" }, "kernelspec": { - "display_name": "Python 3.7.9 ('workflow-calcium-imaging')", + "display_name": "Python 3.9.13 ('ele')", "language": "python", "name": "python3" }, @@ -436,11 +687,11 @@ "name": "python", "nbconvert_exporter": "python", "pygments_lexer": "ipython3", - "version": "3.7.9" + "version": "3.9.13" }, "vscode": { "interpreter": { - "hash": "2da0f701b958ef27dbb9ddfd0ee0eb0d75664c3cb2b4bc67fd0780a51e2b3863" + "hash": "d00c4ad21a7027bf1726d6ae3a9a6ef39c8838928eca5a3d5f51f3eb68720410" } } }, diff --git a/notebooks/py_scripts/2022-allen-institute-workshop.py b/notebooks/py_scripts/2022-allen-institute-workshop.py index 4322c41..1731d8f 100644 --- a/notebooks/py_scripts/2022-allen-institute-workshop.py +++ b/notebooks/py_scripts/2022-allen-institute-workshop.py @@ -6,9 +6,9 @@ # extension: .py # format_name: light # format_version: '1.5' -# jupytext_version: 1.14.0 +# jupytext_version: 1.14.1 # kernelspec: -# display_name: Python 3.7.9 ('workflow-calcium-imaging') +# display_name: Python 3.9.13 ('ele') # language: python # name: python3 # --- @@ -21,31 +21,162 @@ # # + This notebook is meant to be run on CodeBook (`https://codebook.datajoint.io`) which contains example data. # -# + First run the `01-configure` and `04-automate` notebooks to set up your environment and load example data into the database, respectively. +# First, some packages we'll use in this notebook... -# ## Configuration -# -# Import the relevant packages. - -import datajoint as dj +import datajoint as dj import numpy as np from matplotlib import pyplot import os +import getpass + +# ## Configuration + +# These steps are taken from [01-configure](01-configure.ipynb). If you've already saved a config file, you can skip to the next section. # Enter database credentials. A DataJoint workflow requires a connection to an existing relational database. The connection setup parameters are defined in the `dj.config` python dictionary. # + tags=[] -dj.config['custom'] = {'database.prefix': '_allen_ophys_', - 'imaging_root_data_dir': '/home/inbox/0_1_0a2/'} +username_as_prefix = dj.config["database.user"] + "_img_" +dj.config['custom'] = { + 'database.prefix': username_as_prefix, + 'imaging_root_data_dir': '/home/' +} # - -# Import the workflow. The current workflow is composed of multiple database schemas, each of them corresponding to a module within the `workflow_calcium_imaging.pipeline` file. +# Next, we'll use a prompt to securely save your password. + +dj.config["database.password"] = getpass.getpass() + +# Now to save these credentials. + +dj.config.save_global() + +# ## Populating the database + +# Next, we'll populate these schema using some steps from [04-automate](04-automate-optional.ipynb). If your schema are already populated, you can skip this step. For more details on each of these steps, please visit [that notebook](04-automate-optional.ipynb). Additional steps ensure write permissions on output directories. + +# + + +from workflow_calcium_imaging.pipeline import session, imaging # import schemas +from workflow_calcium_imaging.ingest import ingest_subjects, ingest_sessions #csv loaders + +import csv + +sessions_csv_path = f"/home/{dj.config['database.user']}/sessions.csv" +with open(sessions_csv_path, 'w', newline='') as f: + csv_writer = csv.writer(f) + csv_writer.writerow(["subject","session_dir"]) + csv_writer.writerow(["subject3","inbox/0_1_0a2/subject3/210107_run00_orientation_8dir/"]) + +ingest_subjects(subject_csv_path="/home/user_data/subjects.csv") +ingest_sessions(session_csv_path=sessions_csv_path) + +params_suite2p = {'look_one_level_down': 0.0, + 'fast_disk': [], + 'delete_bin': False, + 'mesoscan': False, + 'h5py': [], + 'h5py_key': 'data', + 'save_path0': [], + 'subfolders': [], + 'nplanes': 1, + 'nchannels': 1, + 'functional_chan': 1, + 'tau': 1.0, + 'fs': 10.0, + 'force_sktiff': False, + 'preclassify': 0.0, + 'save_mat': False, + 'combined': True, + 'aspect': 1.0, + 'do_bidiphase': False, + 'bidiphase': 0.0, + 'do_registration': True, + 'keep_movie_raw': False, + 'nimg_init': 300, + 'batch_size': 500, + 'maxregshift': 0.1, + 'align_by_chan': 1, + 'reg_tif': False, + 'reg_tif_chan2': False, + 'subpixel': 10, + 'smooth_sigma': 1.15, + 'th_badframes': 1.0, + 'pad_fft': False, + 'nonrigid': True, + 'block_size': [128, 128], + 'snr_thresh': 1.2, + 'maxregshiftNR': 5.0, + '1Preg': False, + 'spatial_hp': 50.0, + 'pre_smooth': 2.0, + 'spatial_taper': 50.0, + 'roidetect': True, + 'sparse_mode': False, + 'diameter': 12, + 'spatial_scale': 0, + 'connected': True, + 'nbinned': 5000, + 'max_iterations': 20, + 'threshold_scaling': 1.0, + 'max_overlap': 0.75, + 'high_pass': 100.0, + 'inner_neuropil_radius': 2, + 'min_neuropil_pixels': 350, + 'allow_overlap': False, + 'chan2_thres': 0.65, + 'baseline': 'maximin', + 'win_baseline': 60.0, + 'sig_baseline': 10.0, + 'prctile_baseline': 8.0, + 'neucoeff': 0.7, + 'xrange': np.array([0, 0]), + 'yrange': np.array([0, 0])} + +imaging.ProcessingParamSet.insert_new_params( + processing_method='suite2p', + paramset_idx=0, + params=params_suite2p, + paramset_desc='Calcium imaging analysis with Suite2p using default Suite2p parameters') +# - + +# Next, we'll trigger the relevant `populate` commands. + +# + +from workflow_calcium_imaging import process + +process.run() +session_key = (session.Session & 'subject="subject3"').fetch('KEY')[0] +imaging.ProcessingTask.insert1( + dict( + session_key, + scan_id=0, + paramset_idx=0, + processing_output_dir='inbox/0_1_0a2/subject3/210107_run00_orientation_8dir/suite2p' + ), + skip_duplicates=True +) +process.run() +# - + +# And then, we'll insert new Curation to trigger ingestion of curated results, followed by the same `process.run` automation. +# + +key = (imaging.ProcessingTask & session_key).fetch1('KEY') +imaging.Curation().create1_from_processing_task(key) +process.run() + +# ## Exploring the workflow +# +# ### Import the workflow +# +# The current workflow is composed of multiple database schemas, each of them corresponding to a module within the `workflow_calcium_imaging.pipeline` file. from workflow_calcium_imaging.pipeline import lab, subject, session, scan, imaging -# ## Workflow diagram +# ### Diagrams and table design # -# Plot the workflow diagram. In relational databases, the entities (i.e. rows) in different tables are connected to each other. Visualization of this relationship helps one to write accurate queries. For the calcium imaging workflow, this connection is as follows: +# We can plot the workflow diagram. In relational databases, the entities (i.e. rows) in different tables are connected to each other. Visualization of this relationship helps one to write accurate queries. For the calcium imaging workflow, this connection is as follows: # + tags=[] dj.Diagram(lab.Lab) + dj.Diagram(subject.Subject) + dj.Diagram(session.Session) + \ @@ -58,9 +189,9 @@ imaging.Fluorescence() -# ## Fetch data from the database +# ### Fetch data # -# Fetch a fluorescence trace for a single mask and plot these values. +# Here, we fetch a fluorescence trace for a single mask and plot these values. imaging.Fluorescence.Trace() @@ -68,12 +199,12 @@ query_trace = imaging.Fluorescence.Trace & 'subject="subject3"' \ & 'session_datetime="2022-09-01 19:16:44"' \ - & 'mask_id=120' + & 'mask=120' query_trace # Fetch a fluorescence trace from the database. -trace = (query_trace).fetch('fluorescence') +trace = (query_trace).fetch('fluorescence')[0] # Plot the fluorescence trace. @@ -82,15 +213,15 @@ pyplot.plot(np.r_[:trace.size] * 1/sampling_rate, trace, 'k') -pyplot.title('Fluorescence trace for mask 120',labelsize=14) +pyplot.title('Fluorescence trace for mask 120', fontsize=14) pyplot.tick_params(labelsize=14) -pyplot.set_xlabel('Time (s)') -pyplot.set_ylabel('Activity (a.u.)') +pyplot.xlabel('Time (s)') +pyplot.ylabel('Activity (a.u.)') # - -# ## Run analysis +# ## Running an analysis # -# The workflow has already been run for with a parameter set (paramset_idx=1). Let's re-run Suite2p with a different parameter set, changing the cell diameter to 10 microns. +# The workflow has already been run for with a parameter set (`paramset_idx=1`). Let's re-run Suite2p with a different parameter set, changing the cell diameter to 10 microns. dj.Diagram(imaging.Processing)-2 @@ -106,7 +237,7 @@ 'h5py_key': 'data', 'save_path0': [], 'subfolders': [], - 'nplanes': 1, + 'nplanes': 4, 'nchannels': 1, 'functional_chan': 1, 'tau': 1.0, @@ -168,18 +299,25 @@ imaging.ProcessingParamSet() # + -os.makedirs('/home/inbox/0_1_0a2/subject3/210107_run00_orientation_8dir/suite2p_1', exist_ok=True) - -imaging.ProcessingTask.insert1(dict(subject='subject3', - session_datetime='2022-09-01 19:16:44', - scan_id=0, - paramset_idx=1, - processing_output_dir='subject3/210107_run00_orientation_8dir/suite2p_1')) +output_dir = f"{dj.config['database.user']}" +print(output_dir) +os.makedirs(output_dir, exist_ok=True) + +imaging.ProcessingTask.insert1( + dict( + subject='subject3', + session_datetime='2022-09-01 19:16:44', + scan_id=0, + paramset_idx=1, + processing_output_dir=output_dir, + task_mode='trigger' + ) +) # - imaging.ProcessingTask() -# Run Suite2p for the new parameter set and save the results to the respective tables. +# You can then run Suite2p for the new parameter set and save the results to the respective tables. For this dataset (4 channels, 4 depths, 7.5k frames), this may take several hours. # + populate_settings = dict(display_progress=True) diff --git a/requirements.txt b/requirements.txt index a097493..06dfb4e 100644 --- a/requirements.txt +++ b/requirements.txt @@ -7,5 +7,6 @@ element-event @ git+https://github.com/datajoint/element-event.git element-interface @ git+https://github.com/datajoint/element-interface.git ipykernel>=6.0.1 jupytext>=1.13.7 -sbxreader +sbxreader @ git+https://github.com/CBroz1/sbxreader +nd2 suite2p @ git+https://github.com/datajoint-company/suite2p.git \ No newline at end of file