From 7cec3dd3ab64ad193ac312cec4cafaaaa740e87b Mon Sep 17 00:00:00 2001
From: Chris Broz <CBroz@datajoint.com>
Date: Wed, 21 Sep 2022 12:39:51 -0500
Subject: [PATCH 1/6] Revise Allen notebook

---
 CHANGELOG.md                                  |   5 +-
 notebooks/2022-allen-institute-workshop.ipynb | 244 ++++++++++++++++--
 .../2022-allen-institute-workshop.py          | 149 +++++++++--
 3 files changed, 355 insertions(+), 43 deletions(-)

diff --git a/CHANGELOG.md b/CHANGELOG.md
index 04104e0..f6425ee 100644
--- a/CHANGELOG.md
+++ b/CHANGELOG.md
@@ -3,9 +3,9 @@
 Observes [Semantic Versioning](https://semver.org/spec/v2.0.0.html) standard and 
 [Keep a Changelog](https://keepachangelog.com/en/1.0.0/) convention.
 
-## [0.1.3] - 2022-09-20
+## [0.1.4] - 2022-09-20
 
-+ Add - Notebook for Allen Institute workshop
++ Add - Add (0.1.3) and revise (0.1.4) notebook for Allen Institute workshop
 
 ## [0.1.2] - 2022-09-19
 
@@ -43,6 +43,7 @@ Observes [Semantic Versioning](https://semver.org/spec/v2.0.0.html) standard and
 + Add - Containerization for pytests
 + Comment - Phase previously designated 0.1.0 -> 0.0.0
 
+[0.1.4]: https://github.com/datajoint/workflow-calcium-imaging/releases/tag/0.1.4
 [0.1.3]: https://github.com/datajoint/workflow-calcium-imaging/releases/tag/0.1.3
 [0.1.2]: https://github.com/datajoint/workflow-calcium-imaging/releases/tag/0.1.2
 [0.1.1]: https://github.com/datajoint/workflow-calcium-imaging/releases/tag/0.1.1
diff --git a/notebooks/2022-allen-institute-workshop.ipynb b/notebooks/2022-allen-institute-workshop.ipynb
index 259b742..7c41788 100644
--- a/notebooks/2022-allen-institute-workshop.ipynb
+++ b/notebooks/2022-allen-institute-workshop.ipynb
@@ -13,17 +13,7 @@
     "\n",
     "+ This notebook is meant to be run on CodeBook (`https://codebook.datajoint.io`) which contains example data.\n",
     "\n",
-    "+ First run the `01-configure` and `04-automate` notebooks to set up your environment and load example data into the database, respectively."
-   ]
-  },
-  {
-   "cell_type": "markdown",
-   "id": "14a6ba1d",
-   "metadata": {},
-   "source": [
-    "## Configuration\n",
-    "\n",
-    "Import the relevant packages."
+    "First, some packages we'll use in this notebook..."
    ]
   },
   {
@@ -36,7 +26,24 @@
     "import datajoint as dj\n",
     "import numpy as np\n",
     "from matplotlib import pyplot\n",
-    "import os"
+    "import os\n",
+    "import getpass"
+   ]
+  },
+  {
+   "cell_type": "markdown",
+   "id": "14a6ba1d",
+   "metadata": {},
+   "source": [
+    "## Configuration"
+   ]
+  },
+  {
+   "cell_type": "markdown",
+   "id": "cb03797f",
+   "metadata": {},
+   "source": [
+    "These steps are taken from [01-configure](01-configure.ipynb). If you've already saved a config file, you can skip to the next section."
    ]
   },
   {
@@ -56,16 +63,203 @@
    },
    "outputs": [],
    "source": [
-    "dj.config['custom'] = {'database.prefix': '<username>_allen_ophys_',\n",
-    "                       'imaging_root_data_dir': '/home/inbox/0_1_0a2/'}"
+    "username_as_prefix = dj.config[\"database.user\"] + \"_\"\n",
+    "dj.config['custom'] = {\n",
+    "    'database.prefix': username_as_prefix,\n",
+    "    'imaging_root_data_dir': '/home/inbox/0_1_0a2/'\n",
+    "}"
+   ]
+  },
+  {
+   "cell_type": "markdown",
+   "id": "21a69c02",
+   "metadata": {},
+   "source": [
+    "Next, we'll use a prompt to securely save your password."
+   ]
+  },
+  {
+   "cell_type": "code",
+   "execution_count": null,
+   "id": "66bea7cb",
+   "metadata": {},
+   "outputs": [],
+   "source": [
+    "dj.config[\"database.password\"] = getpass.getpass()"
+   ]
+  },
+  {
+   "cell_type": "markdown",
+   "id": "c45a3aad",
+   "metadata": {},
+   "source": [
+    "Now to save these credentials."
+   ]
+  },
+  {
+   "cell_type": "code",
+   "execution_count": null,
+   "id": "4e4f0448",
+   "metadata": {},
+   "outputs": [],
+   "source": [
+    "dj.config.save_global()"
+   ]
+  },
+  {
+   "cell_type": "markdown",
+   "id": "a570822d",
+   "metadata": {},
+   "source": [
+    "## Populating the database"
+   ]
+  },
+  {
+   "cell_type": "markdown",
+   "id": "8ee61c1d",
+   "metadata": {},
+   "source": [
+    "Next, we'll populate these schema using steps from [04-automate](04-automate-optional.ipynb). If your schema are already populated, you can skip this step. For more details on each of these steps, please visit [that notebook](04-automate-optional.ipynb)."
+   ]
+  },
+  {
+   "cell_type": "code",
+   "execution_count": null,
+   "id": "bc01b55c",
+   "metadata": {},
+   "outputs": [],
+   "source": [
+    "from workflow_calcium_imaging.pipeline import session, imaging # import schemas\n",
+    "from workflow_calcium_imaging.ingest import ingest_subjects, ingest_sessions #csv loaders\n",
+    "\n",
+    "ingest_subjects()\n",
+    "ingest_sessions()\n",
+    "\n",
+    "params_suite2p = {'look_one_level_down': 0.0,\n",
+    "                  'fast_disk': [],\n",
+    "                  'delete_bin': False,\n",
+    "                  'mesoscan': False,\n",
+    "                  'h5py': [],\n",
+    "                  'h5py_key': 'data',\n",
+    "                  'save_path0': [],\n",
+    "                  'subfolders': [],\n",
+    "                  'nplanes': 1,\n",
+    "                  'nchannels': 1,\n",
+    "                  'functional_chan': 1,\n",
+    "                  'tau': 1.0,\n",
+    "                  'fs': 10.0,\n",
+    "                  'force_sktiff': False,\n",
+    "                  'preclassify': 0.0,\n",
+    "                  'save_mat': False,\n",
+    "                  'combined': True,\n",
+    "                  'aspect': 1.0,\n",
+    "                  'do_bidiphase': False,\n",
+    "                  'bidiphase': 0.0,\n",
+    "                  'do_registration': True,\n",
+    "                  'keep_movie_raw': False,\n",
+    "                  'nimg_init': 300,\n",
+    "                  'batch_size': 500,\n",
+    "                  'maxregshift': 0.1,\n",
+    "                  'align_by_chan': 1,\n",
+    "                  'reg_tif': False,\n",
+    "                  'reg_tif_chan2': False,\n",
+    "                  'subpixel': 10,\n",
+    "                  'smooth_sigma': 1.15,\n",
+    "                  'th_badframes': 1.0,\n",
+    "                  'pad_fft': False,\n",
+    "                  'nonrigid': True,\n",
+    "                  'block_size': [128, 128],\n",
+    "                  'snr_thresh': 1.2,\n",
+    "                  'maxregshiftNR': 5.0,\n",
+    "                  '1Preg': False,\n",
+    "                  'spatial_hp': 50.0,\n",
+    "                  'pre_smooth': 2.0,\n",
+    "                  'spatial_taper': 50.0,\n",
+    "                  'roidetect': True,\n",
+    "                  'sparse_mode': False,\n",
+    "                  'diameter': 12,\n",
+    "                  'spatial_scale': 0,\n",
+    "                  'connected': True,\n",
+    "                  'nbinned': 5000,\n",
+    "                  'max_iterations': 20,\n",
+    "                  'threshold_scaling': 1.0,\n",
+    "                  'max_overlap': 0.75,\n",
+    "                  'high_pass': 100.0,\n",
+    "                  'inner_neuropil_radius': 2,\n",
+    "                  'min_neuropil_pixels': 350,\n",
+    "                  'allow_overlap': False,\n",
+    "                  'chan2_thres': 0.65,\n",
+    "                  'baseline': 'maximin',\n",
+    "                  'win_baseline': 60.0,\n",
+    "                  'sig_baseline': 10.0,\n",
+    "                  'prctile_baseline': 8.0,\n",
+    "                  'neucoeff': 0.7,\n",
+    "                  'xrange': np.array([0, 0]),\n",
+    "                  'yrange': np.array([0, 0])}\n",
+    "\n",
+    "imaging.ProcessingParamSet.insert_new_params(\n",
+    "    processing_method='suite2p', \n",
+    "    paramset_idx=0, \n",
+    "    params=params_suite2p,\n",
+    "    paramset_desc='Calcium imaging analysis with Suite2p using default Suite2p parameters')"
+   ]
+  },
+  {
+   "cell_type": "markdown",
+   "id": "97019839",
+   "metadata": {},
+   "source": [
+    "Next, we'll trigger the relevant `populate` commands."
+   ]
+  },
+  {
+   "cell_type": "code",
+   "execution_count": null,
+   "id": "4b790374",
+   "metadata": {},
+   "outputs": [],
+   "source": [
+    "from workflow_calcium_imaging import process\n",
+    "\n",
+    "process.run()\n",
+    "session_key = (session.Session & 'subject=\"subject3\"').fetch('KEY')[0]\n",
+    "imaging.ProcessingTask.insert1(dict(session_key, \n",
+    "                                    scan_id=0,\n",
+    "                                    paramset_idx=0,\n",
+    "                                    processing_output_dir='subject3/210107_run00_orientation_8dir/suite2p'), skip_duplicates=True)\n",
+    "process.run()"
+   ]
+  },
+  {
+   "cell_type": "markdown",
+   "id": "6d72a724",
+   "metadata": {},
+   "source": [
+    "And then, we'll insert new Curation to trigger ingestion of curated results, followed by the same `process.run` automation.\n"
+   ]
+  },
+  {
+   "cell_type": "code",
+   "execution_count": null,
+   "id": "07d485e0",
+   "metadata": {},
+   "outputs": [],
+   "source": [
+    "key = (imaging.ProcessingTask & session_key).fetch1('KEY')\n",
+    "imaging.Curation().create1_from_processing_task(key)\n",
+    "process.run()"
    ]
   },
   {
    "cell_type": "markdown",
-   "id": "7c545770",
+   "id": "a6cddd3c",
    "metadata": {},
    "source": [
-    "Import the workflow.  The current workflow is composed of multiple database schemas, each of them corresponding to a module within the `workflow_calcium_imaging.pipeline` file."
+    "## Exploring the workflow\n",
+    "\n",
+    "### Import the workflow\n",
+    "\n",
+    "The current workflow is composed of multiple database schemas, each of them corresponding to a module within the `workflow_calcium_imaging.pipeline` file."
    ]
   },
   {
@@ -83,9 +277,9 @@
    "id": "2bd6d86b",
    "metadata": {},
    "source": [
-    "## Workflow diagram\n",
+    "### Diagrams and table design\n",
     "\n",
-    "Plot the workflow diagram.  In relational databases, the entities (i.e. rows) in different tables are connected to each other. Visualization of this relationship helps one to write accurate queries. For the calcium imaging workflow, this connection is as follows:"
+    "We can plot the workflow diagram.  In relational databases, the entities (i.e. rows) in different tables are connected to each other. Visualization of this relationship helps one to write accurate queries. For the calcium imaging workflow, this connection is as follows:"
    ]
   },
   {
@@ -136,9 +330,9 @@
    "id": "5ca53cb7",
    "metadata": {},
    "source": [
-    "## Fetch data from the database\n",
+    "### Fetch data\n",
     "\n",
-    "Fetch a fluorescence trace for a single mask and plot these values."
+    "Here, we fetch a fluorescence trace for a single mask and plot these values."
    ]
   },
   {
@@ -220,9 +414,9 @@
    "id": "68559c95-3c4e-4c6a-acbb-c06796a8399c",
    "metadata": {},
    "source": [
-    "## Run analysis\n",
+    "## Running an analysis\n",
     "\n",
-    "The workflow has already been run for with a parameter set (paramset_idx=1).  Let's re-run Suite2p with a different parameter set, changing the cell diameter to 10 microns."
+    "The workflow has already been run for with a parameter set (`paramset_idx=1`).  Let's re-run Suite2p with a different parameter set, changing the cell diameter to 10 microns."
    ]
   },
   {
@@ -422,7 +616,7 @@
    "formats": "ipynb,py"
   },
   "kernelspec": {
-   "display_name": "Python 3.7.9 ('workflow-calcium-imaging')",
+   "display_name": "Python 3.9.13 ('ele')",
    "language": "python",
    "name": "python3"
   },
@@ -436,11 +630,11 @@
    "name": "python",
    "nbconvert_exporter": "python",
    "pygments_lexer": "ipython3",
-   "version": "3.7.9"
+   "version": "3.9.13"
   },
   "vscode": {
    "interpreter": {
-    "hash": "2da0f701b958ef27dbb9ddfd0ee0eb0d75664c3cb2b4bc67fd0780a51e2b3863"
+    "hash": "d00c4ad21a7027bf1726d6ae3a9a6ef39c8838928eca5a3d5f51f3eb68720410"
    }
   }
  },
diff --git a/notebooks/py_scripts/2022-allen-institute-workshop.py b/notebooks/py_scripts/2022-allen-institute-workshop.py
index 4322c41..2723c8e 100644
--- a/notebooks/py_scripts/2022-allen-institute-workshop.py
+++ b/notebooks/py_scripts/2022-allen-institute-workshop.py
@@ -6,9 +6,9 @@
 #       extension: .py
 #       format_name: light
 #       format_version: '1.5'
-#       jupytext_version: 1.14.0
+#       jupytext_version: 1.14.1
 #   kernelspec:
-#     display_name: Python 3.7.9 ('workflow-calcium-imaging')
+#     display_name: Python 3.9.13 ('ele')
 #     language: python
 #     name: python3
 # ---
@@ -21,31 +21,148 @@
 #
 # + This notebook is meant to be run on CodeBook (`https://codebook.datajoint.io`) which contains example data.
 #
-# + First run the `01-configure` and `04-automate` notebooks to set up your environment and load example data into the database, respectively.
-
-# ## Configuration
-#
-# Import the relevant packages.
+# First, some packages we'll use in this notebook...
 
 import datajoint as dj
 import numpy as np
 from matplotlib import pyplot
 import os
+import getpass
+
+# ## Configuration
+
+# These steps are taken from [01-configure](01-configure.ipynb). If you've already saved a config file, you can skip to the next section.
 
 # Enter database credentials.  A DataJoint workflow requires a connection to an existing relational database. The connection setup parameters are defined in the `dj.config` python dictionary.
 
 # + tags=[]
-dj.config['custom'] = {'database.prefix': '<username>_allen_ophys_',
-                       'imaging_root_data_dir': '/home/inbox/0_1_0a2/'}
+username_as_prefix = dj.config["database.user"] + "_"
+dj.config['custom'] = {
+    'database.prefix': username_as_prefix,
+    'imaging_root_data_dir': '/home/inbox/0_1_0a2/'
+}
+# -
+
+# Next, we'll use a prompt to securely save your password.
+
+dj.config["database.password"] = getpass.getpass()
+
+# Now to save these credentials.
+
+dj.config.save_global()
+
+# ## Populating the database
+
+# Next, we'll populate these schema using steps from [04-automate](04-automate-optional.ipynb). If your schema are already populated, you can skip this step. For more details on each of these steps, please visit [that notebook](04-automate-optional.ipynb).
+
+# +
+from workflow_calcium_imaging.pipeline import session, imaging # import schemas
+from workflow_calcium_imaging.ingest import ingest_subjects, ingest_sessions #csv loaders
+
+ingest_subjects()
+ingest_sessions()
+
+params_suite2p = {'look_one_level_down': 0.0,
+                  'fast_disk': [],
+                  'delete_bin': False,
+                  'mesoscan': False,
+                  'h5py': [],
+                  'h5py_key': 'data',
+                  'save_path0': [],
+                  'subfolders': [],
+                  'nplanes': 1,
+                  'nchannels': 1,
+                  'functional_chan': 1,
+                  'tau': 1.0,
+                  'fs': 10.0,
+                  'force_sktiff': False,
+                  'preclassify': 0.0,
+                  'save_mat': False,
+                  'combined': True,
+                  'aspect': 1.0,
+                  'do_bidiphase': False,
+                  'bidiphase': 0.0,
+                  'do_registration': True,
+                  'keep_movie_raw': False,
+                  'nimg_init': 300,
+                  'batch_size': 500,
+                  'maxregshift': 0.1,
+                  'align_by_chan': 1,
+                  'reg_tif': False,
+                  'reg_tif_chan2': False,
+                  'subpixel': 10,
+                  'smooth_sigma': 1.15,
+                  'th_badframes': 1.0,
+                  'pad_fft': False,
+                  'nonrigid': True,
+                  'block_size': [128, 128],
+                  'snr_thresh': 1.2,
+                  'maxregshiftNR': 5.0,
+                  '1Preg': False,
+                  'spatial_hp': 50.0,
+                  'pre_smooth': 2.0,
+                  'spatial_taper': 50.0,
+                  'roidetect': True,
+                  'sparse_mode': False,
+                  'diameter': 12,
+                  'spatial_scale': 0,
+                  'connected': True,
+                  'nbinned': 5000,
+                  'max_iterations': 20,
+                  'threshold_scaling': 1.0,
+                  'max_overlap': 0.75,
+                  'high_pass': 100.0,
+                  'inner_neuropil_radius': 2,
+                  'min_neuropil_pixels': 350,
+                  'allow_overlap': False,
+                  'chan2_thres': 0.65,
+                  'baseline': 'maximin',
+                  'win_baseline': 60.0,
+                  'sig_baseline': 10.0,
+                  'prctile_baseline': 8.0,
+                  'neucoeff': 0.7,
+                  'xrange': np.array([0, 0]),
+                  'yrange': np.array([0, 0])}
+
+imaging.ProcessingParamSet.insert_new_params(
+    processing_method='suite2p', 
+    paramset_idx=0, 
+    params=params_suite2p,
+    paramset_desc='Calcium imaging analysis with Suite2p using default Suite2p parameters')
+# -
+
+# Next, we'll trigger the relevant `populate` commands.
+
+# +
+from workflow_calcium_imaging import process
+
+process.run()
+session_key = (session.Session & 'subject="subject3"').fetch('KEY')[0]
+imaging.ProcessingTask.insert1(dict(session_key, 
+                                    scan_id=0,
+                                    paramset_idx=0,
+                                    processing_output_dir='subject3/210107_run00_orientation_8dir/suite2p'), skip_duplicates=True)
+process.run()
 # -
 
-# Import the workflow.  The current workflow is composed of multiple database schemas, each of them corresponding to a module within the `workflow_calcium_imaging.pipeline` file.
+# And then, we'll insert new Curation to trigger ingestion of curated results, followed by the same `process.run` automation.
+#
+
+key = (imaging.ProcessingTask & session_key).fetch1('KEY')
+imaging.Curation().create1_from_processing_task(key)
+process.run()
+
+# ## Exploring the workflow
+#
+# ### Import the workflow
+#
+# The current workflow is composed of multiple database schemas, each of them corresponding to a module within the `workflow_calcium_imaging.pipeline` file.
 
 from workflow_calcium_imaging.pipeline import lab, subject, session, scan, imaging
 
-# ## Workflow diagram
+# ### Diagrams and table design
 #
-# Plot the workflow diagram.  In relational databases, the entities (i.e. rows) in different tables are connected to each other. Visualization of this relationship helps one to write accurate queries. For the calcium imaging workflow, this connection is as follows:
+# We can plot the workflow diagram.  In relational databases, the entities (i.e. rows) in different tables are connected to each other. Visualization of this relationship helps one to write accurate queries. For the calcium imaging workflow, this connection is as follows:
 
 # + tags=[]
 dj.Diagram(lab.Lab) + dj.Diagram(subject.Subject) + dj.Diagram(session.Session) + \
@@ -58,9 +175,9 @@
 
 imaging.Fluorescence()
 
-# ## Fetch data from the database
+# ### Fetch data
 #
-# Fetch a fluorescence trace for a single mask and plot these values.
+# Here, we fetch a fluorescence trace for a single mask and plot these values.
 
 imaging.Fluorescence.Trace()
 
@@ -88,9 +205,9 @@
 pyplot.set_ylabel('Activity (a.u.)')
 # -
 
-# ## Run analysis
+# ## Running an analysis
 #
-# The workflow has already been run for with a parameter set (paramset_idx=1).  Let's re-run Suite2p with a different parameter set, changing the cell diameter to 10 microns.
+# The workflow has already been run for with a parameter set (`paramset_idx=1`).  Let's re-run Suite2p with a different parameter set, changing the cell diameter to 10 microns.
 
 dj.Diagram(imaging.Processing)-2
 

From d0bf5ae56c211397820dacb3db9038bf6bdf3034 Mon Sep 17 00:00:00 2001
From: Chris Broz <CBroz@datajoint.com>
Date: Wed, 21 Sep 2022 13:09:15 -0500
Subject: [PATCH 2/6] Revise Allen notebook 2

---
 notebooks/2022-allen-institute-workshop.ipynb      | 14 +++++++-------
 .../py_scripts/2022-allen-institute-workshop.py    | 14 +++++++-------
 2 files changed, 14 insertions(+), 14 deletions(-)

diff --git a/notebooks/2022-allen-institute-workshop.ipynb b/notebooks/2022-allen-institute-workshop.ipynb
index 7c41788..242c5be 100644
--- a/notebooks/2022-allen-institute-workshop.ipynb
+++ b/notebooks/2022-allen-institute-workshop.ipynb
@@ -23,7 +23,7 @@
    "metadata": {},
    "outputs": [],
    "source": [
-    "import datajoint as dj\n",
+    "import datajoint as dj \n",
     "import numpy as np\n",
     "from matplotlib import pyplot\n",
     "import os\n",
@@ -132,8 +132,8 @@
     "from workflow_calcium_imaging.pipeline import session, imaging # import schemas\n",
     "from workflow_calcium_imaging.ingest import ingest_subjects, ingest_sessions #csv loaders\n",
     "\n",
-    "ingest_subjects()\n",
-    "ingest_sessions()\n",
+    "ingest_subjects(subject_csv_path=\"/home/user_data/subjects.csv\")\n",
+    "ingest_sessions(session_csv_path=\"/home/user_data/sessions.csv\")\n",
     "\n",
     "params_suite2p = {'look_one_level_down': 0.0,\n",
     "                  'fast_disk': [],\n",
@@ -362,7 +362,7 @@
    "source": [
     "query_trace = imaging.Fluorescence.Trace & 'subject=\"subject3\"' \\\n",
     "                                    & 'session_datetime=\"2022-09-01 19:16:44\"' \\\n",
-    "                                    & 'mask_id=120'\n",
+    "                                    & 'mask=120'\n",
     "query_trace"
    ]
   },
@@ -403,10 +403,10 @@
     "\n",
     "pyplot.plot(np.r_[:trace.size] * 1/sampling_rate, trace, 'k')\n",
     "\n",
-    "pyplot.title('Fluorescence trace for mask 120',labelsize=14)\n",
+    "pyplot.title('Fluorescence trace for mask 120', fontsize=14)\n",
     "pyplot.tick_params(labelsize=14)\n",
-    "pyplot.set_xlabel('Time (s)')\n",
-    "pyplot.set_ylabel('Activity (a.u.)')"
+    "pyplot.xlabel('Time (s)')\n",
+    "pyplot.ylabel('Activity (a.u.)')"
    ]
   },
   {
diff --git a/notebooks/py_scripts/2022-allen-institute-workshop.py b/notebooks/py_scripts/2022-allen-institute-workshop.py
index 2723c8e..d6ac1f8 100644
--- a/notebooks/py_scripts/2022-allen-institute-workshop.py
+++ b/notebooks/py_scripts/2022-allen-institute-workshop.py
@@ -23,7 +23,7 @@
 #
 # First, some packages we'll use in this notebook...
 
-import datajoint as dj
+import datajoint as dj 
 import numpy as np
 from matplotlib import pyplot
 import os
@@ -59,8 +59,8 @@
 from workflow_calcium_imaging.pipeline import session, imaging # import schemas
 from workflow_calcium_imaging.ingest import ingest_subjects, ingest_sessions #csv loaders
 
-ingest_subjects()
-ingest_sessions()
+ingest_subjects(subject_csv_path="/home/user_data/subjects.csv")
+ingest_sessions(session_csv_path="/home/user_data/sessions.csv")
 
 params_suite2p = {'look_one_level_down': 0.0,
                   'fast_disk': [],
@@ -185,7 +185,7 @@
 
 query_trace = imaging.Fluorescence.Trace & 'subject="subject3"' \
                                     & 'session_datetime="2022-09-01 19:16:44"' \
-                                    & 'mask_id=120'
+                                    & 'mask=120'
 query_trace
 
 # Fetch a fluorescence trace from the database.
@@ -199,10 +199,10 @@
 
 pyplot.plot(np.r_[:trace.size] * 1/sampling_rate, trace, 'k')
 
-pyplot.title('Fluorescence trace for mask 120',labelsize=14)
+pyplot.title('Fluorescence trace for mask 120', fontsize=14)
 pyplot.tick_params(labelsize=14)
-pyplot.set_xlabel('Time (s)')
-pyplot.set_ylabel('Activity (a.u.)')
+pyplot.xlabel('Time (s)')
+pyplot.ylabel('Activity (a.u.)')
 # -
 
 # ## Running an analysis

From 58ada858d383ce684245d2bf2f9e7707a45e6f8e Mon Sep 17 00:00:00 2001
From: Chris Broz <CBroz@datajoint.com>
Date: Wed, 21 Sep 2022 13:56:21 -0500
Subject: [PATCH 3/6] Revise Allen notebook 3

---
 notebooks/2022-allen-institute-workshop.ipynb         | 2 +-
 notebooks/py_scripts/2022-allen-institute-workshop.py | 2 +-
 2 files changed, 2 insertions(+), 2 deletions(-)

diff --git a/notebooks/2022-allen-institute-workshop.ipynb b/notebooks/2022-allen-institute-workshop.ipynb
index 242c5be..75ce921 100644
--- a/notebooks/2022-allen-institute-workshop.ipynb
+++ b/notebooks/2022-allen-institute-workshop.ipynb
@@ -63,7 +63,7 @@
    },
    "outputs": [],
    "source": [
-    "username_as_prefix = dj.config[\"database.user\"] + \"_\"\n",
+    "username_as_prefix = dj.config[\"database.user\"] + \"_img_\"\n",
     "dj.config['custom'] = {\n",
     "    'database.prefix': username_as_prefix,\n",
     "    'imaging_root_data_dir': '/home/inbox/0_1_0a2/'\n",
diff --git a/notebooks/py_scripts/2022-allen-institute-workshop.py b/notebooks/py_scripts/2022-allen-institute-workshop.py
index d6ac1f8..391e838 100644
--- a/notebooks/py_scripts/2022-allen-institute-workshop.py
+++ b/notebooks/py_scripts/2022-allen-institute-workshop.py
@@ -36,7 +36,7 @@
 # Enter database credentials.  A DataJoint workflow requires a connection to an existing relational database. The connection setup parameters are defined in the `dj.config` python dictionary.
 
 # + tags=[]
-username_as_prefix = dj.config["database.user"] + "_"
+username_as_prefix = dj.config["database.user"] + "_img_"
 dj.config['custom'] = {
     'database.prefix': username_as_prefix,
     'imaging_root_data_dir': '/home/inbox/0_1_0a2/'

From 5e8deffbf2027dde4c8b9fe0c060950acf0ea7ad Mon Sep 17 00:00:00 2001
From: Chris Broz <CBroz@datajoint.com>
Date: Wed, 21 Sep 2022 15:01:41 -0500
Subject: [PATCH 4/6] Revise Allen notebook 4

---
 notebooks/2022-allen-institute-workshop.ipynb | 89 +++++++++++++++----
 .../2022-allen-institute-workshop.py          | 51 +++++++----
 requirements.txt                              |  1 +
 3 files changed, 110 insertions(+), 31 deletions(-)

diff --git a/notebooks/2022-allen-institute-workshop.ipynb b/notebooks/2022-allen-institute-workshop.ipynb
index 75ce921..acc9e5b 100644
--- a/notebooks/2022-allen-institute-workshop.ipynb
+++ b/notebooks/2022-allen-institute-workshop.ipynb
@@ -66,7 +66,7 @@
     "username_as_prefix = dj.config[\"database.user\"] + \"_img_\"\n",
     "dj.config['custom'] = {\n",
     "    'database.prefix': username_as_prefix,\n",
-    "    'imaging_root_data_dir': '/home/inbox/0_1_0a2/'\n",
+    "    'imaging_root_data_dir': '/home/'\n",
     "}"
    ]
   },
@@ -119,21 +119,66 @@
    "id": "8ee61c1d",
    "metadata": {},
    "source": [
-    "Next, we'll populate these schema using steps from [04-automate](04-automate-optional.ipynb). If your schema are already populated, you can skip this step. For more details on each of these steps, please visit [that notebook](04-automate-optional.ipynb)."
+    "Next, we'll populate these schema using some steps from [04-automate](04-automate-optional.ipynb). If your schema are already populated, you can skip this step. For more details on each of these steps, please visit [that notebook](04-automate-optional.ipynb). Additional steps ensure write permissions on output directories."
    ]
   },
   {
    "cell_type": "code",
-   "execution_count": null,
+   "execution_count": 1,
    "id": "bc01b55c",
    "metadata": {},
-   "outputs": [],
+   "outputs": [
+    {
+     "name": "stderr",
+     "output_type": "stream",
+     "text": [
+      "[2022-09-21 14:04:13,920][INFO]: Connecting cbroz@dss-db.datajoint.io:3306\n",
+      "[2022-09-21 14:04:14,701][INFO]: Connected cbroz@dss-db.datajoint.io:3306\n"
+     ]
+    },
+    {
+     "ename": "KeyboardInterrupt",
+     "evalue": "",
+     "output_type": "error",
+     "traceback": [
+      "\u001b[0;31m---------------------------------------------------------------------------\u001b[0m",
+      "\u001b[0;31mKeyboardInterrupt\u001b[0m                         Traceback (most recent call last)",
+      "Cell \u001b[0;32mIn [1], line 1\u001b[0m\n\u001b[0;32m----> 1\u001b[0m \u001b[38;5;28;01mfrom\u001b[39;00m \u001b[38;5;21;01mworkflow_calcium_imaging\u001b[39;00m\u001b[38;5;21;01m.\u001b[39;00m\u001b[38;5;21;01mpipeline\u001b[39;00m \u001b[38;5;28;01mimport\u001b[39;00m session, imaging \u001b[38;5;66;03m# import schemas\u001b[39;00m\n\u001b[1;32m      2\u001b[0m \u001b[38;5;28;01mfrom\u001b[39;00m \u001b[38;5;21;01mworkflow_calcium_imaging\u001b[39;00m\u001b[38;5;21;01m.\u001b[39;00m\u001b[38;5;21;01mingest\u001b[39;00m \u001b[38;5;28;01mimport\u001b[39;00m ingest_subjects, ingest_sessions \u001b[38;5;66;03m#csv loaders\u001b[39;00m\n\u001b[1;32m      4\u001b[0m \u001b[38;5;28;01mimport\u001b[39;00m \u001b[38;5;21;01mcsv\u001b[39;00m\n",
+      "File \u001b[0;32m~/Documents/dev/workflow-calcium-imaging/workflow_calcium_imaging/pipeline.py:74\u001b[0m\n\u001b[1;32m     67\u001b[0m     definition \u001b[39m=\u001b[39m \u001b[39m\"\"\"\u001b[39m\n\u001b[1;32m     68\u001b[0m \u001b[39m    scanner: varchar(32)\u001b[39m\n\u001b[1;32m     69\u001b[0m \u001b[39m    \u001b[39m\u001b[39m\"\"\"\u001b[39m\n\u001b[1;32m     72\u001b[0m \u001b[39m# ------------- Activate \"imaging\" schema -------------\u001b[39;00m\n\u001b[0;32m---> 74\u001b[0m imaging\u001b[39m.\u001b[39;49mactivate(db_prefix \u001b[39m+\u001b[39;49m \u001b[39m\"\u001b[39;49m\u001b[39mimaging\u001b[39;49m\u001b[39m\"\u001b[39;49m, db_prefix \u001b[39m+\u001b[39;49m \u001b[39m\"\u001b[39;49m\u001b[39mscan\u001b[39;49m\u001b[39m\"\u001b[39;49m, linking_module\u001b[39m=\u001b[39;49m\u001b[39m__name__\u001b[39;49m)\n\u001b[1;32m     76\u001b[0m \u001b[39m# ------------- Activate \"analysis\" schema ------------\u001b[39;00m\n\u001b[1;32m     78\u001b[0m analysis\u001b[39m.\u001b[39mactivate(db_prefix \u001b[39m+\u001b[39m \u001b[39m\"\u001b[39m\u001b[39manalysis\u001b[39m\u001b[39m\"\u001b[39m, linking_module\u001b[39m=\u001b[39m\u001b[39m__name__\u001b[39m)\n",
+      "File \u001b[0;32m~/Documents/dev/element-calcium-imaging/element_calcium_imaging/imaging.py:58\u001b[0m, in \u001b[0;36mactivate\u001b[0;34m(imaging_schema_name, scan_schema_name, create_schema, create_tables, linking_module)\u001b[0m\n\u001b[1;32m     50\u001b[0m _linking_module \u001b[39m=\u001b[39m linking_module\n\u001b[1;32m     52\u001b[0m scan\u001b[39m.\u001b[39mactivate(\n\u001b[1;32m     53\u001b[0m     scan_schema_name,\n\u001b[1;32m     54\u001b[0m     create_schema\u001b[39m=\u001b[39mcreate_schema,\n\u001b[1;32m     55\u001b[0m     create_tables\u001b[39m=\u001b[39mcreate_tables,\n\u001b[1;32m     56\u001b[0m     linking_module\u001b[39m=\u001b[39mlinking_module,\n\u001b[1;32m     57\u001b[0m )\n\u001b[0;32m---> 58\u001b[0m schema\u001b[39m.\u001b[39;49mactivate(\n\u001b[1;32m     59\u001b[0m     imaging_schema_name,\n\u001b[1;32m     60\u001b[0m     create_schema\u001b[39m=\u001b[39;49mcreate_schema,\n\u001b[1;32m     61\u001b[0m     create_tables\u001b[39m=\u001b[39;49mcreate_tables,\n\u001b[1;32m     62\u001b[0m     add_objects\u001b[39m=\u001b[39;49m_linking_module\u001b[39m.\u001b[39;49m\u001b[39m__dict__\u001b[39;49m,\n\u001b[1;32m     63\u001b[0m )\n",
+      "File \u001b[0;32m~/miniforge3/envs/ele/lib/python3.9/site-packages/datajoint/schemas.py:155\u001b[0m, in \u001b[0;36mSchema.activate\u001b[0;34m(self, schema_name, connection, create_schema, create_tables, add_objects)\u001b[0m\n\u001b[1;32m    153\u001b[0m \u001b[39mif\u001b[39;00m \u001b[39mself\u001b[39m\u001b[39m.\u001b[39madd_objects:\n\u001b[1;32m    154\u001b[0m     context \u001b[39m=\u001b[39m \u001b[39mdict\u001b[39m(context, \u001b[39m*\u001b[39m\u001b[39m*\u001b[39m\u001b[39mself\u001b[39m\u001b[39m.\u001b[39madd_objects)\n\u001b[0;32m--> 155\u001b[0m \u001b[39mself\u001b[39;49m\u001b[39m.\u001b[39;49m_decorate_master(\u001b[39mcls\u001b[39;49m, context)\n",
+      "File \u001b[0;32m~/miniforge3/envs/ele/lib/python3.9/site-packages/datajoint/schemas.py:198\u001b[0m, in \u001b[0;36mSchema._decorate_master\u001b[0;34m(self, cls, context)\u001b[0m\n\u001b[1;32m    196\u001b[0m part\u001b[39m.\u001b[39m_master \u001b[39m=\u001b[39m \u001b[39mcls\u001b[39m\n\u001b[1;32m    197\u001b[0m \u001b[39m# allow addressing master by name or keyword 'master'\u001b[39;00m\n\u001b[0;32m--> 198\u001b[0m \u001b[39mself\u001b[39;49m\u001b[39m.\u001b[39;49m_decorate_table(\n\u001b[1;32m    199\u001b[0m     part,\n\u001b[1;32m    200\u001b[0m     context\u001b[39m=\u001b[39;49m\u001b[39mdict\u001b[39;49m(\n\u001b[1;32m    201\u001b[0m         context, master\u001b[39m=\u001b[39;49m\u001b[39mcls\u001b[39;49m, \u001b[39mself\u001b[39;49m\u001b[39m=\u001b[39;49mpart, \u001b[39m*\u001b[39;49m\u001b[39m*\u001b[39;49m{\u001b[39mcls\u001b[39;49m\u001b[39m.\u001b[39;49m\u001b[39m__name__\u001b[39;49m: \u001b[39mcls\u001b[39;49m}\n\u001b[1;32m    202\u001b[0m     ),\n\u001b[1;32m    203\u001b[0m )\n",
+      "File \u001b[0;32m~/miniforge3/envs/ele/lib/python3.9/site-packages/datajoint/schemas.py:224\u001b[0m, in \u001b[0;36mSchema._decorate_table\u001b[0;34m(self, table_class, context, assert_declared)\u001b[0m\n\u001b[1;32m    222\u001b[0m \u001b[39m# instantiate the class, declare the table if not already\u001b[39;00m\n\u001b[1;32m    223\u001b[0m instance \u001b[39m=\u001b[39m table_class()\n\u001b[0;32m--> 224\u001b[0m is_declared \u001b[39m=\u001b[39m instance\u001b[39m.\u001b[39;49mis_declared\n\u001b[1;32m    225\u001b[0m \u001b[39mif\u001b[39;00m \u001b[39mnot\u001b[39;00m is_declared:\n\u001b[1;32m    226\u001b[0m     \u001b[39mif\u001b[39;00m \u001b[39mnot\u001b[39;00m \u001b[39mself\u001b[39m\u001b[39m.\u001b[39mcreate_tables \u001b[39mor\u001b[39;00m assert_declared:\n",
+      "File \u001b[0;32m~/miniforge3/envs/ele/lib/python3.9/site-packages/datajoint/table.py:247\u001b[0m, in \u001b[0;36mTable.is_declared\u001b[0;34m(self)\u001b[0m\n\u001b[1;32m    241\u001b[0m \u001b[39m@property\u001b[39m\n\u001b[1;32m    242\u001b[0m \u001b[39mdef\u001b[39;00m \u001b[39mis_declared\u001b[39m(\u001b[39mself\u001b[39m):\n\u001b[1;32m    243\u001b[0m     \u001b[39m\"\"\"\u001b[39;00m\n\u001b[1;32m    244\u001b[0m \u001b[39m    :return: True is the table is declared in the schema.\u001b[39;00m\n\u001b[1;32m    245\u001b[0m \u001b[39m    \"\"\"\u001b[39;00m\n\u001b[1;32m    246\u001b[0m     \u001b[39mreturn\u001b[39;00m (\n\u001b[0;32m--> 247\u001b[0m         \u001b[39mself\u001b[39;49m\u001b[39m.\u001b[39;49mconnection\u001b[39m.\u001b[39;49mquery(\n\u001b[1;32m    248\u001b[0m             \u001b[39m'\u001b[39;49m\u001b[39mSHOW TABLES in `\u001b[39;49m\u001b[39m{database}\u001b[39;49;00m\u001b[39m` LIKE \u001b[39;49m\u001b[39m\"\u001b[39;49m\u001b[39m{table_name}\u001b[39;49;00m\u001b[39m\"\u001b[39;49m\u001b[39m'\u001b[39;49m\u001b[39m.\u001b[39;49mformat(\n\u001b[1;32m    249\u001b[0m                 database\u001b[39m=\u001b[39;49m\u001b[39mself\u001b[39;49m\u001b[39m.\u001b[39;49mdatabase, table_name\u001b[39m=\u001b[39;49m\u001b[39mself\u001b[39;49m\u001b[39m.\u001b[39;49mtable_name\n\u001b[1;32m    250\u001b[0m             )\n\u001b[1;32m    251\u001b[0m         )\u001b[39m.\u001b[39mrowcount\n\u001b[1;32m    252\u001b[0m         \u001b[39m>\u001b[39m \u001b[39m0\u001b[39m\n\u001b[1;32m    253\u001b[0m     )\n",
+      "File \u001b[0;32m~/miniforge3/envs/ele/lib/python3.9/site-packages/datajoint/connection.py:340\u001b[0m, in \u001b[0;36mConnection.query\u001b[0;34m(self, query, args, as_dict, suppress_warnings, reconnect)\u001b[0m\n\u001b[1;32m    338\u001b[0m cursor \u001b[39m=\u001b[39m \u001b[39mself\u001b[39m\u001b[39m.\u001b[39m_conn\u001b[39m.\u001b[39mcursor(cursor\u001b[39m=\u001b[39mcursor_class)\n\u001b[1;32m    339\u001b[0m \u001b[39mtry\u001b[39;00m:\n\u001b[0;32m--> 340\u001b[0m     \u001b[39mself\u001b[39;49m\u001b[39m.\u001b[39;49m_execute_query(cursor, query, args, suppress_warnings)\n\u001b[1;32m    341\u001b[0m \u001b[39mexcept\u001b[39;00m errors\u001b[39m.\u001b[39mLostConnectionError:\n\u001b[1;32m    342\u001b[0m     \u001b[39mif\u001b[39;00m \u001b[39mnot\u001b[39;00m reconnect:\n",
+      "File \u001b[0;32m~/miniforge3/envs/ele/lib/python3.9/site-packages/datajoint/connection.py:294\u001b[0m, in \u001b[0;36mConnection._execute_query\u001b[0;34m(cursor, query, args, suppress_warnings)\u001b[0m\n\u001b[1;32m    291\u001b[0m         \u001b[39mif\u001b[39;00m suppress_warnings:\n\u001b[1;32m    292\u001b[0m             \u001b[39m# suppress all warnings arising from underlying SQL library\u001b[39;00m\n\u001b[1;32m    293\u001b[0m             warnings\u001b[39m.\u001b[39msimplefilter(\u001b[39m\"\u001b[39m\u001b[39mignore\u001b[39m\u001b[39m\"\u001b[39m)\n\u001b[0;32m--> 294\u001b[0m         cursor\u001b[39m.\u001b[39;49mexecute(query, args)\n\u001b[1;32m    295\u001b[0m \u001b[39mexcept\u001b[39;00m client\u001b[39m.\u001b[39merr\u001b[39m.\u001b[39mError \u001b[39mas\u001b[39;00m err:\n\u001b[1;32m    296\u001b[0m     \u001b[39mraise\u001b[39;00m translate_query_error(err, query)\n",
+      "File \u001b[0;32m~/miniforge3/envs/ele/lib/python3.9/site-packages/pymysql/cursors.py:148\u001b[0m, in \u001b[0;36mCursor.execute\u001b[0;34m(self, query, args)\u001b[0m\n\u001b[1;32m    144\u001b[0m     \u001b[39mpass\u001b[39;00m\n\u001b[1;32m    146\u001b[0m query \u001b[39m=\u001b[39m \u001b[39mself\u001b[39m\u001b[39m.\u001b[39mmogrify(query, args)\n\u001b[0;32m--> 148\u001b[0m result \u001b[39m=\u001b[39m \u001b[39mself\u001b[39;49m\u001b[39m.\u001b[39;49m_query(query)\n\u001b[1;32m    149\u001b[0m \u001b[39mself\u001b[39m\u001b[39m.\u001b[39m_executed \u001b[39m=\u001b[39m query\n\u001b[1;32m    150\u001b[0m \u001b[39mreturn\u001b[39;00m result\n",
+      "File \u001b[0;32m~/miniforge3/envs/ele/lib/python3.9/site-packages/pymysql/cursors.py:310\u001b[0m, in \u001b[0;36mCursor._query\u001b[0;34m(self, q)\u001b[0m\n\u001b[1;32m    308\u001b[0m \u001b[39mself\u001b[39m\u001b[39m.\u001b[39m_last_executed \u001b[39m=\u001b[39m q\n\u001b[1;32m    309\u001b[0m \u001b[39mself\u001b[39m\u001b[39m.\u001b[39m_clear_result()\n\u001b[0;32m--> 310\u001b[0m conn\u001b[39m.\u001b[39;49mquery(q)\n\u001b[1;32m    311\u001b[0m \u001b[39mself\u001b[39m\u001b[39m.\u001b[39m_do_get_result()\n\u001b[1;32m    312\u001b[0m \u001b[39mreturn\u001b[39;00m \u001b[39mself\u001b[39m\u001b[39m.\u001b[39mrowcount\n",
+      "File \u001b[0;32m~/miniforge3/envs/ele/lib/python3.9/site-packages/pymysql/connections.py:548\u001b[0m, in \u001b[0;36mConnection.query\u001b[0;34m(self, sql, unbuffered)\u001b[0m\n\u001b[1;32m    546\u001b[0m     sql \u001b[39m=\u001b[39m sql\u001b[39m.\u001b[39mencode(\u001b[39mself\u001b[39m\u001b[39m.\u001b[39mencoding, \u001b[39m\"\u001b[39m\u001b[39msurrogateescape\u001b[39m\u001b[39m\"\u001b[39m)\n\u001b[1;32m    547\u001b[0m \u001b[39mself\u001b[39m\u001b[39m.\u001b[39m_execute_command(COMMAND\u001b[39m.\u001b[39mCOM_QUERY, sql)\n\u001b[0;32m--> 548\u001b[0m \u001b[39mself\u001b[39m\u001b[39m.\u001b[39m_affected_rows \u001b[39m=\u001b[39m \u001b[39mself\u001b[39;49m\u001b[39m.\u001b[39;49m_read_query_result(unbuffered\u001b[39m=\u001b[39;49munbuffered)\n\u001b[1;32m    549\u001b[0m \u001b[39mreturn\u001b[39;00m \u001b[39mself\u001b[39m\u001b[39m.\u001b[39m_affected_rows\n",
+      "File \u001b[0;32m~/miniforge3/envs/ele/lib/python3.9/site-packages/pymysql/connections.py:775\u001b[0m, in \u001b[0;36mConnection._read_query_result\u001b[0;34m(self, unbuffered)\u001b[0m\n\u001b[1;32m    773\u001b[0m \u001b[39melse\u001b[39;00m:\n\u001b[1;32m    774\u001b[0m     result \u001b[39m=\u001b[39m MySQLResult(\u001b[39mself\u001b[39m)\n\u001b[0;32m--> 775\u001b[0m     result\u001b[39m.\u001b[39;49mread()\n\u001b[1;32m    776\u001b[0m \u001b[39mself\u001b[39m\u001b[39m.\u001b[39m_result \u001b[39m=\u001b[39m result\n\u001b[1;32m    777\u001b[0m \u001b[39mif\u001b[39;00m result\u001b[39m.\u001b[39mserver_status \u001b[39mis\u001b[39;00m \u001b[39mnot\u001b[39;00m \u001b[39mNone\u001b[39;00m:\n",
+      "File \u001b[0;32m~/miniforge3/envs/ele/lib/python3.9/site-packages/pymysql/connections.py:1156\u001b[0m, in \u001b[0;36mMySQLResult.read\u001b[0;34m(self)\u001b[0m\n\u001b[1;32m   1154\u001b[0m \u001b[39mdef\u001b[39;00m \u001b[39mread\u001b[39m(\u001b[39mself\u001b[39m):\n\u001b[1;32m   1155\u001b[0m     \u001b[39mtry\u001b[39;00m:\n\u001b[0;32m-> 1156\u001b[0m         first_packet \u001b[39m=\u001b[39m \u001b[39mself\u001b[39;49m\u001b[39m.\u001b[39;49mconnection\u001b[39m.\u001b[39;49m_read_packet()\n\u001b[1;32m   1158\u001b[0m         \u001b[39mif\u001b[39;00m first_packet\u001b[39m.\u001b[39mis_ok_packet():\n\u001b[1;32m   1159\u001b[0m             \u001b[39mself\u001b[39m\u001b[39m.\u001b[39m_read_ok_packet(first_packet)\n",
+      "File \u001b[0;32m~/miniforge3/envs/ele/lib/python3.9/site-packages/pymysql/connections.py:692\u001b[0m, in \u001b[0;36mConnection._read_packet\u001b[0;34m(self, packet_type)\u001b[0m\n\u001b[1;32m    690\u001b[0m buff \u001b[39m=\u001b[39m \u001b[39mbytearray\u001b[39m()\n\u001b[1;32m    691\u001b[0m \u001b[39mwhile\u001b[39;00m \u001b[39mTrue\u001b[39;00m:\n\u001b[0;32m--> 692\u001b[0m     packet_header \u001b[39m=\u001b[39m \u001b[39mself\u001b[39;49m\u001b[39m.\u001b[39;49m_read_bytes(\u001b[39m4\u001b[39;49m)\n\u001b[1;32m    693\u001b[0m     \u001b[39m# if DEBUG: dump_packet(packet_header)\u001b[39;00m\n\u001b[1;32m    695\u001b[0m     btrl, btrh, packet_number \u001b[39m=\u001b[39m struct\u001b[39m.\u001b[39munpack(\u001b[39m\"\u001b[39m\u001b[39m<HBB\u001b[39m\u001b[39m\"\u001b[39m, packet_header)\n",
+      "File \u001b[0;32m~/miniforge3/envs/ele/lib/python3.9/site-packages/pymysql/connections.py:732\u001b[0m, in \u001b[0;36mConnection._read_bytes\u001b[0;34m(self, num_bytes)\u001b[0m\n\u001b[1;32m    730\u001b[0m \u001b[39mwhile\u001b[39;00m \u001b[39mTrue\u001b[39;00m:\n\u001b[1;32m    731\u001b[0m     \u001b[39mtry\u001b[39;00m:\n\u001b[0;32m--> 732\u001b[0m         data \u001b[39m=\u001b[39m \u001b[39mself\u001b[39;49m\u001b[39m.\u001b[39;49m_rfile\u001b[39m.\u001b[39;49mread(num_bytes)\n\u001b[1;32m    733\u001b[0m         \u001b[39mbreak\u001b[39;00m\n\u001b[1;32m    734\u001b[0m     \u001b[39mexcept\u001b[39;00m (\u001b[39mIOError\u001b[39;00m, \u001b[39mOSError\u001b[39;00m) \u001b[39mas\u001b[39;00m e:\n",
+      "File \u001b[0;32m~/miniforge3/envs/ele/lib/python3.9/socket.py:704\u001b[0m, in \u001b[0;36mSocketIO.readinto\u001b[0;34m(self, b)\u001b[0m\n\u001b[1;32m    702\u001b[0m \u001b[39mwhile\u001b[39;00m \u001b[39mTrue\u001b[39;00m:\n\u001b[1;32m    703\u001b[0m     \u001b[39mtry\u001b[39;00m:\n\u001b[0;32m--> 704\u001b[0m         \u001b[39mreturn\u001b[39;00m \u001b[39mself\u001b[39;49m\u001b[39m.\u001b[39;49m_sock\u001b[39m.\u001b[39;49mrecv_into(b)\n\u001b[1;32m    705\u001b[0m     \u001b[39mexcept\u001b[39;00m timeout:\n\u001b[1;32m    706\u001b[0m         \u001b[39mself\u001b[39m\u001b[39m.\u001b[39m_timeout_occurred \u001b[39m=\u001b[39m \u001b[39mTrue\u001b[39;00m\n",
+      "\u001b[0;31mKeyboardInterrupt\u001b[0m: "
+     ]
+    }
+   ],
    "source": [
+    "\n",
     "from workflow_calcium_imaging.pipeline import session, imaging # import schemas\n",
     "from workflow_calcium_imaging.ingest import ingest_subjects, ingest_sessions #csv loaders\n",
     "\n",
+    "import csv\n",
+    "\n",
+    "sessions_csv_path = f\"/home/{dj.config['database.user']}/sessions.csv\"\n",
+    "with open(sessions_csv_path, 'w', newline='') as f:\n",
+    "    csv_writer = csv.writer(f)\n",
+    "    csv_writer.writerow([\"subject\",\"session_dir\"])\n",
+    "    csv_writer.writerow([\"subject3\",\"inbox/0_1_0a2/subject3/210107_run00_orientation_8dir/\"])\n",
+    "\n",
     "ingest_subjects(subject_csv_path=\"/home/user_data/subjects.csv\")\n",
-    "ingest_sessions(session_csv_path=\"/home/user_data/sessions.csv\")\n",
+    "ingest_sessions(session_csv_path=sessions_csv_path)\n",
     "\n",
     "params_suite2p = {'look_one_level_down': 0.0,\n",
     "                  'fast_disk': [],\n",
@@ -223,10 +268,15 @@
     "\n",
     "process.run()\n",
     "session_key = (session.Session & 'subject=\"subject3\"').fetch('KEY')[0]\n",
-    "imaging.ProcessingTask.insert1(dict(session_key, \n",
-    "                                    scan_id=0,\n",
-    "                                    paramset_idx=0,\n",
-    "                                    processing_output_dir='subject3/210107_run00_orientation_8dir/suite2p'), skip_duplicates=True)\n",
+    "imaging.ProcessingTask.insert1(\n",
+    "    dict(\n",
+    "        session_key, \n",
+    "        scan_id=0,\n",
+    "        paramset_idx=0,\n",
+    "        processing_output_dir='inbox/0_1_0a2/subject3/210107_run00_orientation_8dir/suite2p'\n",
+    "    ),\n",
+    "    skip_duplicates=True\n",
+    ")\n",
     "process.run()"
    ]
   },
@@ -381,7 +431,7 @@
    "metadata": {},
    "outputs": [],
    "source": [
-    "trace = (query_trace).fetch('fluorescence')"
+    "trace = (query_trace).fetch('fluorescence')[0]"
    ]
   },
   {
@@ -549,13 +599,20 @@
    "metadata": {},
    "outputs": [],
    "source": [
-    "os.makedirs('/home/inbox/0_1_0a2/subject3/210107_run00_orientation_8dir/suite2p_1', exist_ok=True)\n",
+    "output_dir = f\"{dj.config['database.user']}\"\n",
+    "print(output_dir)\n",
+    "os.makedirs(output_dir, exist_ok=True)\n",
     "\n",
-    "imaging.ProcessingTask.insert1(dict(subject='subject3', \n",
-    "                                    session_datetime='2022-09-01 19:16:44', \n",
-    "                                    scan_id=0,\n",
-    "                                    paramset_idx=1,\n",
-    "                                    processing_output_dir='subject3/210107_run00_orientation_8dir/suite2p_1'))"
+    "imaging.ProcessingTask.insert1(\n",
+    "    dict(\n",
+    "        subject='subject3', \n",
+    "        session_datetime='2022-09-01 19:16:44', \n",
+    "        scan_id=0,\n",
+    "        paramset_idx=1,\n",
+    "        processing_output_dir=output_dir,\n",
+    "        task_mode='trigger'\n",
+    "    )\n",
+    ")"
    ]
   },
   {
diff --git a/notebooks/py_scripts/2022-allen-institute-workshop.py b/notebooks/py_scripts/2022-allen-institute-workshop.py
index 391e838..bfef113 100644
--- a/notebooks/py_scripts/2022-allen-institute-workshop.py
+++ b/notebooks/py_scripts/2022-allen-institute-workshop.py
@@ -39,7 +39,7 @@
 username_as_prefix = dj.config["database.user"] + "_img_"
 dj.config['custom'] = {
     'database.prefix': username_as_prefix,
-    'imaging_root_data_dir': '/home/inbox/0_1_0a2/'
+    'imaging_root_data_dir': '/home/'
 }
 # -
 
@@ -53,14 +53,23 @@
 
 # ## Populating the database
 
-# Next, we'll populate these schema using steps from [04-automate](04-automate-optional.ipynb). If your schema are already populated, you can skip this step. For more details on each of these steps, please visit [that notebook](04-automate-optional.ipynb).
+# Next, we'll populate these schema using some steps from [04-automate](04-automate-optional.ipynb). If your schema are already populated, you can skip this step. For more details on each of these steps, please visit [that notebook](04-automate-optional.ipynb). Additional steps ensure write permissions on output directories.
 
 # +
+
 from workflow_calcium_imaging.pipeline import session, imaging # import schemas
 from workflow_calcium_imaging.ingest import ingest_subjects, ingest_sessions #csv loaders
 
+import csv
+
+sessions_csv_path = f"/home/{dj.config['database.user']}/sessions.csv"
+with open(sessions_csv_path, 'w', newline='') as f:
+    csv_writer = csv.writer(f)
+    csv_writer.writerow(["subject","session_dir"])
+    csv_writer.writerow(["subject3","inbox/0_1_0a2/subject3/210107_run00_orientation_8dir/"])
+
 ingest_subjects(subject_csv_path="/home/user_data/subjects.csv")
-ingest_sessions(session_csv_path="/home/user_data/sessions.csv")
+ingest_sessions(session_csv_path=sessions_csv_path)
 
 params_suite2p = {'look_one_level_down': 0.0,
                   'fast_disk': [],
@@ -138,10 +147,15 @@
 
 process.run()
 session_key = (session.Session & 'subject="subject3"').fetch('KEY')[0]
-imaging.ProcessingTask.insert1(dict(session_key, 
-                                    scan_id=0,
-                                    paramset_idx=0,
-                                    processing_output_dir='subject3/210107_run00_orientation_8dir/suite2p'), skip_duplicates=True)
+imaging.ProcessingTask.insert1(
+    dict(
+        session_key, 
+        scan_id=0,
+        paramset_idx=0,
+        processing_output_dir='inbox/0_1_0a2/subject3/210107_run00_orientation_8dir/suite2p'
+    ),
+    skip_duplicates=True
+)
 process.run()
 # -
 
@@ -190,7 +204,7 @@
 
 # Fetch a fluorescence trace from the database.
 
-trace = (query_trace).fetch('fluorescence')
+trace = (query_trace).fetch('fluorescence')[0]
 
 # Plot the fluorescence trace.
 
@@ -285,13 +299,20 @@
 imaging.ProcessingParamSet()
 
 # +
-os.makedirs('/home/inbox/0_1_0a2/subject3/210107_run00_orientation_8dir/suite2p_1', exist_ok=True)
-
-imaging.ProcessingTask.insert1(dict(subject='subject3', 
-                                    session_datetime='2022-09-01 19:16:44', 
-                                    scan_id=0,
-                                    paramset_idx=1,
-                                    processing_output_dir='subject3/210107_run00_orientation_8dir/suite2p_1'))
+output_dir = f"{dj.config['database.user']}"
+print(output_dir)
+os.makedirs(output_dir, exist_ok=True)
+
+imaging.ProcessingTask.insert1(
+    dict(
+        subject='subject3', 
+        session_datetime='2022-09-01 19:16:44', 
+        scan_id=0,
+        paramset_idx=1,
+        processing_output_dir=output_dir,
+        task_mode='trigger'
+    )
+)
 # -
 
 imaging.ProcessingTask()
diff --git a/requirements.txt b/requirements.txt
index a097493..83cbf03 100644
--- a/requirements.txt
+++ b/requirements.txt
@@ -8,4 +8,5 @@ element-interface @ git+https://github.com/datajoint/element-interface.git
 ipykernel>=6.0.1
 jupytext>=1.13.7
 sbxreader
+nd2
 suite2p @ git+https://github.com/datajoint-company/suite2p.git
\ No newline at end of file

From 9c8761e7f58a3a11a326aa855b53eee4c61e7db0 Mon Sep 17 00:00:00 2001
From: Chris Broz <CBroz@datajoint.com>
Date: Wed, 21 Sep 2022 15:40:16 -0500
Subject: [PATCH 5/6] Revise Allen notebook 5

---
 notebooks/2022-allen-institute-workshop.ipynb         | 4 +++-
 notebooks/py_scripts/2022-allen-institute-workshop.py | 4 +++-
 2 files changed, 6 insertions(+), 2 deletions(-)

diff --git a/notebooks/2022-allen-institute-workshop.ipynb b/notebooks/2022-allen-institute-workshop.ipynb
index acc9e5b..44d088f 100644
--- a/notebooks/2022-allen-institute-workshop.ipynb
+++ b/notebooks/2022-allen-institute-workshop.ipynb
@@ -630,7 +630,9 @@
    "id": "637b9ec1",
    "metadata": {},
    "source": [
-    "Run Suite2p for the new parameter set and save the results to the respective tables."
+    "If you have write access to the data, you can then run Suite2p for the new parameter set and save the results to the respective tables.\n",
+    "\n",
+    "**Note:** CodeBook does not yet support write access for shared datasets. To see Suite2p triggering in action, contact Support@DataJoint.com to find out how to upload your own data."
    ]
   },
   {
diff --git a/notebooks/py_scripts/2022-allen-institute-workshop.py b/notebooks/py_scripts/2022-allen-institute-workshop.py
index bfef113..3429596 100644
--- a/notebooks/py_scripts/2022-allen-institute-workshop.py
+++ b/notebooks/py_scripts/2022-allen-institute-workshop.py
@@ -317,7 +317,9 @@
 
 imaging.ProcessingTask()
 
-# Run Suite2p for the new parameter set and save the results to the respective tables.
+# If you have write access to the data, you can then run Suite2p for the new parameter set and save the results to the respective tables.
+#
+# **Note:** CodeBook does not yet support write access for shared datasets. To see Suite2p triggering in action, contact Support@DataJoint.com to find out how to upload your own data.
 
 # +
 populate_settings = dict(display_progress=True)

From 27a9f4eea01006184345b2939ce25fdb87a1e503 Mon Sep 17 00:00:00 2001
From: Chris Broz <CBroz@datajoint.com>
Date: Wed, 21 Sep 2022 17:37:50 -0500
Subject: [PATCH 6/6] Revise Allen notebook 6 - read only sbxreader memmap
 requirement

---
 notebooks/2022-allen-institute-workshop.ipynb         | 6 ++----
 notebooks/py_scripts/2022-allen-institute-workshop.py | 6 ++----
 requirements.txt                                      | 2 +-
 3 files changed, 5 insertions(+), 9 deletions(-)

diff --git a/notebooks/2022-allen-institute-workshop.ipynb b/notebooks/2022-allen-institute-workshop.ipynb
index 44d088f..9d607ba 100644
--- a/notebooks/2022-allen-institute-workshop.ipynb
+++ b/notebooks/2022-allen-institute-workshop.ipynb
@@ -514,7 +514,7 @@
     "                  'h5py_key': 'data',\n",
     "                  'save_path0': [],\n",
     "                  'subfolders': [],\n",
-    "                  'nplanes': 1,\n",
+    "                  'nplanes': 4,\n",
     "                  'nchannels': 1,\n",
     "                  'functional_chan': 1,\n",
     "                  'tau': 1.0,\n",
@@ -630,9 +630,7 @@
    "id": "637b9ec1",
    "metadata": {},
    "source": [
-    "If you have write access to the data, you can then run Suite2p for the new parameter set and save the results to the respective tables.\n",
-    "\n",
-    "**Note:** CodeBook does not yet support write access for shared datasets. To see Suite2p triggering in action, contact Support@DataJoint.com to find out how to upload your own data."
+    "You can then run Suite2p for the new parameter set and save the results to the respective tables. For this dataset (4 channels, 4 depths, 7.5k frames), this may take several hours."
    ]
   },
   {
diff --git a/notebooks/py_scripts/2022-allen-institute-workshop.py b/notebooks/py_scripts/2022-allen-institute-workshop.py
index 3429596..1731d8f 100644
--- a/notebooks/py_scripts/2022-allen-institute-workshop.py
+++ b/notebooks/py_scripts/2022-allen-institute-workshop.py
@@ -237,7 +237,7 @@
                   'h5py_key': 'data',
                   'save_path0': [],
                   'subfolders': [],
-                  'nplanes': 1,
+                  'nplanes': 4,
                   'nchannels': 1,
                   'functional_chan': 1,
                   'tau': 1.0,
@@ -317,9 +317,7 @@
 
 imaging.ProcessingTask()
 
-# If you have write access to the data, you can then run Suite2p for the new parameter set and save the results to the respective tables.
-#
-# **Note:** CodeBook does not yet support write access for shared datasets. To see Suite2p triggering in action, contact Support@DataJoint.com to find out how to upload your own data.
+# You can then run Suite2p for the new parameter set and save the results to the respective tables. For this dataset (4 channels, 4 depths, 7.5k frames), this may take several hours.
 
 # +
 populate_settings = dict(display_progress=True)
diff --git a/requirements.txt b/requirements.txt
index 83cbf03..06dfb4e 100644
--- a/requirements.txt
+++ b/requirements.txt
@@ -7,6 +7,6 @@ element-event @ git+https://github.com/datajoint/element-event.git
 element-interface @ git+https://github.com/datajoint/element-interface.git
 ipykernel>=6.0.1
 jupytext>=1.13.7
-sbxreader
+sbxreader @ git+https://github.com/CBroz1/sbxreader
 nd2
 suite2p @ git+https://github.com/datajoint-company/suite2p.git
\ No newline at end of file