Merge pull request #106 from kabilar/main

Update demo notebooks

.devcontainer/Dockerfile

@@ -24,4 +24,6 @@ ENV DJ_HOST fakeservices.datajoint.io
 ENV DJ_USER root
 ENV DJ_PASS simple
 
-ENV DJ_PUBLIC_S3_MOUNT_PATH /workspaces/workflow-calcium-imaging/example_data
+ENV DJ_PUBLIC_S3_MOUNT_PATH /workspaces/workflow-calcium-imaging/example_data
+ENV IMAGING_ROOT_DATA_DIR /workspaces/workflow-calcium-imaging/example_data
+ENV DATABASE_PREFIX neuro_

notebooks/demo_prepare.ipynb

@@ -0,0 +1,184 @@
+{
+ "cells": [
+  {
+   "attachments": {},
+   "cell_type": "markdown",
+   "metadata": {},
+   "source": [
+    "### Demo Preparation Notebook\n",
+    "\n",
+    "**Please Note**: This notebook and demo are NOT intended to be used as learning materials. To gain\n",
+    "a thorough understanding of the DataJoint workflow for calcium imaging, please\n",
+    "see the [`tutorial`](./tutorial.ipynb) notebook."
+   ]
+  },
+  {
+   "cell_type": "code",
+   "execution_count": null,
+   "metadata": {},
+   "outputs": [],
+   "source": [
+    "import datajoint as dj\n",
+    "import datetime\n",
+    "import numpy as np\n",
+    "from workflow_calcium_imaging.pipeline import subject, session, scan, imaging, Equipment\n",
+    "from element_calcium_imaging import imaging_report\n",
+    "import suite2p"
+   ]
+  },
+  {
+   "cell_type": "code",
+   "execution_count": null,
+   "metadata": {},
+   "outputs": [],
+   "source": [
+    "subject.Subject.insert1(\n",
+    "    dict(\n",
+    "        subject='subject1',\n",
+    "        subject_birth_date='2023-01-01',\n",
+    "        sex='U',\n",
+    "    )\n",
+    ")"
+   ]
+  },
+  {
+   "cell_type": "code",
+   "execution_count": null,
+   "metadata": {},
+   "outputs": [],
+   "source": [
+    "Equipment.insert1(dict(scanner=\"Mesoscope\"))"
+   ]
+  },
+  {
+   "cell_type": "code",
+   "execution_count": null,
+   "metadata": {},
+   "outputs": [],
+   "source": [
+    "session_key = dict(subject='subject1', \n",
+    "                   session_datetime=datetime.datetime.now())\n",
+    "\n",
+    "session.Session.insert1(session_key)\n",
+    "\n",
+    "session.SessionDirectory.insert1(\n",
+    "    dict(\n",
+    "        session_key, \n",
+    "        session_dir='subject1/session1'\n",
+    "    )\n",
+    ")"
+   ]
+  },
+  {
+   "cell_type": "code",
+   "execution_count": null,
+   "metadata": {},
+   "outputs": [],
+   "source": [
+    "scan.Scan.insert1(\n",
+    "    dict(\n",
+    "        session_key,\n",
+    "        scan_id=0,\n",
+    "        scanner=\"Mesoscope\",\n",
+    "        acq_software='ScanImage',\n",
+    "    )\n",
+    ")"
+   ]
+  },
+  {
+   "cell_type": "code",
+   "execution_count": null,
+   "metadata": {},
+   "outputs": [],
+   "source": [
+    "populate_settings = {\"display_progress\": True}\n",
+    "\n",
+    "scan.ScanInfo.populate(**populate_settings)"
+   ]
+  },
+  {
+   "cell_type": "code",
+   "execution_count": null,
+   "metadata": {},
+   "outputs": [],
+   "source": [
+    "suite2p_params = suite2p.default_ops()\n",
+    "suite2p_params['nonrigid']=False\n",
+    "\n",
+    "imaging.ProcessingParamSet.insert_new_params(\n",
+    "    processing_method=\"suite2p\",\n",
+    "    paramset_idx=0,\n",
+    "    params=suite2p_params,\n",
+    "    paramset_desc='Default parameter set for suite2p'\n",
+    ")"
+   ]
+  },
+  {
+   "cell_type": "code",
+   "execution_count": null,
+   "metadata": {},
+   "outputs": [],
+   "source": [
+    "imaging.ProcessingTask.insert1(\n",
+    "    dict(\n",
+    "        session_key,\n",
+    "        scan_id=0,\n",
+    "        paramset_idx=0,\n",
+    "        task_mode='load', # load or trigger\n",
+    "        processing_output_dir='subject1/session1/suite2p',\n",
+    "    )\n",
+    ")\n",
+    "\n",
+    "imaging.Processing.populate(**populate_settings)"
+   ]
+  },
+  {
+   "cell_type": "code",
+   "execution_count": null,
+   "metadata": {},
+   "outputs": [],
+   "source": [
+    "imaging.Curation.insert1(\n",
+    "    dict(\n",
+    "        session_key,\n",
+    "        scan_id=0,\n",
+    "        paramset_idx=0,\n",
+    "        curation_id=0,\n",
+    "        curation_time=datetime.datetime.now(),\n",
+    "        curation_output_dir='subject1/session1/suite2p',\n",
+    "        manual_curation=False,\n",
+    "    )\n",
+    ")\n",
+    "\n",
+    "imaging.MotionCorrection.populate(**populate_settings)\n",
+    "imaging.Segmentation.populate(**populate_settings)\n",
+    "imaging.Fluorescence.populate(**populate_settings)\n",
+    "imaging.Activity.populate(**populate_settings)\n",
+    "imaging_report.ScanLevelReport.populate(**populate_settings)\n",
+    "imaging_report.TraceReport.populate(**populate_settings)"
+   ]
+  }
+ ],
+ "metadata": {
+  "kernelspec": {
+   "display_name": "Python 3",
+   "language": "python",
+   "name": "python3"
+  },
+  "language_info": {
+   "codemirror_mode": {
+    "name": "ipython",
+    "version": 3
+   },
+   "file_extension": ".py",
+   "mimetype": "text/x-python",
+   "name": "python",
+   "nbconvert_exporter": "python",
+   "pygments_lexer": "ipython3",
+   "version": "3.7.16"
+  },
+  "orig_nbformat": 4
+ },
+ "nbformat": 4,
+ "nbformat_minor": 2
+}

notebooks/demo_run.ipynb

@@ -0,0 +1,154 @@
+{
+ "cells": [
+  {
+   "attachments": {},
+   "cell_type": "markdown",
+   "metadata": {},
+   "source": [
+    "# DataJoint Workflow for Calcium Imaging\n",
+    "\n",
+    "+ This notebook demonstrates using the open-source DataJoint Element to build a workflow for\n",
+    "calcium imaging.\n",
+    "+ For a detailed tutorial, please see the [tutorial notebook](./tutorial.ipynb)."
+   ]
+  },
+  {
+   "attachments": {},
+   "cell_type": "markdown",
+   "metadata": {},
+   "source": [
+    "<img src =../images/rawscans.gif title=\"value\" width=\"200\" height=\"200\"/>\n",
+    "<img src =../images/motioncorrectedscans.gif width=\"200\" height=\"200\"/>\n",
+    "<img src =../images/cellsegmentation.png width=\"200\" height=\"200\"/>\n",
+    "<img src =../images/calciumtraces.png width=\"200\" height=\"200\"/> \n",
+    "\n",
+    "Left to right: Raw scans, Motion corrected scans, Cell segmentations, Calcium events"
+   ]
+  },
+  {
+   "attachments": {},
+   "cell_type": "markdown",
+   "metadata": {},
+   "source": [
+    "### Import dependencies"
+   ]
+  },
+  {
+   "cell_type": "code",
+   "execution_count": null,
+   "metadata": {
+    "tags": [
+     "hide-input"
+    ]
+   },
+   "outputs": [],
+   "source": [
+    "import datajoint as dj\n",
+    "from workflow_calcium_imaging.pipeline import subject, session, scan, imaging\n",
+    "from element_calcium_imaging.plotting.widget import main"
+   ]
+  },
+  {
+   "attachments": {},
+   "cell_type": "markdown",
+   "metadata": {},
+   "source": [
+    "### View workflow"
+   ]
+  },
+  {
+   "cell_type": "code",
+   "execution_count": null,
+   "metadata": {},
+   "outputs": [],
+   "source": [
+    "dj.Diagram(subject.Subject) + dj.Diagram(session.Session) + dj.Diagram(scan) + dj.Diagram(imaging)"
+   ]
+  },
+  {
+   "attachments": {},
+   "cell_type": "markdown",
+   "metadata": {},
+   "source": [
+    "### Insert an entry in a manual table by calling the `insert()` method\n",
+    "\n",
+    "```python\n",
+    "subject.Subject.insert1(\n",
+    "     dict(subject='subject1',\n",
+    "          subject_birth_date='2023-01-01',\n",
+    "          sex='U'\n",
+    "     )\n",
+    ")\n",
+    "```"
+   ]
+  },
+  {
+   "attachments": {},
+   "cell_type": "markdown",
+   "metadata": {},
+   "source": [
+    "### Automatically process data with the `populate()` method\n",
+    "\n",
+    "+ Once data is inserted into manual tables, the `populate()` function automatically runs the ingestion and processing routines. \n",
+    "\n",
+    "+ For example, to run Suite2p processing in the `Processing` table:\n",
+    "\n",
+    "     ```python\n",
+    "     imaging.Processing.populate()\n",
+    "     ```"
+   ]
+  },
+  {
+   "attachments": {},
+   "cell_type": "markdown",
+   "metadata": {},
+   "source": [
+    "### Visualize processed data"
+   ]
+  },
+  {
+   "cell_type": "code",
+   "execution_count": null,
+   "metadata": {},
+   "outputs": [],
+   "source": [
+    "main(imaging)"
+   ]
+  },
+  {
+   "attachments": {},
+   "cell_type": "markdown",
+   "metadata": {},
+   "source": [
+    "For an in-depth tutorial please see the [tutorial notebook](./tutorial.ipynb)."
+   ]
+  }
+ ],
+ "metadata": {
+  "kernelspec": {
+   "display_name": "Python 3",
+   "language": "python",
+   "name": "python3"
+  },
+  "language_info": {
+   "codemirror_mode": {
+    "name": "ipython",
+    "version": 3
+   },
+   "file_extension": ".py",
+   "mimetype": "text/x-python",
+   "name": "python",
+   "nbconvert_exporter": "python",
+   "pygments_lexer": "ipython3",
+   "version": "3.9.12"
+  },
+  "orig_nbformat": 4,
+  "vscode": {
+   "interpreter": {
+    "hash": "949777d72b0d2535278d3dc13498b2535136f6dfe0678499012e853ee9abcab1"
+   }
+  }
+ },
+ "nbformat": 4,
+ "nbformat_minor": 2
+}