From bc98d63ba292bfb77eafce12d2930d3c321a29b4 Mon Sep 17 00:00:00 2001 From: Kabilar Gunalan Date: Wed, 8 Mar 2023 13:24:20 -0600 Subject: [PATCH 01/11] Update `prepare_demo` --- notebooks/prepare_demo.ipynb | 172 +++++++++++------------------------ 1 file changed, 54 insertions(+), 118 deletions(-) diff --git a/notebooks/prepare_demo.ipynb b/notebooks/prepare_demo.ipynb index 7f6ac16..8831e7b 100644 --- a/notebooks/prepare_demo.ipynb +++ b/notebooks/prepare_demo.ipynb @@ -5,10 +5,10 @@ "cell_type": "markdown", "metadata": {}, "source": [ - "### Calcium Imaging Demo Preparation Notebook\n", + "### Demo Preparation Notebook\n", "\n", "**Please Note**: This notebook and demo are NOT intended to be used as learning materials. To gain\n", - "a thorough understanding of the DataJoint workflows for calcium imaging, please\n", + "a thorough understanding of the DataJoint workflow for calcium imaging, please\n", "see our [`tutorial`](./tutorial.ipynb) notebook." ] }, @@ -34,9 +34,10 @@ "\n", "import datajoint as dj\n", "import datetime\n", - "import matplotlib.pyplot as plt\n", "import numpy as np\n", - "from workflow_calcium_imaging.pipeline import subject, session, scan, imaging" + "from workflow_calcium_imaging.pipeline import subject, session, scan, imaging\n", + "from element_calcium_imaging import imaging_report\n", + "import suite2p" ] }, { @@ -50,28 +51,34 @@ " subject='subject1',\n", " subject_birth_date='2023-01-01',\n", " sex='U',\n", - " ),\n", - " skip_duplicates=True\n", + " )\n", ")" ] }, + { + "cell_type": "code", + "execution_count": null, + "metadata": {}, + "outputs": [], + "source": [ + "Equipment.insert1(dict(scanner=\"Mesoscope\"))" + ] + }, { "cell_type": "code", "execution_count": 3, "metadata": {}, "outputs": [], "source": [ - "session.Session.insert1(\n", - " dict(\n", - " subject='subject1',\n", - " session_datetime=datetime.datetime.now()\n", - " )\n", - ")\n", - "session_key = (session.Session & \"subject = 'subject1'\").fetch(\"KEY\")[-1]\n", + "session_key = dict(subject='subject1', \n", + " session_datetime=datetime.datetime.now())\n", + "\n", + "session.Session.insert1(session_key)\n", + "\n", "session.SessionDirectory.insert1(\n", " dict(\n", " session_key, \n", - " session_dir='workspaces/workflow-calcium-imaging/example_data/subject1/session1'\n", + " session_dir='subject1/session1'\n", " )\n", ")" ] @@ -86,86 +93,37 @@ " dict(\n", " session_key,\n", " scan_id=0,\n", + " scanner=\"Mesoscope\",\n", " acq_software='ScanImage',\n", - " ),\n", - " skip_duplicates=True\n", + " )\n", ")" ] }, { "cell_type": "code", - "execution_count": 5, + "execution_count": null, "metadata": {}, "outputs": [], "source": [ - "# Define and insert suite2p parameters\n", + "populate_settings = {\"display_progress\": True}\n", "\n", - "suite2p_params = {\n", - " \"look_one_level_down\": 0.0,\n", - " \"fast_disk\": [],\n", - " \"delete_bin\": False,\n", - " \"mesoscan\": False,\n", - " \"h5py\": [],\n", - " \"h5py_key\": \"data\",\n", - " \"save_path0\": [],\n", - " \"subfolders\": [],\n", - " \"functional_chan\": 1,\n", - " \"tau\": 1.0,\n", - " \"force_sktiff\": True,\n", - " \"preclassify\": 0.0,\n", - " \"save_mat\": False,\n", - " \"combined\": True,\n", - " \"aspect\": 1.0,\n", - " \"do_bidiphase\": False,\n", - " \"bidiphase\": 0.0,\n", - " \"do_registration\": True,\n", - " \"keep_movie_raw\": False,\n", - " \"nimg_init\": 300,\n", - " \"batch_size\": 500,\n", - " \"maxregshift\": 0.1,\n", - " \"align_by_chan\": 1,\n", - " \"reg_tif\": False,\n", - " \"reg_tif_chan2\": False,\n", - " \"subpixel\": 10,\n", - " \"smooth_sigma\": 1.15,\n", - " \"th_badframes\": 1.0,\n", - " \"pad_fft\": False,\n", - " \"nonrigid\": False,\n", - " \"block_size\": [128, 128],\n", - " \"snr_thresh\": 1.2,\n", - " \"maxregshiftNR\": 5.0,\n", - " \"1Preg\": False,\n", - " \"spatial_hp\": 50.0,\n", - " \"pre_smooth\": 2.0,\n", - " \"spatial_taper\": 50.0,\n", - " \"roidetect\": True,\n", - " \"sparse_mode\": False,\n", - " \"diameter\": 12,\n", - " \"spatial_scale\": 0,\n", - " \"connected\": True,\n", - " \"nbinned\": 5000,\n", - " \"max_iterations\": 20,\n", - " \"threshold_scaling\": 1.0,\n", - " \"max_overlap\": 0.75,\n", - " \"high_pass\": 100.0,\n", - " \"inner_neuropil_radius\": 2,\n", - " \"min_neuropil_pixels\": 350,\n", - " \"allow_overlap\": False,\n", - " \"chan2_thres\": 0.65,\n", - " \"baseline\": \"maximin\",\n", - " \"win_baseline\": 60.0,\n", - " \"sig_baseline\": 10.0,\n", - " \"prctile_baseline\": 8.0,\n", - " \"neucoeff\": 0.7,\n", - " \"xrange\": np.array([0, 0]),\n", - " \"yrange\": np.array([0, 0]),\n", - "}\n", + "scan.ScanInfo.populate(**populate_settings)" + ] + }, + { + "cell_type": "code", + "execution_count": 5, + "metadata": {}, + "outputs": [], + "source": [ + "suite2p_params = suite2p.default_ops()\n", + "suite2p_params['nonrigid']=False\n", "\n", "imaging.ProcessingParamSet.insert_new_params(\n", " processing_method=\"suite2p\",\n", - " paramset_desc='Default parameter set for suite2p',\n", - " params=suite2p_params,\n", " paramset_idx=0,\n", + " params=suite2p_params,\n", + " paramset_desc='Default parameter set for suite2p'\n", ")" ] }, @@ -175,16 +133,17 @@ "metadata": {}, "outputs": [], "source": [ - "scan_key = (scan.Scan & \"subject = 'subject1'\").fetch(\"KEY\")[-1]\n", "imaging.ProcessingTask.insert1(\n", " dict(\n", - " scan_key,\n", + " session_key,\n", + " scan_id=0,\n", " paramset_idx=0,\n", - " processing_output_dir='workspaces/workflow-calcium-imaging/example_data/subject1/session1',\n", - " task_mode='trigger',\n", - " ),\n", - " skip_duplicates=True\n", - ")" + " task_mode='load', # load or trigger\n", + " processing_output_dir='subject1/session1/suite2p',\n", + " )\n", + ")\n", + "\n", + "imaging.Processing.populate(**populate_settings)" ] }, { @@ -251,48 +210,25 @@ } ], "source": [ - "populate_settings = {\"display_progress\": True}\n", - "scan.ScanInfo.populate(**populate_settings)\n", - "imaging.Processing.populate(**populate_settings)\n", - "\n", - "# Optional Curation Step\n", - "\n", - "processing_key = imaging.Processing.fetch(\"KEY\")[-1]\n", "imaging.Curation.insert1(\n", " dict(\n", - " processing_key,\n", - " curation_id=0,\n", - " curation_time=datetime.datetime.now(),\n", - " curation_output_dir='workspaces/workflow-calcium-imaging/example_data',\n", - " manual_curation=False,\n", + " session_key,\n", + " scan_id=0,\n", + " paramset_idx=0,\n", + " curation_id=0,\n", + " curation_time=datetime.datetime.now(),\n", + " curation_output_dir='subject1/session1/suite2p',\n", + " manual_curation=False,\n", " )\n", ")\n", "\n", "imaging.MotionCorrection.populate(**populate_settings)\n", "imaging.Segmentation.populate(**populate_settings)\n", - "imaging.MaskClassification.populate(**populate_settings)\n", "imaging.Fluorescence.populate(**populate_settings)\n", - "imaging.Activity.populate(**populate_settings)" - ] - }, - { - "cell_type": "code", - "execution_count": 12, - "metadata": {}, - "outputs": [], - "source": [ - "from element_calcium_imaging import imaging_report\n", - "\n", + "imaging.Activity.populate(**populate_settings)\n", "imaging_report.ScanLevelReport.populate()\n", "imaging_report.TraceReport.populate()" ] - }, - { - "cell_type": "code", - "execution_count": null, - "metadata": {}, - "outputs": [], - "source": [] } ], "metadata": { From db631141303cd7bb07e301fbca20091226ed8669 Mon Sep 17 00:00:00 2001 From: Kabilar Gunalan Date: Wed, 8 Mar 2023 13:27:41 -0600 Subject: [PATCH 02/11] Rename notebooks --- notebooks/{prepare_demo.ipynb => demo_prepare.ipynb} | 0 notebooks/{quick_demo.ipynb => demo_run.ipynb} | 0 2 files changed, 0 insertions(+), 0 deletions(-) rename notebooks/{prepare_demo.ipynb => demo_prepare.ipynb} (100%) rename notebooks/{quick_demo.ipynb => demo_run.ipynb} (100%) diff --git a/notebooks/prepare_demo.ipynb b/notebooks/demo_prepare.ipynb similarity index 100% rename from notebooks/prepare_demo.ipynb rename to notebooks/demo_prepare.ipynb diff --git a/notebooks/quick_demo.ipynb b/notebooks/demo_run.ipynb similarity index 100% rename from notebooks/quick_demo.ipynb rename to notebooks/demo_run.ipynb From 0720614d938e0d71ec7b00961206a0e894d4dbf0 Mon Sep 17 00:00:00 2001 From: Kabilar Gunalan Date: Wed, 8 Mar 2023 13:28:16 -0600 Subject: [PATCH 03/11] Add tutorial notebook placeholder --- notebooks/tutorial.ipynb | 32 ++++++++++++++++++++++++++++++++ 1 file changed, 32 insertions(+) create mode 100644 notebooks/tutorial.ipynb diff --git a/notebooks/tutorial.ipynb b/notebooks/tutorial.ipynb new file mode 100644 index 0000000..de29eff --- /dev/null +++ b/notebooks/tutorial.ipynb @@ -0,0 +1,32 @@ +{ + "cells": [ + { + "cell_type": "code", + "execution_count": null, + "metadata": {}, + "outputs": [], + "source": [ + "Coming soon!" + ] + } + ], + "metadata": { + "kernelspec": { + "display_name": "python3p10", + "language": "python", + "name": "python3" + }, + "language_info": { + "name": "python", + "version": "3.10.4 (main, Mar 31 2022, 03:38:35) [Clang 12.0.0 ]" + }, + "orig_nbformat": 4, + "vscode": { + "interpreter": { + "hash": "ff52d424e56dd643d8b2ec122f40a2e279e94970100b4e6430cb9025a65ba4cf" + } + } + }, + "nbformat": 4, + "nbformat_minor": 2 +} From da31c4facc8f554abe6c980582f635248eeeae88 Mon Sep 17 00:00:00 2001 From: Kabilar Gunalan Date: Wed, 8 Mar 2023 13:29:40 -0600 Subject: [PATCH 04/11] Update `demo_prepare` --- notebooks/demo_prepare.ipynb | 87 +++++------------------------------- 1 file changed, 10 insertions(+), 77 deletions(-) diff --git a/notebooks/demo_prepare.ipynb b/notebooks/demo_prepare.ipynb index 8831e7b..93208dc 100644 --- a/notebooks/demo_prepare.ipynb +++ b/notebooks/demo_prepare.ipynb @@ -9,23 +9,14 @@ "\n", "**Please Note**: This notebook and demo are NOT intended to be used as learning materials. To gain\n", "a thorough understanding of the DataJoint workflow for calcium imaging, please\n", - "see our [`tutorial`](./tutorial.ipynb) notebook." + "see the [`tutorial`](./tutorial.ipynb) notebook." ] }, { "cell_type": "code", - "execution_count": 1, + "execution_count": null, "metadata": {}, - "outputs": [ - { - "name": "stderr", - "output_type": "stream", - "text": [ - "[2023-03-07 17:53:28,952][INFO]: Connecting root@fakeservices.datajoint.io:3306\n", - "[2023-03-07 17:53:28,959][INFO]: Connected root@fakeservices.datajoint.io:3306\n" - ] - } - ], + "outputs": [], "source": [ "import os\n", "\n", @@ -42,7 +33,7 @@ }, { "cell_type": "code", - "execution_count": 2, + "execution_count": null, "metadata": {}, "outputs": [], "source": [ @@ -66,7 +57,7 @@ }, { "cell_type": "code", - "execution_count": 3, + "execution_count": null, "metadata": {}, "outputs": [], "source": [ @@ -85,7 +76,7 @@ }, { "cell_type": "code", - "execution_count": 4, + "execution_count": null, "metadata": {}, "outputs": [], "source": [ @@ -112,7 +103,7 @@ }, { "cell_type": "code", - "execution_count": 5, + "execution_count": null, "metadata": {}, "outputs": [], "source": [ @@ -129,7 +120,7 @@ }, { "cell_type": "code", - "execution_count": 6, + "execution_count": null, "metadata": {}, "outputs": [], "source": [ @@ -148,67 +139,9 @@ }, { "cell_type": "code", - "execution_count": 7, + "execution_count": null, "metadata": {}, - "outputs": [ - { - "name": "stderr", - "output_type": "stream", - "text": [ - "ScanInfo: 100%|██████████| 1/1 [00:00<00:00, 2.75it/s]\n", - "Processing: 0%| | 0/1 [00:00>>>>>>>>>>>>>>>>>>>> PLANE 0 <<<<<<<<<<<<<<<<<<<<<<\n", - "NOTE: not running registration, plane already registered\n", - "binary path: /workspaces/workflow-calcium-imaging/example_data/subject1/session1/suite2p/plane0/data.bin\n", - "NOTE: Applying builtin classifier at /usr/local/lib/python3.7/site-packages/suite2p/classifiers/classifier.npy\n", - "----------- ROI DETECTION\n", - "Binning movie in chunks of length 29\n", - "Binned movie of size [103,504,506] created in 2.29 sec.\n", - "ROIs: 200, cost: 0.2719, time: 37.3356\n", - "ROIs: 400, cost: 0.2142, time: 63.2455\n", - "ROIs: 547, cost: 0.1907, time: 84.7919\n", - "ROIs: 583, cost: 0.1853, time: 98.0249\n", - "ROIs: 599, cost: 0.1835, time: 109.7812\n", - "ROIs: 599, cost: 0.4262, time: 114.3391\n", - "ROIs: 599, cost: 0.3851, time: 126.5808\n", - "ROIs: 599, cost: 0.3825, time: 140.0235\n", - "Detected 599 ROIs, 161.39 sec\n", - "After removing overlaps, 563 ROIs remain\n", - "----------- Total 165.00 sec.\n", - "----------- EXTRACTION\n", - "Masks created, 2.40 sec.\n", - "Extracted fluorescence from 563 ROIs in 3000 frames, 7.91 sec.\n", - "----------- Total 10.40 sec.\n", - "----------- CLASSIFICATION\n", - "['npix_norm', 'compact', 'skew']\n", - "----------- SPIKE DECONVOLUTION\n", - "----------- Total 0.08 sec.\n", - "Plane 0 processed in 175.58 sec (can open in GUI).\n", - "total = 175.77 sec.\n", - "TOTAL RUNTIME 175.77 sec\n" - ] - }, - { - "name": "stderr", - "output_type": "stream", - "text": [ - "Processing: 100%|██████████| 1/1 [02:58<00:00, 178.28s/it]\n", - "MotionCorrection: 100%|██████████| 1/1 [00:00<00:00, 1.53it/s]\n", - "Segmentation: 100%|██████████| 1/1 [00:00<00:00, 1.76it/s]\n", - "MaskClassification: 100%|██████████| 1/1 [00:00<00:00, 520.32it/s]\n", - "Fluorescence: 100%|██████████| 1/1 [00:01<00:00, 1.29s/it]\n", - "Activity: 100%|██████████| 1/1 [00:00<00:00, 3.69it/s]\n" - ] - } - ], + "outputs": [], "source": [ "imaging.Curation.insert1(\n", " dict(\n", From b19880a2ef645d3ad9dac515de6e809bad4d3ca7 Mon Sep 17 00:00:00 2001 From: Kabilar Gunalan Date: Wed, 8 Mar 2023 13:39:20 -0600 Subject: [PATCH 05/11] Add DataJoint environment variables --- .devcontainer/Dockerfile | 4 +++- workflow_calcium_imaging/__init__.py | 12 +++++++++++- 2 files changed, 14 insertions(+), 2 deletions(-) diff --git a/.devcontainer/Dockerfile b/.devcontainer/Dockerfile index c2809ad..38df635 100644 --- a/.devcontainer/Dockerfile +++ b/.devcontainer/Dockerfile @@ -24,4 +24,6 @@ ENV DJ_HOST fakeservices.datajoint.io ENV DJ_USER root ENV DJ_PASS simple -ENV DJ_PUBLIC_S3_MOUNT_PATH /workspaces/workflow-calcium-imaging/example_data \ No newline at end of file +ENV DJ_PUBLIC_S3_MOUNT_PATH /workspaces/workflow-calcium-imaging/example_data +ENV IMAGING_ROOT_DATA_DIR /workspaces/workflow-calcium-imaging/example_data +ENV DATABASE_PREFIX neuro_ \ No newline at end of file diff --git a/workflow_calcium_imaging/__init__.py b/workflow_calcium_imaging/__init__.py index 111bd2f..46925dd 100644 --- a/workflow_calcium_imaging/__init__.py +++ b/workflow_calcium_imaging/__init__.py @@ -3,4 +3,14 @@ if "custom" not in dj.config: dj.config["custom"] = {} -db_prefix = dj.config["custom"].get("database.prefix", "") +# overwrite dj.config['custom'] values with environment variables if available + +dj.config["custom"]["database.prefix"] = os.getenv( + "DATABASE_PREFIX", dj.config["custom"].get("database.prefix", "") +) + +dj.config["custom"]["imaging_root_data_dir"] = os.getenv( + "IMAGING_ROOT_DATA_DIR", dj.config["custom"].get("imaging_root_data_dir", "") +) + +db_prefix = dj.config["custom"].get("database.prefix", "") \ No newline at end of file From 2ba00913e15b96707a879b4c8fb52f6fe6e9e580 Mon Sep 17 00:00:00 2001 From: Kabilar Gunalan Date: Wed, 8 Mar 2023 13:39:45 -0600 Subject: [PATCH 06/11] Add import --- notebooks/demo_prepare.ipynb | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/notebooks/demo_prepare.ipynb b/notebooks/demo_prepare.ipynb index 93208dc..2b425a1 100644 --- a/notebooks/demo_prepare.ipynb +++ b/notebooks/demo_prepare.ipynb @@ -26,7 +26,7 @@ "import datajoint as dj\n", "import datetime\n", "import numpy as np\n", - "from workflow_calcium_imaging.pipeline import subject, session, scan, imaging\n", + "from workflow_calcium_imaging.pipeline import subject, session, scan, imaging, Equipment\n", "from element_calcium_imaging import imaging_report\n", "import suite2p" ] From 924a77c158d3e92b29d6c7e28230c8007a14bfce Mon Sep 17 00:00:00 2001 From: Kabilar Gunalan Date: Wed, 8 Mar 2023 13:56:15 -0600 Subject: [PATCH 07/11] Update imports --- notebooks/demo_prepare.ipynb | 5 ----- 1 file changed, 5 deletions(-) diff --git a/notebooks/demo_prepare.ipynb b/notebooks/demo_prepare.ipynb index 2b425a1..3811c66 100644 --- a/notebooks/demo_prepare.ipynb +++ b/notebooks/demo_prepare.ipynb @@ -18,11 +18,6 @@ "metadata": {}, "outputs": [], "source": [ - "import os\n", - "\n", - "if os.path.basename(os.getcwd()) == \"notebooks\":\n", - " os.chdir(\"..\")\n", - "\n", "import datajoint as dj\n", "import datetime\n", "import numpy as np\n", From cbc269f2a10b3de1f988e2bfce3c2f7a311c0689 Mon Sep 17 00:00:00 2001 From: Kabilar Gunalan Date: Wed, 8 Mar 2023 21:54:56 -0600 Subject: [PATCH 08/11] Update demo notebook --- notebooks/demo_run.ipynb | 120 +++++++++++++-------------------------- 1 file changed, 38 insertions(+), 82 deletions(-) diff --git a/notebooks/demo_run.ipynb b/notebooks/demo_run.ipynb index db119e5..5c7e1c6 100644 --- a/notebooks/demo_run.ipynb +++ b/notebooks/demo_run.ipynb @@ -5,10 +5,11 @@ "cell_type": "markdown", "metadata": {}, "source": [ - "# DataJoint Workflow Calcium Imaging Demo\n", + "# DataJoint Workflow for Calcium Imaging\n", "\n", - "This notebook demonstrates using DataJoint's open-source Elements packages to build a workflow for\n", - "calcium imaging. For a detailed tutorial, please see our tutorial notebooks." + "+ This notebook demonstrates using the open-source DataJoint Element to build a workflow for\n", + "calcium imaging.\n", + "+ For a detailed tutorial, please see the [tutorial notebook](./tutorial.ipynb)." ] }, { @@ -16,13 +17,12 @@ "cell_type": "markdown", "metadata": {}, "source": [ - "![Raw Scans](../images/rawscans.gif) ![Motion Corrected\n", - "Scans](../images/motioncorrectedscans.gif) \n", + "\n", + "\n", + "\n", + " \n", "\n", - "![Cell Segmentation](../images/cellsegmentation.png) ![Calcium\n", - "Events](../images/calciumtraces.png) \n", - "\n", - "Left to right: Raw scans, Motion corrected scans, Cell segmentation, Calcium events" + "Left to right: Raw scans, Motion corrected scans, Cell segmentations, Calcium events" ] }, { @@ -30,13 +30,12 @@ "cell_type": "markdown", "metadata": {}, "source": [ - "We'll begin by importing the dependencies, then viewing all the tables in the workflow.\n", - "After this, we will insert, populate, and analyze calcium imaging data" + "### Import dependencies" ] }, { "cell_type": "code", - "execution_count": 3, + "execution_count": null, "metadata": { "tags": [ "hide-input" @@ -44,44 +43,26 @@ }, "outputs": [], "source": [ - "# Import Statements\n", - "\n", - "import os\n", - "\n", - "if os.path.basename(os.getcwd()) == \"notebooks\":\n", - " os.chdir(\"..\")\n", - "\n", "import datajoint as dj\n", - "import datetime\n", - "import matplotlib.pyplot as plt\n", - "import numpy as np\n", "from workflow_calcium_imaging.pipeline import subject, session, scan, imaging\n", "from element_calcium_imaging.plotting.widget import main" ] }, + { + "attachments": {}, + "cell_type": "markdown", + "metadata": {}, + "source": [ + "### View workflow" + ] + }, { "cell_type": "code", - "execution_count": 4, + "execution_count": null, "metadata": {}, - "outputs": [ - { - "data": { - "image/svg+xml": "\n\n%3\n\n\n\n0\n\n0\n\n\n\nimaging.Fluorescence.Trace\n\n\nimaging.Fluorescence.Trace\n\n\n\n\n\n0->imaging.Fluorescence.Trace\n\n\n\n\n1\n\n1\n\n\n\nimaging.Segmentation.Mask\n\n\nimaging.Segmentation.Mask\n\n\n\n\n\n1->imaging.Segmentation.Mask\n\n\n\n\n2\n\n2\n\n\n\nimaging.MotionCorrection\n\n\nimaging.MotionCorrection\n\n\n\n\n\n2->imaging.MotionCorrection\n\n\n\n\nsubject.Subject\n\n\nsubject.Subject\n\n\n\n\n\nsession.Session\n\n\nsession.Session\n\n\n\n\n\nsubject.Subject->session.Session\n\n\n\n\nscan.Scan\n\n\nscan.Scan\n\n\n\n\n\nsession.Session->scan.Scan\n\n\n\n\nscan.ScanLocation\n\n\nscan.ScanLocation\n\n\n\n\n\nscan.Scan->scan.ScanLocation\n\n\n\n\nscan.ScanInfo\n\n\nscan.ScanInfo\n\n\n\n\n\nscan.Scan->scan.ScanInfo\n\n\n\n\nimaging.ProcessingTask\n\n\nimaging.ProcessingTask\n\n\n\n\n\nscan.Scan->imaging.ProcessingTask\n\n\n\n\nscan.ScanInfo.ScanFile\n\n\nscan.ScanInfo.ScanFile\n\n\n\n\n\nscan.ScanInfo.Field\n\n\nscan.ScanInfo.Field\n\n\n\n\n\nimaging.MotionCorrection.Summary\n\n\nimaging.MotionCorrection.Summary\n\n\n\n\n\nscan.ScanInfo.Field->imaging.MotionCorrection.Summary\n\n\n\n\nscan.ScanInfo->scan.ScanInfo.ScanFile\n\n\n\n\nscan.ScanInfo->scan.ScanInfo.Field\n\n\n\n\nscan.Channel\n\n\nscan.Channel\n\n\n\n\n\nscan.Channel->0\n\n\n\n\nscan.Channel->1\n\n\n\n\nscan.Channel->2\n\n\n\n\nscan.AcquisitionSoftware\n\n\nscan.AcquisitionSoftware\n\n\n\n\n\nscan.AcquisitionSoftware->scan.Scan\n\n\n\n\nimaging.Processing\n\n\nimaging.Processing\n\n\n\n\n\nimaging.ProcessingTask->imaging.Processing\n\n\n\n\nimaging.Curation\n\n\nimaging.Curation\n\n\n\n\n\nimaging.Curation->imaging.MotionCorrection\n\n\n\n\nimaging.Segmentation\n\n\nimaging.Segmentation\n\n\n\n\n\nimaging.Curation->imaging.Segmentation\n\n\n\n\nimaging.MotionCorrection.RigidMotionCorrection\n\n\nimaging.MotionCorrection.RigidMotionCorrection\n\n\n\n\n\nimaging.MotionCorrection.NonRigidMotionCorrection\n\n\nimaging.MotionCorrection.NonRigidMotionCorrection\n\n\n\n\n\nimaging.MotionCorrection.Block\n\n\nimaging.MotionCorrection.Block\n\n\n\n\n\nimaging.MotionCorrection.NonRigidMotionCorrection->imaging.MotionCorrection.Block\n\n\n\n\nimaging.MotionCorrection->imaging.MotionCorrection.Summary\n\n\n\n\nimaging.MotionCorrection->imaging.MotionCorrection.RigidMotionCorrection\n\n\n\n\nimaging.MotionCorrection->imaging.MotionCorrection.NonRigidMotionCorrection\n\n\n\n\nimaging.MaskClassification.MaskType\n\n\nimaging.MaskClassification.MaskType\n\n\n\n\n\nimaging.Segmentation.Mask->imaging.MaskClassification.MaskType\n\n\n\n\nimaging.Segmentation.Mask->imaging.Fluorescence.Trace\n\n\n\n\nimaging.Segmentation->imaging.Segmentation.Mask\n\n\n\n\nimaging.MaskClassification\n\n\nimaging.MaskClassification\n\n\n\n\n\nimaging.Segmentation->imaging.MaskClassification\n\n\n\n\nimaging.Fluorescence\n\n\nimaging.Fluorescence\n\n\n\n\n\nimaging.Segmentation->imaging.Fluorescence\n\n\n\n\nimaging.Processing->imaging.Curation\n\n\n\n\nimaging.MaskClassification->imaging.MaskClassification.MaskType\n\n\n\n\nimaging.Activity.Trace\n\n\nimaging.Activity.Trace\n\n\n\n\n\nimaging.Fluorescence.Trace->imaging.Activity.Trace\n\n\n\n\nimaging.Fluorescence->imaging.Fluorescence.Trace\n\n\n\n\nimaging.Activity\n\n\nimaging.Activity\n\n\n\n\n\nimaging.Fluorescence->imaging.Activity\n\n\n\n\nimaging.Activity->imaging.Activity.Trace\n\n\n\n\nimaging.ProcessingParamSet\n\n\nimaging.ProcessingParamSet\n\n\n\n\n\nimaging.ProcessingParamSet->imaging.ProcessingTask\n\n\n\n\nimaging.ProcessingMethod\n\n\nimaging.ProcessingMethod\n\n\n\n\n\nimaging.ProcessingMethod->imaging.ProcessingParamSet\n\n\n\n\nimaging.MaskType\n\n\nimaging.MaskType\n\n\n\n\n\nimaging.MaskType->imaging.MaskClassification.MaskType\n\n\n\n\nimaging.MaskClassificationMethod\n\n\nimaging.MaskClassificationMethod\n\n\n\n\n\nimaging.MaskClassificationMethod->imaging.MaskClassification\n\n\n\n\nimaging.CellCompartment\n\n\nimaging.CellCompartment\n\n\n\n\n\nimaging.ActivityExtractionMethod\n\n\nimaging.ActivityExtractionMethod\n\n\n\n\n\nimaging.ActivityExtractionMethod->imaging.Activity\n\n\n\n", - "text/plain": [ - "" - ] - }, - "execution_count": 4, - "metadata": {}, - "output_type": "execute_result" - } - ], + "outputs": [], "source": [ - "# Run this cell to view a diagram of the tables that we will use in this demo\n", - "\n", - "(\n", - " dj.Diagram(subject.Subject) + dj.Diagram(session.Session) + dj.Diagram(scan) + dj.Diagram(imaging)\n", - ")" + "dj.Diagram(subject.Subject) + dj.Diagram(session.Session) + dj.Diagram(scan) + dj.Diagram(imaging)" ] }, { @@ -89,17 +70,14 @@ "cell_type": "markdown", "metadata": {}, "source": [ - "### Inserting entries on manual tables calls on the `insert()` function within DataJoint\n", + "### Insert an entry in a manual table by calling the `insert()` method\n", "\n", - "For example, to insert data about a subject: \n", - "```\n", + "```python\n", "subject.Subject.insert1(\n", - " dict(\n", - " subject='subject1',\n", - " subject_birth_date='2023-01-01',\n", - " sex='U',\n", - " ),\n", - " skip_duplicates=True\n", + " dict(subject='subject1',\n", + " subject_birth_date='2023-01-01',\n", + " sex='U'\n", + " )\n", ")\n", "```" ] @@ -109,16 +87,15 @@ "cell_type": "markdown", "metadata": {}, "source": [ - "### Automatically process sessions with `Populate()`\n", + "### Automatically process data with the `populate()` method\n", "\n", + "+ Once data is inserted into manual tables, the `populate()` function automatically runs the ingestion and processing routines. \n", "\n", - "Once data is inserted into manual tables, the `.populate()` function in DataJoint\n", - "triggers automatic processing and ingestion routines. \n", + "+ For example, to run Suite2p processing in the `Processing` table:\n", "\n", - "For example, to trigger suite2p processing via the `Processing` table:\n", - "```\n", - "imaging.Processing.populate()\n", - "```" + " ```python\n", + " imaging.Processing.populate()\n", + " ```" ] }, { @@ -126,30 +103,14 @@ "cell_type": "markdown", "metadata": {}, "source": [ - "### Visualize Ingested Data" + "### Visualize processed data" ] }, { "cell_type": "code", - "execution_count": 2, + "execution_count": null, "metadata": {}, - "outputs": [ - { - "data": { - "application/vnd.jupyter.widget-view+json": { - "model_id": "c6d32fd732dc416bb5c7045812e00020", - "version_major": 2, - "version_minor": 0 - }, - "text/plain": [ - "VBox(children=(HBox(children=(Dropdown(description='Result:', layout=Layout(display='flex', flex_flow='row', g…" - ] - }, - "execution_count": 2, - "metadata": {}, - "output_type": "execute_result" - } - ], + "outputs": [], "source": [ "main(imaging)" ] @@ -159,13 +120,8 @@ "cell_type": "markdown", "metadata": {}, "source": [ - "For a much more in-depth tutorial, be sure to check out our `full_demo.ipynb`." + "For an in-depth tutorial please see the [tutorial notebook](./tutorial.ipynb)." ] - }, - { - "cell_type": "markdown", - "metadata": {}, - "source": [] } ], "metadata": { From d03f5e748e9017bdd4ae7bbe7c6c3aaa1add562c Mon Sep 17 00:00:00 2001 From: Kabilar Gunalan Date: Wed, 8 Mar 2023 22:07:38 -0600 Subject: [PATCH 09/11] Add import --- workflow_calcium_imaging/__init__.py | 1 + 1 file changed, 1 insertion(+) diff --git a/workflow_calcium_imaging/__init__.py b/workflow_calcium_imaging/__init__.py index 46925dd..bd08cdc 100644 --- a/workflow_calcium_imaging/__init__.py +++ b/workflow_calcium_imaging/__init__.py @@ -1,3 +1,4 @@ +import os import datajoint as dj if "custom" not in dj.config: From 051e734b1932275de861f96e2ac2d387bef4272c Mon Sep 17 00:00:00 2001 From: Kabilar Gunalan Date: Wed, 8 Mar 2023 22:24:58 -0600 Subject: [PATCH 10/11] Update notebook --- notebooks/demo_prepare.ipynb | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/notebooks/demo_prepare.ipynb b/notebooks/demo_prepare.ipynb index 3811c66..25e1b07 100644 --- a/notebooks/demo_prepare.ipynb +++ b/notebooks/demo_prepare.ipynb @@ -154,8 +154,8 @@ "imaging.Segmentation.populate(**populate_settings)\n", "imaging.Fluorescence.populate(**populate_settings)\n", "imaging.Activity.populate(**populate_settings)\n", - "imaging_report.ScanLevelReport.populate()\n", - "imaging_report.TraceReport.populate()" + "imaging_report.ScanLevelReport.populate(**populate_settings)\n", + "imaging_report.TraceReport.populate(**populate_settings)" ] } ], From 3d0b582e80d4acd1e9e80f6acda7877ad9341b0d Mon Sep 17 00:00:00 2001 From: Kabilar Gunalan Date: Wed, 8 Mar 2023 22:30:33 -0600 Subject: [PATCH 11/11] Update requirements --- requirements.txt | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/requirements.txt b/requirements.txt index 08267c3..a577616 100644 --- a/requirements.txt +++ b/requirements.txt @@ -1,6 +1,6 @@ datajoint>=0.13.0 element-animal>=0.1.2 -element-calcium-imaging>=0.3.0 +element-calcium-imaging>=0.5.4 element-event>=0.1.2 element-interface>=0.5.0 element-lab>=0.1.1