Merge pull request #53 from kushalbakshi/codespace

Add Codespaces + update tutorial notebook

.devcontainer/Dockerfile

@@ -0,0 +1,50 @@
+FROM python:3.9-slim@sha256:5f0192a4f58a6ce99f732fe05e3b3d00f12ae62e183886bca3ebe3d202686c7f
+
+ENV PATH /usr/local/bin:$PATH
+ENV PYTHON_VERSION 3.9.17
+
+RUN \
+    adduser --system --disabled-password --shell /bin/bash vscode && \
+    # install docker
+    apt-get update && \
+    apt-get install ca-certificates curl gnupg lsb-release -y && \
+    mkdir -m 0755 -p /etc/apt/keyrings && \
+    curl -fsSL https://download.docker.com/linux/debian/gpg | gpg --dearmor -o /etc/apt/keyrings/docker.gpg && \
+    echo "deb [arch=$(dpkg --print-architecture) signed-by=/etc/apt/keyrings/docker.gpg] https://download.docker.com/linux/debian $(lsb_release -cs) stable" | tee /etc/apt/sources.list.d/docker.list > /dev/null && \
+    apt-get update && \
+    apt-get install docker-ce docker-ce-cli containerd.io docker-buildx-plugin docker-compose-plugin -y && \
+    usermod -aG docker vscode && \
+    apt-get clean
+
+RUN \
+    # dev setup
+    apt update && \
+    apt-get install sudo git bash-completion graphviz default-mysql-client s3fs procps -y && \
+    usermod -aG sudo vscode && \
+    echo '%sudo ALL=(ALL) NOPASSWD:ALL' >> /etc/sudoers && \
+    pip install --no-cache-dir --upgrade black pip nbconvert && \
+    echo '. /etc/bash_completion' >> /home/vscode/.bashrc && \
+    echo 'export PS1="\[\e[32;1m\]\u\[\e[m\]@\[\e[34;1m\]\H\[\e[m\]:\[\e[33;1m\]\w\[\e[m\]$ "' >> /home/vscode/.bashrc && \
+    apt-get clean
+
+COPY ./ /tmp/element-miniscope/
+
+RUN \
+    # pipeline dependencies
+    apt-get install gcc g++ ffmpeg libsm6 libxext6 -y && \
+    pip install numpy Cython && \
+    pip install --no-cache-dir -e /tmp/element-miniscope[elements,caiman_requirements,caiman] && \
+    caimanmanager.py install && \
+    # clean up
+    rm -rf /tmp/element-miniscope && \
+    apt-get clean
+
+ENV DJ_HOST fakeservices.datajoint.io
+ENV DJ_USER root
+ENV DJ_PASS simple
+
+ENV MINISCOPE_ROOT_DATA_DIR /workspaces/element-miniscope/example_data
+ENV DATABASE_PREFIX neuro_
+
+USER vscode
+CMD bash -c "sudo rm /var/run/docker.pid; sudo dockerd"

.devcontainer/devcontainer.json

@@ -0,0 +1,30 @@
+{
+	"name": "Environment + Data",
+	"dockerComposeFile": "docker-compose.yaml",
+	"service": "app",
+	"workspaceFolder": "/workspaces/${localWorkspaceFolderBasename}",
+	"remoteEnv": {
+		"LOCAL_WORKSPACE_FOLDER": "${localWorkspaceFolder}"
+	},
+	"onCreateCommand": "mkdir -p ${MINISCOPE_ROOT_DATA_DIR} && pip install -e .",
+	"postStartCommand": "docker volume prune -f && s3fs ${DJ_PUBLIC_S3_LOCATION} ${MINISCOPE_ROOT_DATA_DIR} -o nonempty,multipart_size=530,endpoint=us-east-1,url=http://s3.amazonaws.com,public_bucket=1",
+	"hostRequirements": {
+		"cpus": 4,
+		"memory": "8gb",
+		"storage": "32gb"
+	},
+	"forwardPorts": [
+		3306
+	],
+	"customizations": {
+		"settings": {
+			"python.pythonPath": "/usr/local/bin/python"
+		},
+		"vscode": {
+			"extensions": [
+				"[email protected]",
+				"[email protected]"
+			]
+		}
+	}
+}

.devcontainer/docker-compose.yaml

@@ -0,0 +1,25 @@
+version: "3"
+services:
+  app:
+    cpus: 4
+    mem_limit: 8g
+    build:
+      context: ..
+      dockerfile: ./.devcontainer/Dockerfile
+    # image: datajoint/element_array_ephys:latest
+    extra_hosts:
+      - fakeservices.datajoint.io:127.0.0.1
+    environment:
+      - DJ_PUBLIC_S3_LOCATION=djhub.vathes.datapub.elements:/workflow-miniscope/v1
+    devices:
+      - /dev/fuse
+    cap_add:
+      - SYS_ADMIN
+    security_opt:
+      - apparmor:unconfined
+    volumes:
+      - ..:/workspaces/element-miniscope:cached
+      - docker_data:/var/lib/docker # persist docker images
+    privileged: true # only because of dind
+volumes:
+  docker_data:

.github/workflows/release.yaml

@@ -0,0 +1,27 @@
+name: Release
+on:
+  workflow_dispatch:
+jobs:
+  make_github_release:
+    uses: datajoint/.github/.github/workflows/make_github_release.yaml@main
+  pypi_release:
+    needs: make_github_release
+    uses: datajoint/.github/.github/workflows/pypi_release.yaml@main
+    secrets:
+      TWINE_USERNAME: ${{secrets.TWINE_USERNAME}}
+      TWINE_PASSWORD: ${{secrets.TWINE_PASSWORD}}
+    with:
+      UPLOAD_URL: ${{needs.make_github_release.outputs.release_upload_url}}
+  mkdocs_release:
+    uses: datajoint/.github/.github/workflows/mkdocs_release.yaml@main
+    permissions: 
+      contents: write
+  devcontainer-build:
+    uses: datajoint/.github/.github/workflows/devcontainer-build.yaml@main
+  devcontainer-publish:
+    needs: 
+      - devcontainer-build
+    uses: datajoint/.github/.github/workflows/devcontainer-publish.yaml@main
+    secrets:
+      DOCKERHUB_USERNAME: ${{secrets.DOCKERHUB_USERNAME}}
+      DOCKERHUB_TOKEN: ${{secrets.DOCKERHUB_TOKEN_FOR_ELEMENTS}}

.github/workflows/test.yaml

@@ -0,0 +1,36 @@
+name: Test
+on:
+  push:
+  pull_request:
+  workflow_dispatch:
+  schedule:
+    - cron: "0 8 * * 1"
+jobs:
+  devcontainer-build:
+    uses: datajoint/.github/.github/workflows/devcontainer-build.yaml@main
+  tests:
+    runs-on: ubuntu-latest
+    strategy:
+      matrix:
+        py_ver: ["3.9", "3.10"]
+        mysql_ver: ["8.0", "5.7"]
+        include:
+          - py_ver: "3.8"
+            mysql_ver: "5.7"
+          - py_ver: "3.7"
+            mysql_ver: "5.7"
+    steps:
+      - uses: actions/checkout@v3
+      - name: Set up Python ${{matrix.py_ver}}
+        uses: actions/setup-python@v4
+        with:
+          python-version: ${{matrix.py_ver}}
+      - name: Install dependencies
+        run: |
+          python -m pip install --upgrade pip
+          pip install flake8 "black[jupyter]"
+      - name: Run style tests
+        run: |
+          python_version=${{matrix.py_ver}}
+          black element_miniscope --check --verbose --target-version py${python_version//.}
+          black notebooks --check --verbose --target-version py${python_version//.}

.gitignore

@@ -92,7 +92,6 @@ site
 **/.#*
 docker-compose.yml
 
-.vscode*
 
 # datajoint, notes, nwb export
 dj_local_c*.json
@@ -105,3 +104,6 @@ temp/*
 /docs/src/tutorials/*ipynb
 /docs/mike-mkdocs*
 element_miniscope/qc.py
+
+# DevContainer/Codespaces
+example_data/

.pre-commit-config.yaml

@@ -25,7 +25,7 @@ repos:
 
   # isort
   - repo: https://github.com/pycqa/isort
-    rev: 5.11.2
+    rev: 5.12.0
     hooks:
       - id: isort
         args: ["--profile", "black"]

.vscode/launch.json

@@ -0,0 +1,16 @@
+{
+    // Use IntelliSense to learn about possible attributes.
+    // Hover to view descriptions of existing attributes.
+    // For more information, visit: https://go.microsoft.com/fwlink/?linkid=830387
+    "version": "0.2.0",
+    "configurations": [
+        {
+            "name": "Python: Current File",
+            "type": "python",
+            "request": "launch",
+            "program": "${file}",
+            "console": "integratedTerminal",
+            "justMyCode": false
+        }
+    ]
+}

.vscode/settings.json

@@ -0,0 +1,15 @@
+{
+    "editor.formatOnPaste": false,
+    "editor.formatOnSave": true,
+    "editor.rulers": [
+        88
+    ],
+    "python.formatting.provider": "black",
+    "[python]": {
+        "editor.defaultFormatter": null
+    },
+    "[markdown]": {
+        "editor.defaultFormatter": "disable"
+    },
+    "files.autoSave": "off"
+}

CHANGELOG.md

@@ -3,6 +3,14 @@
 Observes [Semantic Versioning](https://semver.org/spec/v2.0.0.html) standard and
 [Keep a Changelog](https://keepachangelog.com/en/1.0.0/) convention.
 
+## [0.4.0] - 2023-10-31
+
++ Add - DevContainer for codespaces
++ Add - `tutorial_pipeline.py`
++ Add - 60 min tutorial using Jupyter Notebooks
++ Update - General improvements to increase consistency with other DataJoint Elements
++ Update - `paramset_id` attribute to `paramset_idx`
+
 ## [0.3.2] - 2023-07-20
 
 + Add `opencv-python` to requirements
@@ -50,6 +58,7 @@ Observes [Semantic Versioning](https://semver.org/spec/v2.0.0.html) standard and
 + Add - Load data acquired with Miniscope-DAQ-V3
 + Add - Load data analyzed with MiniscopeAnalysis
 
+[0.4.0]: https://github.com/datajoint/element-miniscope/releases/tag/0.4.0
 [0.3.2]: https://github.com/datajoint/element-miniscope/releases/tag/0.3.2
 [0.3.1]: https://github.com/datajoint/element-miniscope/releases/tag/0.3.1
 [0.3.0]: https://github.com/datajoint/element-miniscope/releases/tag/0.3.0

CONTRIBUTING.md

@@ -1,5 +1,5 @@
 # Contribution Guidelines
 
 This project follows the
-[DataJoint Contribution Guidelines](https://datajoint.com/docs/community/contribute/).
+[DataJoint Contribution Guidelines](https://datajoint.com/docs/about/contribute/).
 Please reference the link for more full details.

LICENSE

@@ -1,6 +1,6 @@
 MIT License
 
-Copyright (c) 2022 DataJoint
+Copyright (c) 2023 DataJoint
 
 Permission is hereby granted, free of charge, to any person obtaining a copy
 of this software and associated documentation files (the "Software"), to deal