diff --git a/.devcontainer/Dockerfile b/.devcontainer/Dockerfile index 1d6a055..0b0fd38 100644 --- a/.devcontainer/Dockerfile +++ b/.devcontainer/Dockerfile @@ -13,6 +13,7 @@ RUN \ usermod -aG docker vscode && \ apt-get clean +COPY ./requirements_dev.txt /tmp/ RUN \ # dev setup apt update && \ @@ -22,6 +23,8 @@ RUN \ pip install --no-cache-dir --upgrade black pip && \ echo '. /etc/bash_completion' >> /home/vscode/.bashrc && \ echo 'export PS1="\[\e[32;1m\]\u\[\e[m\]@\[\e[34;1m\]\H\[\e[m\]:\[\e[33;1m\]\w\[\e[m\]$ "' >> /home/vscode/.bashrc && \ + pip install --no-cache-dir -r /tmp/requirements_dev.txt && \ + rm /tmp/requirements_dev.txt && \ # dircolors -b >> /home/vscode/.bashrc && \ # somehow fix colors apt-get clean COPY ./requirements.txt /tmp/ diff --git a/.devcontainer/devcontainer.json b/.devcontainer/devcontainer.json index 40cbfe0..bebf37a 100644 --- a/.devcontainer/devcontainer.json +++ b/.devcontainer/devcontainer.json @@ -1,5 +1,5 @@ { - "name": "Tutorial environment", + "name": "Environment + Data", "dockerComposeFile": "docker-compose.yaml", "service": "app", "workspaceFolder": "/workspaces/${localWorkspaceFolderBasename}", @@ -9,17 +9,17 @@ "onCreateCommand": "mkdir -p ${IMAGING_ROOT_DATA_DIR} && pip install -e . && MYSQL_VER=8.0 docker compose down && MYSQL_VER=8.0 docker compose up --build --wait", "postStartCommand": "docker volume prune -f && s3fs ${DJ_PUBLIC_S3_LOCATION} ${IMAGING_ROOT_DATA_DIR} -o nonempty,multipart_size=530,endpoint=us-east-1,url=http://s3.amazonaws.com,public_bucket=1", "hostRequirements": { - "cpus": 2, + "cpus": 4, "memory": "16gb", "storage": "32gb" }, "forwardPorts": [ 3306 ], - "customizations": { + "customizations": { "settings": { "python.pythonPath": "/usr/local/bin/python" - }, + }, "vscode": { "extensions": [ "ms-python.python", diff --git a/.devcontainer/docker-compose.yaml b/.devcontainer/docker-compose.yaml index e8107cd..ca9788b 100644 --- a/.devcontainer/docker-compose.yaml +++ b/.devcontainer/docker-compose.yaml @@ -2,7 +2,7 @@ version: "3" services: app: extends: - file: ./local-data/docker-compose.yaml + file: ./local/docker-compose.yaml service: app environment: - DJ_PUBLIC_S3_LOCATION=djhub.vathes.datapub.elements:/workflow-calcium-imaging-data-v2 diff --git a/.devcontainer/local-test/devcontainer.json b/.devcontainer/local-test/devcontainer.json deleted file mode 100644 index b1c1d77..0000000 --- a/.devcontainer/local-test/devcontainer.json +++ /dev/null @@ -1,30 +0,0 @@ -{ - "name": "Dev environment for local pytests with remote data", - "dockerComposeFile": "docker-compose.yaml", - "service": "app", - "workspaceFolder": "/workspaces/${localWorkspaceFolderBasename}", - "remoteEnv": { - "LOCAL_WORKSPACE_FOLDER": "${localWorkspaceFolder}" - }, - "onCreateCommand": "pip install -r ./requirements_dev.txt && pip install -e . && pip install -e ../element-calcium-imaging && MYSQL_VER=8.0 docker compose down && MYSQL_VER=8.0 docker compose up --build --wait", - "postStartCommand": "docker volume prune -f && s3fs ${DJ_PUBLIC_S3_LOCATION} ${IMAGING_ROOT_DATA_DIR} -o nonempty,multipart_size=530,endpoint=us-east-1,url=http://s3.amazonaws.com,public_bucket=1", - "hostRequirements": { - "cpus": 2, - "memory": "16gb", - "storage": "32gb" - }, - "forwardPorts": [ - 3306 - ], - "customizations": { - "settings": { - "python.pythonPath": "/usr/local/bin/python" - }, - "vscode": { - "extensions": [ - "ms-python.python", - "ms-toolsai.jupyter" - ] - } - } -} \ No newline at end of file diff --git a/.devcontainer/local-test/docker-compose.yaml b/.devcontainer/local-test/docker-compose.yaml deleted file mode 100644 index add45d0..0000000 --- a/.devcontainer/local-test/docker-compose.yaml +++ /dev/null @@ -1,15 +0,0 @@ -version: "3" -services: - app: - cpus: 2 - mem_limit: 16g - environment: - - DJ_PUBLIC_S3_LOCATION=djhub.vathes.datapub.elements:/workflow-calcium-imaging-data-v2 - build: - context: ../.. - dockerfile: ./.devcontainer/Dockerfile - extra_hosts: - - fakeservices.datajoint.io:127.0.0.1 - volumes: - - ../../..:/workspaces:cached - privileged: true # only because of dind \ No newline at end of file diff --git a/.devcontainer/local-data/devcontainer.json b/.devcontainer/local/devcontainer.json similarity index 88% rename from .devcontainer/local-data/devcontainer.json rename to .devcontainer/local/devcontainer.json index 5cc077e..81b73f5 100644 --- a/.devcontainer/local-data/devcontainer.json +++ b/.devcontainer/local/devcontainer.json @@ -1,5 +1,5 @@ { - "name": "Dev environment with local data", + "name": "Environment", "dockerComposeFile": "docker-compose.yaml", "service": "app", "workspaceFolder": "/workspaces/${localWorkspaceFolderBasename}", @@ -9,17 +9,17 @@ "onCreateCommand": "pip install -e . && MYSQL_VER=8.0 docker compose down && MYSQL_VER=8.0 docker compose up --build --wait", "postStartCommand": "docker volume prune -f", "hostRequirements": { - "cpus": 2, + "cpus": 4, "memory": "16gb", "storage": "32gb" }, "forwardPorts": [ 3306 ], - "customizations": { + "customizations": { "settings": { "python.pythonPath": "/usr/local/bin/python" - }, + }, "vscode": { "extensions": [ "ms-python.python", diff --git a/.devcontainer/local-data/docker-compose.yaml b/.devcontainer/local/docker-compose.yaml similarity index 53% rename from .devcontainer/local-data/docker-compose.yaml rename to .devcontainer/local/docker-compose.yaml index 80d22c4..44c5f62 100644 --- a/.devcontainer/local-data/docker-compose.yaml +++ b/.devcontainer/local/docker-compose.yaml @@ -1,15 +1,16 @@ version: "3" services: app: - cpus: 2 + cpus: 4 mem_limit: 16g - build: - context: ../.. - dockerfile: ./.devcontainer/Dockerfile + # build: + # context: ../.. + # dockerfile: ./.devcontainer/Dockerfile + image: datajoint/workflow_calcium_imaging_devcontainer:latest extra_hosts: - fakeservices.datajoint.io:127.0.0.1 volumes: - - ../../..:/workspaces:cached + - ../..:/workspaces/workflow-calcium-imaging:cached - docker_data:/var/lib/docker # persist docker images privileged: true # only because of dind volumes: diff --git a/.github/.staging_workflows/anchored_u24_workflow_before_release.yaml b/.github/.staging_workflows/anchored_u24_workflow_before_release.yaml deleted file mode 100644 index 76ea65f..0000000 --- a/.github/.staging_workflows/anchored_u24_workflow_before_release.yaml +++ /dev/null @@ -1,29 +0,0 @@ -name: u24_workflow_before_release_0.0.1 -on: - pull_request: - push: - branches: - - '**' - tags-ignore: - - '**' - workflow_dispatch: - -anchor-dev-build-call: &dev-build-call - uses: yambottle/djsciops-cicd/.github/workflows/u24_workflow_build.yaml@main - -anchor-test-build-call: &test-build-call - uses: yambottle/djsciops-cicd/.github/workflows/u24_workflow_build.yaml@main - -anchor-prod-build-call: &prod-build-call - uses: dj-sciops/djsciops-cicd/.github/workflows/u24_workflow_build.yaml@main - -jobs: - call_context_check: - uses: dj-sciops/djsciops-cicd/.github/workflows/context_check.yaml@main - - call_u24_workflow_build_debian: - !!merge <<: *$STAGE-build-call - with: - jhub_ver: 1.4.2 - py_ver: 3.9 - dist: debian \ No newline at end of file diff --git a/.github/.staging_workflows/anchored_u24_workflow_release_call.yaml b/.github/.staging_workflows/anchored_u24_workflow_release_call.yaml deleted file mode 100644 index d9fc757..0000000 --- a/.github/.staging_workflows/anchored_u24_workflow_release_call.yaml +++ /dev/null @@ -1,46 +0,0 @@ -name: u24_workflow_release_call_0.0.1 - -on: - workflow_run: - workflows: ["u24_workflow_tag_to_release_0.0.1"] - types: - - completed - -anchor-dev-release-call: &dev-release-call - uses: yambottle/djsciops-cicd/.github/workflows/u24_workflow_release.yaml@main - -anchor-test-release-call: &test-release-call - uses: yambottle/djsciops-cicd/.github/workflows/u24_workflow_release.yaml@main - -anchor-prod-release-call: &prod-release-call - uses: dj-sciops/djsciops-cicd/.github/workflows/u24_workflow_release.yaml@main - -anchor-dev-release-if: &dev-release-if - if: >- - github.event.workflow_run.conclusion == 'success' && - github.repository_owner == 'yambottle' - -anchor-test-release-if: &test-release-if - if: >- - github.event.workflow_run.conclusion == 'success' && - github.repository_owner == 'yambottle' - -anchor-prod-release-if: &prod-release-if - if: >- - github.event.workflow_run.conclusion == 'success' && - github.repository_owner == 'datajoint' - -jobs: - call_context_check: - uses: dj-sciops/djsciops-cicd/.github/workflows/context_check.yaml@main - - call_u24_workflow_release_debian: - !!merge <<: *$STAGE-release-if - !!merge <<: *$STAGE-release-call - with: - jhub_ver: 1.4.2 - py_ver: 3.9 - dist: debian - secrets: - REGISTRY_USERNAME: ${{secrets.DOCKER_USERNAME}} - REGISTRY_PASSWORD: ${{secrets.DOCKER_PASSWORD}} \ No newline at end of file diff --git a/.github/.staging_workflows/anchored_u24_workflow_tag_to_release.yaml b/.github/.staging_workflows/anchored_u24_workflow_tag_to_release.yaml deleted file mode 100644 index 982f287..0000000 --- a/.github/.staging_workflows/anchored_u24_workflow_tag_to_release.yaml +++ /dev/null @@ -1,26 +0,0 @@ -name: u24_workflow_tag_to_release_0.0.1 - -on: - push: - tags: - - '*.*.*' - - 'test*.*.*' - -anchor-dev-build-call: &dev-build-call - uses: yambottle/djsciops-cicd/.github/workflows/u24_workflow_build.yaml@main - -anchor-test-build-call: &test-build-call - uses: yambottle/djsciops-cicd/.github/workflows/u24_workflow_build.yaml@main - -anchor-prod-build-call: &prod-build-call - uses: dj-sciops/djsciops-cicd/.github/workflows/u24_workflow_build.yaml@main - -jobs: - call_context_check: - uses: dj-sciops/djsciops-cicd/.github/workflows/context_check.yaml@main - call_u24_workflow_build_debian: - !!merge <<: *$STAGE-build-call - with: - jhub_ver: 1.4.2 - py_ver: 3.9 - dist: debian \ No newline at end of file diff --git a/.github/.staging_workflows/normalize.sh b/.github/.staging_workflows/normalize.sh deleted file mode 100644 index 08619ee..0000000 --- a/.github/.staging_workflows/normalize.sh +++ /dev/null @@ -1,14 +0,0 @@ -#!/bin/bash -# For Github Action that doesn't support anchor yet... -# https://github.com/actions/runner/issues/1182 - -STAGE=$1 -# .yaml in .staging_workflows has to be named using a prefix 'anchored_', this will be removed when normalizing -PREFIX="anchored_" -SCRIPT_DIR=$( cd -- "$( dirname -- "${BASH_SOURCE[0]}" )" &> /dev/null && pwd ) -for source in $(ls $SCRIPT_DIR | grep yaml) -do - target=${source#$PREFIX} - export STAGE - envsubst '${STAGE}' < $SCRIPT_DIR/$source | yq e 'explode(.) | del(.anchor-*)' > $SCRIPT_DIR/../workflows/$target -done \ No newline at end of file diff --git a/.github/.test/.secrets b/.github/.test/.secrets deleted file mode 100644 index 0cdd0de..0000000 --- a/.github/.test/.secrets +++ /dev/null @@ -1,4 +0,0 @@ -RAW_DEPLOY_KEY= -GITHUB_TOKEN= -REGISTRY_USERNAME= -REGISTRY_PASSWORD= \ No newline at end of file diff --git a/.github/.test/README.md b/.github/.test/README.md deleted file mode 100644 index 3ffc63a..0000000 --- a/.github/.test/README.md +++ /dev/null @@ -1,3 +0,0 @@ -For local testing using act, you need to create few things: -- make a `.test/artifacts` dir for `act --artifact-server-path ./.test/artifacts/` -- make a `.test/.secrets` file similar as `.env` for `act --secret-file ./.test/.secrets` \ No newline at end of file diff --git a/.github/make-dev.sh b/.github/make-dev.sh deleted file mode 100644 index e9bdb38..0000000 --- a/.github/make-dev.sh +++ /dev/null @@ -1,2 +0,0 @@ -#!/bin/bash -bash ./.staging_workflows/normalize.sh dev \ No newline at end of file diff --git a/.github/make-prod.sh b/.github/make-prod.sh deleted file mode 100644 index 806e39e..0000000 --- a/.github/make-prod.sh +++ /dev/null @@ -1,2 +0,0 @@ -#!/bin/bash -bash ./.staging_workflows/normalize.sh prod \ No newline at end of file diff --git a/.github/make-test.sh b/.github/make-test.sh deleted file mode 100644 index 3870e77..0000000 --- a/.github/make-test.sh +++ /dev/null @@ -1,2 +0,0 @@ -#!/bin/bash -bash ./.staging_workflows/normalize.sh test \ No newline at end of file diff --git a/.github/run-act.sh b/.github/run-act.sh deleted file mode 100644 index 4ae9cdf..0000000 --- a/.github/run-act.sh +++ /dev/null @@ -1,7 +0,0 @@ -#!/bin/bash -SCRIPT_DIR=$( cd -- "$( dirname -- "${BASH_SOURCE[0]}" )" &> /dev/null && pwd ) -bash $SCRIPT_DIR/.staging_workflows/normalize.sh dev -cd .. -act -P ubuntu-latest=drewyangdev/ubuntu:act-latest \ - --secret-file $SCRIPT_DIR/.test/.secrets \ - --artifact-server-path $SCRIPT_DIR/.test/artifacts/ \ No newline at end of file diff --git a/.github/workflows/release.yaml b/.github/workflows/release.yaml new file mode 100644 index 0000000..5109e17 --- /dev/null +++ b/.github/workflows/release.yaml @@ -0,0 +1,13 @@ +name: Release +on: + workflow_dispatch: +jobs: + devcontainer-build: + uses: datajoint/.github/.github/workflows/devcontainer-build.yaml@main + devcontainer-publish: + needs: + - devcontainer-build + uses: datajoint/.github/.github/workflows/devcontainer-publish.yaml@main + secrets: + DOCKERHUB_USERNAME: ${{secrets.DOCKERHUB_USERNAME}} + DOCKERHUB_TOKEN: ${{secrets.DOCKERHUB_TOKEN}} diff --git a/.github/workflows/test.yaml b/.github/workflows/test.yaml new file mode 100644 index 0000000..6b8420d --- /dev/null +++ b/.github/workflows/test.yaml @@ -0,0 +1,7 @@ +name: Test +on: + push: + pull_request: +jobs: + devcontainer-build: + uses: datajoint/.github/.github/workflows/devcontainer-build.yaml@main diff --git a/.github/workflows/u24_workflow_before_release.yaml b/.github/workflows/u24_workflow_before_release.yaml deleted file mode 100644 index 28a5ff5..0000000 --- a/.github/workflows/u24_workflow_before_release.yaml +++ /dev/null @@ -1,18 +0,0 @@ -name: u24_workflow_before_release_0.0.1 -on: - pull_request: - push: - branches: - - '**' - tags-ignore: - - '**' - workflow_dispatch: -jobs: - call_context_check: - uses: dj-sciops/djsciops-cicd/.github/workflows/context_check.yaml@main - call_u24_workflow_build_debian: - uses: dj-sciops/djsciops-cicd/.github/workflows/u24_workflow_build.yaml@main - with: - jhub_ver: 1.4.2 - py_ver: 3.9 - dist: debian diff --git a/.github/workflows/u24_workflow_release_call.yaml b/.github/workflows/u24_workflow_release_call.yaml deleted file mode 100644 index 8196673..0000000 --- a/.github/workflows/u24_workflow_release_call.yaml +++ /dev/null @@ -1,20 +0,0 @@ -name: u24_workflow_release_call_0.0.1 -on: - workflow_run: - workflows: ["u24_workflow_tag_to_release_0.0.1"] - types: - - completed -jobs: - call_context_check: - uses: dj-sciops/djsciops-cicd/.github/workflows/context_check.yaml@main - call_u24_workflow_release_debian: - if: >- - github.event.workflow_run.conclusion == 'success' && github.repository_owner == 'datajoint' - uses: dj-sciops/djsciops-cicd/.github/workflows/u24_workflow_release.yaml@main - with: - jhub_ver: 1.4.2 - py_ver: 3.9 - dist: debian - secrets: - REGISTRY_USERNAME: ${{secrets.DOCKER_USERNAME}} - REGISTRY_PASSWORD: ${{secrets.DOCKER_PASSWORD}} diff --git a/.github/workflows/u24_workflow_tag_to_release.yaml b/.github/workflows/u24_workflow_tag_to_release.yaml deleted file mode 100644 index 3a6ce58..0000000 --- a/.github/workflows/u24_workflow_tag_to_release.yaml +++ /dev/null @@ -1,15 +0,0 @@ -name: u24_workflow_tag_to_release_0.0.1 -on: - push: - tags: - - '*.*.*' - - 'test*.*.*' -jobs: - call_context_check: - uses: dj-sciops/djsciops-cicd/.github/workflows/context_check.yaml@main - call_u24_workflow_build_debian: - uses: dj-sciops/djsciops-cicd/.github/workflows/u24_workflow_build.yaml@main - with: - jhub_ver: 1.4.2 - py_ver: 3.9 - dist: debian diff --git a/README.md b/README.md index a196084..0e5c080 100644 --- a/README.md +++ b/README.md @@ -35,15 +35,15 @@ NIS`, or `PrairieView` acquisition software, using [Suite2p](https://github.com/ Here are some options that provide a great experience: -- Cloud-based Environment (*recommended*) +- (*recommended*) Cloud-based Environment - Launch using [GitHub Codespaces](https://github.com/features/codespaces) using the `+` option which will `Create codespace on main` in the codebase repository on your fork with default options. For more control, see the `...` where you may create `New with options...`. - - Build time for a codespace is several minutes. This is done infrequently and cached for convenience. + - Build time for a codespace is a few minutes. This is done infrequently and cached for convenience. - Start time for a codespace is less than 1 minute. This will pull the built codespace from cache when you need it. - *Tip*: Each month, GitHub renews a [free-tier](https://docs.github.com/en/billing/managing-billing-for-github-codespaces/about-billing-for-github-codespaces#monthly-included-storage-and-core-hours-for-personal-accounts) quota of compute and storage. Typically we run into the storage limits before anything else since Codespaces consume storage while stopped. It is best to delete Codespaces when not actively in use and recreate when needed. We'll soon be creating prebuilds to avoid larger build times. Once any portion of your quota is reached, you will need to wait for it to be reset at the end of your cycle or add billing info to your GitHub account to handle overages. - *Tip*: GitHub auto names the codespace but you can rename the codespace so that it is easier to identify later. - Local Environment - - Note: On Windows, running the tutorial notebook with the example data in a Dev Container is not currently possible due to a s3fs mounting issue. Please use the `Cloud-based Development Environment` option above. + > *Note: Access to example data is currently limited to MacOS and Linux due to the s3fs utility. Windows users are recommended to use the above environment.* - Install [Git](https://git-scm.com/book/en/v2/Getting-Started-Installing-Git) - Install [Docker](https://docs.docker.com/get-docker/) - Install [VSCode](https://code.visualstudio.com/)