Implement WebAuthn

.circleci/config.yml

@@ -52,6 +52,9 @@ jobs:
             docker-compose up -d postgres
             sleep 3
             docker-compose run compile tools/postgres/dbtool.js check-evolutions-schema
+      - run:
+          name: Assert that all migrations are mentioned in one migration guide and that they have a reversion counterpart.
+          command: tools/assert-complete-migrations.sh
       - run:
           name: Build frontend documentation
           command: |
@@ -118,6 +121,7 @@ jobs:
       - run:
           name: Run end-to-end tests
           command: |
+            mkdir -p binaryData/Organization_X && chmod 777 binaryData/Organization_X
             for i in {1..3}; do # retry
               .circleci/not-on-master.sh docker-compose run e2e-tests && s=0 && break || s=$?
             done
@@ -200,99 +204,6 @@ jobs:
           name: Send Slack notification (master only)
           command: .circleci/slack-notification.sh
 
-  nightly:
-    docker:
-      - image: scalableminds/puppeteer:master
-    resource_class: large
-    steps:
-      - checkout
-      - run:
-          name: Remove dev-deployment
-          command: >
-            curl
-            -X POST
-            -H "X-Auth-Token: $RELEASE_API_TOKEN"
-            https://kubernetix.scm.io/hooks/remove/webknossos/dev/master?user=CI+%28nightly%29
-      - run:
-          name: Wait 3min
-          command: sleep 180
-      - run:
-          name: Install dev-deployment
-          command: >
-            curl
-            -X POST
-            -H "X-Auth-Token: $RELEASE_API_TOKEN"
-            https://kubernetix.scm.io/hooks/install/webknossos/dev/master?user=CI+%28nightly%29
-      - run:
-          name: Install dependencies and sleep at least 3min
-          command: |
-            yarn install --immutable &
-            sleep 180 &
-            wait
-      - run:
-          name: Refresh datasets
-          command: curl -X POST --fail https://master.webknossos.xyz/data/triggers/checkInboxBlocking?token=$WK_AUTH_TOKEN
-      - run:
-          name: Run screenshot-tests
-          command: |
-            # CircleCI cancels the job after 60 minutes. To ensure that screenshots are still
-            # uploaded as artifacts, we define a timeout of 50 minutes for the screenshot tests.
-            URL=https://master.webknossos.xyz/ \
-            timeout 3000 \
-            yarn test-screenshot
-
-      - store_artifacts:
-          path: frontend/javascripts/test/screenshots
-
-      - store_artifacts:
-          path: frontend/javascripts/test/snapshots/type-check
-
-      - run:
-          name: Bundle screenshots
-          when: always
-          command: |
-            tar -czvf screenshots.tar frontend/javascripts/test/screenshots
-
-      - store_artifacts:
-          path: screenshots.tar
-
-  wkorg_nightly:
-    docker:
-      - image: scalableminds/puppeteer:master
-    resource_class: large
-    steps:
-      - checkout
-      - run:
-          name: Install dependencies
-          command: |
-            yarn install --immutable
-
-      - run:
-          name: Assert GZIP
-          command: |
-            # Test gzipped assets
-            curl -s -I -H "Accept-Encoding: gzip" https://webknossos.org/assets/bundle/main.js | grep -q "content-encoding: gzip"
-            curl -s -I -H "Accept-Encoding: gzip" https://webknossos.org/assets/bundle/main.css | grep -q "content-encoding: gzip"
-            # Test gzipped buckets
-            curl -s -i \
-              -H 'accept: application/octet-stream' \
-              -H 'Accept-Encoding: gzip' \
-              -H 'content-type: application/json' \
-              --data-raw '[{"position":[2752,4320,1728],"additionalCoordinates":[],"mag":[1,1,1],"cubeSize":32,"fourBit":false}]' \
-              'https://data-humerus.webknossos.org/data/datasets/scalable_minds/l4dense_motta_et_al_demo/layers/segmentation/data?token=' \
-              | grep -q "content-encoding: gzip"
-            echo Success.
-
-      - run:
-          name: Run screenshot-tests
-          command: |
-            # CircleCI cancels the job after 60 minutes. To ensure that screenshots are still
-            # uploaded as artifacts, we define a timeout of 50 minutes for the screenshot tests.
-            timeout 3000 \
-            yarn test-wkorg-screenshot
-
-      - store_artifacts:
-          path: frontend/javascripts/test/screenshots-wkorg
 
 workflows:
   version: 2
@@ -304,25 +215,3 @@ workflows:
           filters:
             tags:
               only: /.*/
-  circleci_nightly:
-    jobs:
-      - nightly
-    triggers:
-      - schedule:
-          # 02:15 AM UTC
-          cron: "15 2 * * *"
-          filters:
-            branches:
-              only:
-                - master
-  circleci_wkorg_nightly:
-    jobs:
-      - wkorg_nightly
-    triggers:
-      - schedule:
-          # 03:15 AM UTC
-          cron: "15 3 * * *"
-          filters:
-            branches:
-              only:
-                - master

.codespellrc

@@ -1,6 +1,6 @@
 [codespell]
 # Ref: https://github.com/codespell-project/codespell#using-a-config-file
-skip = *.svg,*.sublime-workspace,*.lock,.codespellrc,./util/target/,./binaryData,./node_modules,./pg,./project/target,./target,./webknossos-datastore/target,./webknossos-jni/target,./webknossos-tracingstore/target,./util/target,./coverage,./public-test,./tools/proxy/node_modules,./docs/publications.md,./public/bundle
+skip = *.svg,*.sublime-workspace,*.lock,.codespellrc,./util/target/,./binaryData,./node_modules,./pg,./project/target,./target,./webknossos-datastore/target,./webknossos-jni/target,./webknossos-tracingstore/target,./util/target,./coverage,./public-test,./tools/proxy/node_modules,./docs/publications.md,./public/bundle,./tools/migration-unified-annotation-versioning/venv
 # some names and camelCased variables etc
 ignore-regex = \b([a-z]+[A-Z][a-zA-Z]*|H Mattern|Manuel|Nat Commun)\b
 ignore-words-list = lod,nd,ue

.editorconfig

@@ -9,6 +9,9 @@ charset = utf-8
 trim_trailing_whitespace = true
 insert_final_newline = true
 
+[*.py]
+indent_size = 4
+
 [*.md]
 trim_trailing_whitespace = false
 

.github/common_edge_cases.md

@@ -14,3 +14,5 @@ Consider SQL pitfalls:
  - `x IN ()` statements must never called with empty list
  - `ARRAY_AGG(x)` may have nullable values (use `ARRAY_REMOVE(ARRAY_AGG(x), null)` instead)
  - Complex SQL queries may have a fanout effect due to multiple left joins, leading to duplicates
+
+When changing the API version, also adapt `ApiVersioning.CURRENT_API_VERSION`

.github/workflows/build_test_deploy.yml

@@ -0,0 +1,13 @@
+name: CI Pipeline
+
+on:
+  workflow_dispatch:
+
+jobs:
+  foo:
+    runs-on: ubuntu-20.04
+    steps:
+    - name: Checkout code
+      uses: actions/checkout@v3
+      with:
+        fetch-depth: 5

.github/workflows/nightly.yml

@@ -0,0 +1,62 @@
+name: Nightly Test Pipeline
+
+on:
+  workflow_dispatch: {}
+  schedule:
+    # Runs every day at 12:00 AM UTC
+    - cron: '0 0 * * *'
+
+jobs:
+  nightly-screenshot-tests:
+    runs-on: ubuntu-latest
+
+    steps:
+      - name: Checkout code
+        uses: actions/checkout@v3
+
+      - name: Remove dev-deployment
+        run: |
+          curl -X POST \
+            -H "X-Auth-Token: ${{ secrets.RELEASE_API_TOKEN }}" \
+            https://kubernetix.scm.io/hooks/remove/webknossos/dev/master?user=CI+%28nightly%29
+
+      - name: Wait 3 minutes
+        run: sleep 180
+
+      - name: Install dev-deployment
+        run: |
+          curl -X POST \
+            -H "X-Auth-Token: ${{ secrets.RELEASE_API_TOKEN }}" \
+            https://kubernetix.scm.io/hooks/install/webknossos/dev/master?user=CI+%28nightly%29
+
+      - uses: actions/setup-node@v4
+        with:
+            node-version: 18
+
+      - name: Install dependencies and wait
+        run: |
+            corepack enable && yarn install --immutable & 
+            sleep 180 &
+            wait
+
+      - name: Refresh datasets
+        run: |
+          curl -X POST --fail https://master.webknossos.xyz/data/triggers/checkInboxBlocking?token=${{ secrets.WK_AUTH_TOKEN }}
+
+      - name: Run screenshot tests
+        run: |
+          URL=https://master.webknossos.xyz/ \
+          timeout 3000 \
+          yarn test-screenshot
+        env:
+          URL: https://master.webknossos.xyz/
+          WK_AUTH_TOKEN: ${{ secrets.WK_AUTH_TOKEN }}
+          BROWSERSTACK_USERNAME: ${{ secrets.BROWSERSTACK_USERNAME }}
+          BROWSERSTACK_ACCESS_KEY : ${{ secrets.BROWSERSTACK_ACCESS_KEY }}
+
+      - name: Upload screenshots as artifact
+        uses: actions/upload-artifact@v4
+        if: always()
+        with:
+          name: screenshots
+          path: frontend/javascripts/test/screenshots

.github/workflows/wkorg-nightly.yaml

@@ -0,0 +1,60 @@
+name: Nightly WK.org Test Pipeline
+
+on:
+  workflow_dispatch: {}
+  schedule:
+    # Runs every day at 12:00 AM UTC
+    - cron: '0 0 * * *'
+
+jobs:
+  nightly-screenshot-tests:
+    runs-on: ubuntu-latest
+    timeout-minutes: 50
+
+    steps:
+      - name: Checkout code
+        uses: actions/checkout@v3
+
+      - uses: actions/setup-node@v4
+        with:
+            node-version: 18
+
+      - name: Install dependencies
+        run: |
+            corepack enable && yarn install --immutable
+
+      - name: Run screenshot tests
+        run: |
+          yarn test-wkorg-screenshot
+        env:
+          URL: https://webknossos.org/
+          WK_AUTH_TOKEN: ${{ secrets.WK_AUTH_TOKEN }}
+          BROWSERSTACK_USERNAME: ${{ secrets.BROWSERSTACK_USERNAME }}
+          BROWSERSTACK_ACCESS_KEY : ${{ secrets.BROWSERSTACK_ACCESS_KEY }}
+
+      - name: Upload screenshots as artifact
+        uses: actions/upload-artifact@v4
+        if: always()
+        with:
+          name: screenshots
+          path: frontend/javascripts/test/screenshots-wkorg
+
+
+  nightly-gzip-test:
+    runs-on: ubuntu-latest
+
+    steps:
+      - name: Assert GZIP is enabled
+        run: |
+          # Test gzipped assets
+          curl -s -I -H "Accept-Encoding: gzip" https://webknossos.org/assets/bundle/main.js | grep -q "content-encoding: gzip"
+          curl -s -I -H "Accept-Encoding: gzip" https://webknossos.org/assets/bundle/main.css | grep -q "content-encoding: gzip"
+          # Test gzipped buckets
+          curl -s -i \
+            -H 'accept: application/octet-stream' \
+            -H 'Accept-Encoding: gzip' \
+            -H 'content-type: application/json' \
+            --data-raw '[{"position":[2752,4320,1728],"additionalCoordinates":[],"mag":[1,1,1],"cubeSize":32,"fourBit":false}]' \
+            'https://data-humerus.webknossos.org/data/datasets/scalable_minds/l4dense_motta_et_al_demo/layers/segmentation/data?token=' \
+            | grep -q "content-encoding: gzip"
+          echo Success.

.gitignore

@@ -24,6 +24,7 @@ RUNNING_PID
 .bloop
 .metals
 metals.sbt
+__pycache__/
 
 
 # Webknossos
@@ -89,6 +90,15 @@ flow-coverage/
 .nyc_output
 coverage
 coverage-ts
+*.vscode
+.bsp
+temp-webknossos-schema**
+conf/application.conf-e
+.env
+.bloop
+.metals
+metals.sbt
+
 **/screenshots/*.diff.png
 **/screenshots/*.new.png
 
@@ -100,4 +110,4 @@ coverage-ts
 !.yarn/releases
 !.yarn/sdks
 !.yarn/versions
-tools/proxy/.yarn/*
+tools/**/.yarn/*