Merge remote-tracking branch 'origin/main' into rstudio-vre

.github/workflows/build-container.yml

@@ -39,7 +39,7 @@ jobs:
       with:
         name: ${{ inputs.wheel_artifact_name }}
         path: docker/${{ inputs.docker_folder }}
-    - uses: mikefarah/yq@v4.42.1
+    - uses: mikefarah/yq@v4.44.1
 
     - name: add whl and start files
       run: |
@@ -69,11 +69,21 @@ jobs:
     - name: Set up Docker Buildx
       uses: docker/setup-buildx-action@v3
 
+    - name: Check and truncate version
+      id: check_and_truncate
+      run: |
+        version=${{ steps.get_version.outputs.version }}
+        if [ ${#version} -gt 10 ]; then
+          echo "::set-output name=version::${version:0:10}"
+        else
+          echo "::set-output name=version::$version"
+        fi
+
     - name: Build docker
       uses: docker/build-push-action@v5
       with:
         context: docker/${{ inputs.docker_folder }}
         push: ${{ inputs.push }}
-        tags: qcdis/${{ inputs.tag }}:${{ steps.get_version.outputs.version }},qcdis/${{ inputs.tag }}:latest
+        tags: qcdis/${{ inputs.tag }}:${{ steps.check_and_truncate.outputs.version }},qcdis/${{ inputs.tag }}:latest
         cache-from: type=gha
         cache-to: type=gha,mode=min

.github/workflows/test-build wheel.yml → .github/workflows/check_secrets_and_variables.yml

@@ -1,4 +1,4 @@
-name: Build wheel
+name: Check secrets and variables
 
 on:
   workflow_call:
@@ -9,6 +9,9 @@ on:
       api_endpoint:
         required: true
         type: string
+      argo_url:
+        required: true
+        type: string
       cell_github:
         required: true
         type: string
@@ -31,6 +34,8 @@ on:
         required: true
         type: string
     secrets:
+      argo_api_token:
+        required: true
       cell_github_token:
         required: true
       naavre_api_token:
@@ -49,12 +54,14 @@ env:
   MODULE_MAPPING_URL: ${{ inputs.module_mapping_url }}
   SEARCH_API_ENDPOINT: ${{ inputs.search_api_endpoint }}
   SEARCH_API_TOKEN: ${{ secrets.search_api_token }}
+  ARGO_URL: ${{ inputs.ARGO_URL }}
+  ARGO_API_TOKEN: ${{ secrets.ARGO_API_TOKEN }}
   ASYNC_TEST_TIMEOUT: 30
   DEBUG: ${{ inputs.debug }}
 
 
 jobs:
-  build-linux:
+  check-secrets-and-vars:
     runs-on: ${{ inputs.runs-on }}
     strategy:
       fail-fast: true
@@ -108,6 +115,16 @@ jobs:
             concatenated_error_message="$concatenated_error_message SEARCH_API_TOKEN not set"
             vars_not_set=1
           fi
+          if [ -z "ARGO_URL" ]; then
+            echo "ARGO_URL not set"
+            concatenated_error_message="$concatenated_error_message ARGO_URL not set"
+            vars_not_set=1
+          fi
+          if [ -z "ARGO_API_TOKEN" ]; then
+            echo "ARGO_API_TOKEN not set"
+            concatenated_error_message="$concatenated_error_message ARGO_API_TOKEN not set"
+            vars_not_set=1
+          fi
           if [ -z "$DEBUG" ]; then
             echo "DEBUG not set"
             concatenated_error_message="$concatenated_error_message DEBUG not set"
@@ -118,67 +135,4 @@ jobs:
             echo "Exiting with error"
             echo $concatenated_error_message
             exit 1
-          fi
-
-      - uses: actions/checkout@v4
-
-      - uses: conda-incubator/setup-miniconda@v3
-        with:
-          auto-update-conda: true
-          miniforge-version: latest
-          miniforge-variant: Mambaforge
-          python-version: 3.11
-          path: ${{ env.CONDA }}/envs
-          environment-file: environment.yml
-          activate-environment: jupyterlab
-          use-mamba: true
-
-      - name: Cache Conda environment
-        uses: actions/cache@v4
-        with:
-          path: ${{ env.CONDA }}/envs
-          key: conda-environment-${{ runner.os }}-${{ hashFiles('environment.yml') }}
-          restore-keys: |
-            conda-environment-${{ runner.os }}-
-        id: cache-conda
-
-#      - name: Update environment
-#        if: steps.cache-conda.outputs.cache-hit != 'true'
-#        run:
-#          mamba env update -n jupyterlab -f environment.yml
-
-      - name: Lint with flake8
-        shell: bash -l {0}
-        run: |
-          conda activate jupyterlab
-          # stop the build if there are Python syntax errors or undefined names
-          flake8 . --count --select=E9,F63,F7,F82 --show-source --statistics
-          # exit-zero treats all errors as warnings. The GitHub editor is 127 chars wide
-          flake8 . --count --exit-zero --max-complexity=10 --max-line-length=127 --statistics
-
-      - name: pytest
-        shell: bash -l {0}
-        run: |
-          conda activate jupyterlab
-          python docker/repo_utils/conf_vl_repos.py  --force=True
-          pip install --upgrade build
-          mkdir -p /tmp/data
-          pytest -s --ignore=docker --ignore=cypress --no-header -v
-
-      - name: make release
-        shell: bash -el {0}
-        run: make release
-
-      - name: 'Upload wheel artifact'
-        uses: actions/upload-artifact@v3
-        with:
-          name: jupyterlab_vre
-          path: dist/jupyterlab_vre-0.1.0-py3-none-any.whl
-
-      - name: Print conda environment
-        shell: bash -el {0}
-        run: conda env export
-
-      - name: Print conda yarn.lock
-        shell: bash -el {0}
-        run: cat yarn.lock
+          fi

.github/workflows/lint.yml

@@ -0,0 +1,46 @@
+name: Lint
+
+on:
+  workflow_call:
+    inputs:
+      runs-on:
+        default: ubuntu-latest
+        type: string
+
+
+jobs:
+  lint:
+    runs-on: ${{ inputs.runs-on }}
+    strategy:
+      fail-fast: true
+    steps:
+      - uses: actions/checkout@v4
+
+      - uses: conda-incubator/setup-miniconda@v3
+        with:
+          auto-update-conda: true
+          miniforge-version: latest
+          miniforge-variant: Mambaforge
+          python-version: 3.11
+          path: ${{ env.CONDA }}/envs
+          environment-file: environment.yml
+          activate-environment: jupyterlab
+          use-mamba: true
+
+      - name: Cache Conda environment
+        uses: actions/cache@v4
+        with:
+          path: ${{ env.CONDA }}/envs
+          key: conda-environment-${{ runner.os }}-${{ hashFiles('environment.yml') }}
+          restore-keys: |
+            conda-environment-${{ runner.os }}-
+        id: cache-conda
+
+      - name: Lint with flake8
+        shell: bash -l {0}
+        run: |
+          conda activate jupyterlab
+          # stop the build if there are Python syntax errors or undefined names
+          flake8 . --count --select=E9,F63,F7,F82 --show-source --statistics
+          # exit-zero treats all errors as warnings. The GitHub editor is 127 chars wide
+          flake8 . --count --exit-zero --max-complexity=10 --max-line-length=127 --statistics

.github/workflows/make-release.yml

@@ -0,0 +1,92 @@
+name: make release
+
+on:
+  workflow_call:
+    inputs:
+      runs-on:
+        default: ubuntu-latest
+        type: string
+      api_endpoint:
+        required: true
+        type: string
+      cell_github:
+        required: true
+        type: string
+      debug:
+        required: true
+        type: string
+      jupyterhub_user:
+        required: true
+        type: string
+      module_mapping_url:
+        required: true
+        type: string
+      registry_url:
+        required: true
+        type: string
+      search_api_endpoint:
+        required: true
+        type: string
+      vlab_slug:
+        required: true
+        type: string
+    secrets:
+      cell_github_token:
+        required: true
+      naavre_api_token:
+        required: true
+      search_api_token:
+        required: true
+
+env:
+  CELL_GITHUB: ${{ inputs.cell_github }}
+  CELL_GITHUB_TOKEN: ${{ secrets.cell_github_token }}
+  REGISTRY_URL: ${{ inputs.registry_url }}
+  NAAVRE_API_TOKEN: ${{ secrets.naavre_api_token }}
+  JUPYTERHUB_USER: ${{ inputs.jupyterhub_user }}
+  API_ENDPOINT: ${{ inputs.api_endpoint }}
+  VLAB_SLUG: ${{ inputs.vlab_slug }}
+  MODULE_MAPPING_URL: ${{ inputs.module_mapping_url }}
+  SEARCH_API_ENDPOINT: ${{ inputs.search_api_endpoint }}
+  SEARCH_API_TOKEN: ${{ secrets.search_api_token }}
+  ASYNC_TEST_TIMEOUT: 30
+  DEBUG: ${{ inputs.debug }}
+
+
+jobs:
+  make-release:
+    runs-on: ${{ inputs.runs-on }}
+    strategy:
+      fail-fast: true
+    steps:
+      - uses: actions/checkout@v4
+
+      - uses: conda-incubator/setup-miniconda@v3
+        with:
+          auto-update-conda: true
+          miniforge-version: latest
+          miniforge-variant: Mambaforge
+          python-version: 3.11
+          path: ${{ env.CONDA }}/envs
+          environment-file: environment.yml
+          activate-environment: jupyterlab
+          use-mamba: true
+
+      - name: Cache Conda environment
+        uses: actions/cache@v4
+        with:
+          path: ${{ env.CONDA }}/envs
+          key: conda-environment-${{ runner.os }}-${{ hashFiles('environment.yml') }}
+          restore-keys: |
+            conda-environment-${{ runner.os }}-
+        id: cache-conda
+
+      - name: make release
+        shell: bash -el {0}
+        run: make release
+
+      - name: 'Upload wheel artifact'
+        uses: actions/upload-artifact@v3
+        with:
+          name: jupyterlab_vre
+          path: dist/jupyterlab_vre-0.1.0-py3-none-any.whl

.github/workflows/make.yml

@@ -31,10 +31,62 @@ jobs:
           fallback-runner: ubuntu-latest
           github-token: ${{ secrets.RUNNER_ACCESS_TOKEN }}
 
+  check_secrets_and_variables:
+    uses: ./.github/workflows/check_secrets_and_variables.yml
+    needs: [configure-workflow]
+    with:
+      runs-on: ${{ needs.configure-workflow.outputs.runner }}
+      jupyterhub_user: ${{ vars.JUPYTERHUB_USER }}
+      vlab_slug: ${{ vars.VLAB_SLUG }}
+      cell_github: ${{ vars.CELL_GITHUB }}
+      api_endpoint: ${{ vars.API_ENDPOINT }}
+      module_mapping_url: ${{ vars.MODULE_MAPPING_URL }}
+      search_api_endpoint: ${{ vars.SEARCH_API_ENDPOINT }}
+      registry_url: ${{ vars.REGISTRY_URL }}
+      argo_url: ${{ vars.ARGO_URL }}
+      debug: ${{ vars.DEBUG }}
+    secrets:
+      argo_api_token: ${{ secrets.ARGO_API_TOKEN }}
+      cell_github_token:  ${{ secrets.CELL_GITHUB_TOKEN }}
+      naavre_api_token: ${{ secrets.NAAVRE_API_TOKEN }}
+      search_api_token: ${{ secrets.SEARCH_API_TOKEN }}
+  lint:
+    uses: ./.github/workflows/lint.yml
+    needs: [configure-workflow]
+    if: ${{ needs.configure-workflow.outputs.should_skip != 'true' }}
+    strategy:
+      matrix:
+        group: [1]
+    with:
+      runs-on: ${{ needs.configure-workflow.outputs.runner }}
+
   call-pytest:
-    uses: ./.github/workflows/test-build wheel.yml
+    uses: ./.github/workflows/run-tests.yml
     needs: [configure-workflow]
     if: ${{ needs.configure-workflow.outputs.should_skip != 'true' }}
+#     strategy:
+#       matrix:
+#         group: ["1","2"]
+    with:
+      runs-on: ${{ needs.configure-workflow.outputs.runner }}
+      jupyterhub_user: ${{ vars.JUPYTERHUB_USER }}
+      vlab_slug: ${{ vars.VLAB_SLUG }}
+      cell_github: ${{ vars.CELL_GITHUB }}
+      api_endpoint: ${{ vars.API_ENDPOINT }}
+      module_mapping_url: ${{ vars.MODULE_MAPPING_URL }}
+      search_api_endpoint: ${{ vars.SEARCH_API_ENDPOINT }}
+      registry_url: ${{ vars.REGISTRY_URL }}
+      debug: ${{ vars.DEBUG }}
+#       group: ${{ matrix.group }}
+    secrets:
+      cell_github_token:  ${{ secrets.CELL_GITHUB_TOKEN }}
+      naavre_api_token: ${{ secrets.NAAVRE_API_TOKEN }}
+      search_api_token: ${{ secrets.SEARCH_API_TOKEN }}
+
+  make-release:
+    uses: ./.github/workflows/make-release.yml
+    needs: [configure-workflow,call-pytest,lint,check_secrets_and_variables]
+    if: ${{ needs.configure-workflow.outputs.should_skip != 'true' }}
     with:
       runs-on: ${{ needs.configure-workflow.outputs.runner }}
       jupyterhub_user: ${{ vars.JUPYTERHUB_USER }}
@@ -51,7 +103,7 @@ jobs:
       search_api_token: ${{ secrets.SEARCH_API_TOKEN }}
 
   call-build-container-vanilla:
-    needs: [call-pytest]
+    needs: [make-release]
     uses: ./.github/workflows/build-container.yml
     with:
       wheel_artifact_name: jupyterlab_vre