More efficient CI

.github/workflows/doc-qa.yaml

@@ -2,24 +2,11 @@ name: Doc QA
 on:
   # Triggers the workflow on pull request events for the main branch
   pull_request_target:
-    types:
-      - opened
-
-permissions:
-  pull-requests: write
+    branches:
+      - develop
+      - develop-1.9
 
 jobs:
-  pyspell:
-    runs-on: ubuntu-latest
-    steps:
-      # Spellcheck
-      - uses: actions/checkout@v4
-        with:
-          fetch-depth: 0
-
-      - uses: igsekor/[email protected]
-        name: Spellcheck
-
   doctor-rst:
     name: Lint (DOCtor-RST)
     runs-on: ubuntu-latest
@@ -44,10 +31,3 @@ jobs:
           uses: docker://oskarstark/doctor-rst:1.23.0
           with:
               args: --short --error-format=github --cache-file=/github/workspace/.cache/doctor-rst.cache
-
-  documentation-preview:
-    runs-on: ubuntu-latest
-    steps:
-      - uses: readthedocs/actions/preview@v1
-        with:
-          project-slug: "datacube-core"

.github/workflows/main.yml

@@ -1,34 +1,66 @@
 name: build
 
 on:
-  pull_request:
-    paths:
-      - '**'
-
   push:
-    paths:
-      - '**'
-      - '!docs/**'
-      - '!contrib/**'
+    branches:
+      - main
+  pull_request:
+    branches:
+      - develop
+      - develop-1.9
 
-  workflow_dispatch:
 
-env:
-  DOCKER_USER: gadockersvc
-  DOCKER_IMAGE: opendatacube/datacube-tests:latest1.9
-  # NB Restore standard image name when merge into develop
-  # DOCKER_IMAGE: opendatacube/datacube-tests:latest
+concurrency:
+  # Here the group is defined by the head_ref of the PR
+  group: ${{ github.head_ref }}
+  # Here we specify that we'll cancel any "in progress" workflow of the same group. Thus if we push, ammend a commit and push
+  # again the previous workflow will be cancelled, thus saving us github action build minutes and avoid any conflicts
+  cancel-in-progress: true
 
 
 jobs:
+  lint:
+    runs-on: ubuntu-latest
+    strategy:
+      matrix:
+        python-version: ["3.12"]
+        lint-command:
+          - "pylint --j 2 --reports no datacube"
+          - "--group types mypy datacube"
+          - "pycodestyle tests integration_tests examples --max-line-length 120"
+
+    name: Code Lint
+    steps:
+      - name: checkout git
+        uses: actions/checkout@v4
+        with:
+          fetch-depth: 0
+      - name: Install uv
+        uses: astral-sh/setup-uv@v2
+        with:
+          version: 0.5.5
+          enable-cache: 'true'
+          cache-suffix: ${{ matrix.python-version }}
+      - name: Run ${{ matrix.lint-command }}
+        run: |
+          uv run ${{ matrix.lint-command }}
+
   main:
     runs-on: ubuntu-latest
+    needs: ['lint']
 
     steps:
     - uses: actions/checkout@v4
       with:
         fetch-depth: 0
 
+    - name: Install uv
+      uses: astral-sh/setup-uv@v2
+      with:
+        version: 0.5.5
+        enable-cache: 'true'
+        cache-suffix: 3.12
+
     - name: Config
       id: cfg
       run: |
@@ -40,102 +72,35 @@ jobs:
           echo "push_pypi=yes" >> $GITHUB_OUTPUT
         fi
 
-    - uses: dorny/paths-filter@v3
-      id: changes
-      if: |
-        github.event_name == 'push'
-      with:
-        filters: |
-          docker:
-            - 'docker/**'
-    - name: Pull Docker
-      if: steps.changes.outputs.docker == 'false'
-      run: |
-        docker pull "${{ env.DOCKER_IMAGE }}"
-
-    - name: Set up Docker Buildx
-      if: steps.changes.outputs.docker == 'true'
-      uses: docker/setup-buildx-action@v3
-
-    - name: Cache Docker layers
-      if: steps.changes.outputs.docker == 'true'
-      uses: pat-s/[email protected]
-      with:
-        path: /tmp/.buildx-cache
-        key: ${{ runner.os }}-buildx-${{ github.sha }}
-        restore-keys: |
-          ${{ runner.os }}-buildx-
-
-    - name: DockerHub Login
-      id: login
-      if: |
-        github.event_name == 'push'
-        && github.ref == 'refs/heads/develop'
-      uses: docker/login-action@v3
-      with:
-        username: ${{ env.DOCKER_USER }}
-        password: ${{ secrets.GADOCKERSVC_PASSWORD }}
-
-    - name: Build Docker
-      uses: docker/build-push-action@v6
-      with:
-        file: docker/Dockerfile
-        context: .
-        tags: ${{ env.DOCKER_IMAGE }}
-        load: true
-
     - name: Verify and Run Tests
       run: |
-        echo "Verify that twine is installed"
-        docker run --rm ${{ env.DOCKER_IMAGE }} twine --version
-
         echo "Run tests"
-        cat <<EOF | docker run --rm -i -v $(pwd):/code ${{ env.DOCKER_IMAGE }} bash -
-          pip install -e /code/tests/drivers/fail_drivers --no-deps
-          pip install -e /code/examples/io_plugin --no-deps
-          pytest -r a \
-            --cov datacube \
-            --cov-report=xml \
-            --doctest-ignore-import-errors \
-            --durations=5 \
-            datacube \
-            tests \
-            integration_tests
-        EOF
-
-    - name: DockerHub Push
-      if: |
-        github.event_name == 'push'
-        && github.ref == 'refs/heads/develop'
-        && steps.changes.outputs.docker == 'true'
-      uses: docker/build-push-action@v6
-      with:
-        file: docker/Dockerfile
-        context: .
-        push: true
-        tags: ${{env.DOCKER_IMAGE}}
+        uv sync 
+        source .venv/bin/activate
+        uv pip install -e ./tests/drivers/fail_drivers --no-deps
+        uv pip install -e ./examples/io_plugin --no-deps
+        pytest -r a \
+          --cov datacube \
+          --cov-report=xml \
+          --doctest-ignore-import-errors \
+          --durations=5 \
+          datacube \
+          tests \
+          integration_tests
+
 
     - name: Build Packages
       run: |
-        cat <<EOF | docker run --rm -i  \
-                  -v $(pwd):/code \
-                  -e SKIP_DB=yes \
-                  ${{ env.DOCKER_IMAGE }} bash -
-        python setup.py bdist_wheel sdist
+        uv run --with build python-build
         ls -lh ./dist/
-        twine check ./dist/*
-        EOF
+        uv run --with twine twine check ./dist/*
 
     - name: Publish to Test PyPi
       if: |
         steps.cfg.outputs.push_test_pypi == 'yes'
       run: |
         if [ -n "${TWINE_PASSWORD}" ]; then
-          docker run --rm  \
-            -v $(pwd):/code \
-            -e SKIP_DB=yes \
-            ${{ env.DOCKER_IMAGE }} \
-            twine upload \
+            uv run --with twine twine upload \
               --verbose \
               --non-interactive \
               --disable-progress-bar \
@@ -156,11 +121,7 @@ jobs:
         && steps.cfg.outputs.push_pypi == 'yes'
       run: |
         if [ -n "${TWINE_PASSWORD}" ]; then
-          docker run --rm  \
-            -v $(pwd):/code \
-            -e SKIP_DB=yes \
-            ${{ env.DOCKER_IMAGE }} \
-            twine upload \
+            uv run --with twine twine upload \
               --verbose \
               --non-interactive \
               --disable-progress-bar \

.github/workflows/rtd-preview.yaml

@@ -0,0 +1,21 @@
+name: RTD Preview
+
+# Runs once on a new Pull Request
+# Lets Read the Docs add a comment and link to the preview documentation build
+
+
+on:
+  pull_request_target:
+    types:
+      - opened
+
+permissions:
+  pull-requests: write
+
+jobs:
+  documentation-preview:
+    runs-on: ubuntu-latest
+    steps:
+      - uses: readthedocs/actions/preview@v1
+        with:
+          project-slug: "datacube-core"