DAS-1713 - Implement CI/CD pipeline to publish Docker images to ghcr.io.

.github/pull_request_template.md

@@ -10,3 +10,4 @@ A short description of the changes in this PR.
 * [ ] Jira ticket acceptance criteria met.
 * [ ] Tests added/updated and passing.
 * [ ] Documentation updated (if needed).
+* [ ] `version.txt` and `CHANGELOG.md` updated (if publishing a new release).

.github/workflows/publish_docker_image.yml

@@ -0,0 +1,93 @@
+# This workflow will run when changes are detected in the `main` branch. It
+# will first trigger the reusable workflow in `.github/workflows/run_tests.yml`,
+# which sets up the local Python environment and runs the `unittest` suite.
+# If that workflow is successful, a tag is added to the latest git commit, a
+# GitHub release is created and the latest version of the service Docker image
+# is pushed to ghcr.io.
+
+name: Publish Harmony NetCDF-to-Zarr Docker image to ghcr.io
+
+on:
+  push:
+    branches: [ main ]
+    paths: version.txt
+
+env:
+  IMAGE_NAME: ${{ github.repository }}
+  REGISTRY: ghcr.io
+
+jobs:
+  run_tests:
+    uses: ./.github/workflows/run_tests.yml
+
+  create_github_release:
+    needs: run_tests
+    permissions:
+      # write permission is required to create a GitHub release
+      contents: write
+    runs-on: ubuntu-20.04
+
+    steps:
+      - name: Checkout harmony-netcdf-to-zarr repository
+        uses: actions/checkout@v3
+
+      - name: Set up Python
+        uses: actions/setup-python@v4
+        with:
+          python-version: 3.9
+
+      - name: Extract semantic version number
+        run: echo "semantic_version=$(cat version.txt)" >> $GITHUB_ENV
+
+      - name: Extract markdown notes from CHANGELOG.md
+        run: |
+          python bin/extract_version_notes.py
+
+      - name: Create Git tag
+        uses: mathieudutour/[email protected]
+        with:
+          github_token: ${{ secrets.GITHUB_TOKEN }}
+          custom_tag: ${{ env.semantic_version }}
+
+      - name: Create GitHub release
+        uses: ncipollo/release-action@v1
+        with:
+          bodyFile: "version_notes.md"
+          tag: ${{ env.semantic_version }}
+
+  build_and_publish_image:
+    needs: run_tests
+    runs-on: ubuntu-20.04
+    strategy:
+      fail-fast: false
+
+    steps:
+      - name: Checkout harmony-netcdf-to-zarr repository
+        uses: actions/checkout@v3
+
+      - name: Extract semantic version number
+        run: echo "semantic_version=$(cat version.txt)" >> $GITHUB_ENV
+
+      - name: Log-in to ghcr.io registry
+        uses: docker/login-action@v2
+        with:
+          registry: ${{ env.REGISTRY }}
+          username: ${{ github.actor }}
+          password: ${{ secrets.GITHUB_TOKEN }}
+
+      - name: Add tags to the Docker image
+        id: meta
+        uses: docker/metadata-action@v4
+        with:
+          images: ${{ env.REGISTRY }}/${{ env.IMAGE_NAME }}
+          tags: |
+            type=semver,pattern={{version}},value=${{ env.semantic_version }}
+
+      - name: Push Docker image
+        uses: docker/build-push-action@v3
+        with:
+          context: .
+          file: Dockerfile
+          push: true
+          tags: ${{ steps.meta.outputs.tags }}
+          labels: ${{ steps.meta.outputs.labels }}

.github/workflows/run_tests.yml

@@ -0,0 +1,64 @@
+# This workflow will install Python dependencies, run tests,
+# and report test results and code coverage as artifacts. It will
+# be called by the workflow that run tests against new PRs and as
+# a first step in the workflow that publishes new Docker images.
+
+name: A reusable workflow to build and run the unit test suite
+
+on:
+  workflow_call
+
+jobs:
+  build_and_test:
+    runs-on: ubuntu-20.04
+    strategy:
+      fail-fast: false
+
+    steps:
+      - name: Retrieve harmony-netcdf-to-zarr repository
+        uses: actions/checkout@v3
+
+      - name: Set up Python 3.9 (version used by service)
+        uses: actions/setup-python@v4
+        with:
+          python-version: 3.9
+
+      - name: Install dependencies
+        run: |
+          python -m pip install --upgrade pip
+          pip install -r requirements/core.txt -r requirements/dev.txt
+
+      - name: Run lint
+        run: |
+          flake8 --ignore=W503 harmony_netcdf_to_zarr
+
+      - name: Run tests
+        run: |
+          bin/test >& test_results.txt
+
+      - name: Generate coverage report
+        if: ${{ always() }}
+        run: |
+          coverage report -m >& coverage_report.txt
+          coverage html --dir htmlcov
+
+      - name: Archive test results
+        if: ${{ always() }}
+        uses: actions/upload-artifact@v3
+        with:
+          name: test result (Python 3.9)
+          path: test_results.txt
+
+      - name: Archive code coverage report (plain text)
+        if: ${{ always() }}
+        uses: actions/upload-artifact@v3
+        with:
+          name: code coverage report (plain text)
+          path: coverage_report.txt
+
+      - name: Archive code coverage report (HTML)
+        if: ${{ always() }}
+        uses: actions/upload-artifact@v3
+        with:
+          name: code coverage report (HTML)
+          path: htmlcov/*

.github/workflows/run_tests_on_pull_requests.yml

@@ -0,0 +1,13 @@
+# This workflow will run when a PR is opened against the `main` branch. It will
+# trigger the reusable workflow in `.github/workflows/run_tests.yml`, which
+# sets up the local Python environment and runs the `unittest` suite.
+
+name: Build and test PR branches
+
+on:
+  pull_request:
+    branches: [ main ]
+
+jobs:
+  build_and_test:
+    uses: ./.github/workflows/run_tests.yml

CHANGELOG.md

@@ -0,0 +1,44 @@
+## v1.0.3
+### 2022-12-13
+
+* DAS-1713 - Publish Docker images to ghcr.io.
+* DAS-1712 - Disable DockerHub publication actions.
+* DAS-1695 - Ensure correct metadata is written to store.
+* Give `compute_chunksize` a side effect to prevent pickling error in testing end to end.
+* Check multiprocess exit codes.
+* DAS-1685 - Ensure single granule requests don't attempt aggregation.
+* DAS 1673 - Implement smart chunking.
+* DAS-1536 - Ensure raised exception messages are correctly tested.
+* HARMONY-1189 - Update harmony-service-lib dependency to 1.0.20.
+* HARMONY-1178 - Handle paged STAC input
+* DAS-1438 - Ensure zero-dimensional variables, like a CRS, resolve full paths.
+* DAS-1438 - Ensure Zarr groups always use a `ProcessSynchronizer`.
+* DAS-1455 - Copy exact data from NetCDF-4 input to Zarr store, including scale and offset metadata.
+* DAS-1379 - Only aggregate temporal dimensions.
+* DAS-1432 - Ensure only the Zarr store is in the STAC.
+* DAS-1379 - Write many input NetCDF-4 granules to single Zarr store.
+* DAS-1414 - Check input granule is NetCDF-4 via media type or extension.
+* DAS-1400 - Support bounds during dimension aggregation.
+* DAS-1376 - Update `HarmonyAdapter` to perform many-to-one operations.
+* DAS-1375 - Scale dimension values to integers before finding differences in resolution calculation.
+* DAS-1375 - Add dimension aggregation to `DimensionsMapping` class.
+* DAS-1374 - Add NetCDF-4 dimension parsing classes, ready for aggregation.
+
+## v1.0.2
+### 2021-11-29
+
+* Add trigger on release publish.
+* HARMONY-388 - Make publish-image consistent with service-example.
+* Performance and chunk sizing improvements (HARMONY-953, HARMONY-953, HARMONY-992, HARMONY-877, HARMONY-855).
+
+## v1.0.1
+### 2021-06-17
+
+* HARMONY-388 - Improve consistency across Python repositories.
+
+## v1.0.0
+### 2021-05-17
+
+* HARMONY-817 - Change occurrences of "harmony" to "harmonyservices" for Docker images.
+* HARMONY-817 - Publish to DockerHub on merge and release.
+* HARMONY-816 - Add user agent logging.

Dockerfile

@@ -11,15 +11,6 @@ RUN apt-get update && apt-get install -y build-essential git
 COPY requirements/core.txt requirements/core.txt
 RUN pip3 install -r requirements/core.txt
 
-# This is below the preceding layer to prevent Docker from rebuilding the
-# previous layer (forcing a reload of dependencies) whenever the
-# status of a local service library changes
-ARG service_lib_dir=NO_SUCH_DIR
-
-# Install a local harmony-service-lib-py if we have one
-COPY deps ./deps/
-RUN if [ -d deps/${service_lib_dir} ]; then echo "Installing from local copy of harmony-service-lib"; pip install -e deps/${service_lib_dir}; fi
-
 COPY . .
 
 ENTRYPOINT ["python3", "-m", "harmony_netcdf_to_zarr"]