Merge pull request #28 from crim-ca/devtools

.dockerignore

@@ -0,0 +1,28 @@
+## IDE
+.idea/
+.vscode/
+
+## SCM
+.git*
+
+## Configurations
+.*
+*.rc
+
+## Environment
+.conda/
+.env*
+*.env
+.venv/
+jupyter/
+
+## Tests
+.coverage
+.pytest_cache
+reports
+
+## Caches
+**/__pycache__/
+STACpopulator.egg-info/
+build
+*.pyc

.github/workflows/release.yml

@@ -15,6 +15,7 @@ jobs:
   release:
     name: release
     runs-on: ubuntu-latest
+    if: ${{ success() && (contains(github.ref, 'refs/tags') || github.ref == 'refs/heads/master') }}
     steps:
       - name: Checkout
         uses: actions/checkout@v2
@@ -47,5 +48,6 @@ jobs:
         uses: docker/build-push-action@v3
         with:
           context: .
+          file: docker/Dockerfile
           push: true
           tags: ${{ env.REGISTRY }}/${{ env.IMAGE_NAME }}:${{ steps.extract_branch.outputs.branch }}

.github/workflows/tests.yml

@@ -0,0 +1,156 @@
+# run test suites
+
+name: Tests
+on:
+  - pull_request
+  - push
+  - release
+  - workflow_dispatch
+
+# cancel the current workflow if another commit was pushed on the same PR or reference
+# uses the GitHub workflow name to avoid collision with other workflows running on the same PR/reference
+concurrency:
+  group: ${{ github.workflow }}-${{ github.event.pull_request.number || github.ref }}
+  cancel-in-progress: true
+
+jobs:
+  # see: https://github.com/fkirc/skip-duplicate-actions
+#  skip_duplicate:
+#    continue-on-error: true
+#    runs-on: ubuntu-latest
+#    outputs:
+#      should_skip: ${{ steps.skip_check.outputs.should_skip && ! contains(github.ref, 'refs/tags') }}
+#    steps:
+#      - id: skip_check
+#        uses: fkirc/skip-duplicate-actions@master
+#        with:
+#          concurrent_skipping: "same_content"
+#          skip_after_successful_duplicate: "true"
+#          do_not_skip: '["pull_request", "workflow_dispatch", "schedule", "release"]'
+
+  # see: https://github.com/actions/setup-python
+  tests:
+    # FIXME: https://github.com/fkirc/skip-duplicate-actions/issues/90
+    #needs: skip_duplicate
+    #if: ${{ needs.skip_duplicate.outputs.should_skip != 'true' }}
+    runs-on: ${{ matrix.os }}
+    continue-on-error: ${{ matrix.allow-failure }}
+    env:
+      # override make command to install directly in active python
+      CONDA_CMD: ""
+
+    strategy:
+      matrix:
+        os: [ubuntu-latest]
+        python-version: ["3.10", "3.11", "3.12"]
+        allow-failure: [false]
+        test-case: [test-cov]
+#        include:
+#          # experimental python
+#          - os: ubuntu-latest
+#            python-version: "3.13"
+#            allow-failure: true
+#            test-case: test-unit-only
+#          - os: ubuntu-latest
+#            python-version: "3.13"
+#            allow-failure: true
+#            test-case: test-func-only
+#          # linter tests
+#          - os: ubuntu-latest
+#            python-version: "3.10"
+#            allow-failure: false
+#            test-case: check-all
+#          # documentation build
+#          - os: ubuntu-latest
+#            python-version: "3.10"
+#            allow-failure: false
+#            test-case: docs
+#          # coverage test
+#          - os: ubuntu-latest
+#            python-version: "3.10"
+#            allow-failure: false
+#            test-case: test-coverage-only
+#          # smoke test of Docker image
+#          - os: ubuntu-latest
+#            python-version: "3.10"  # doesn't matter which one (in docker), but match default of repo
+#            allow-failure: false
+#            test-case: test-docker
+    steps:
+      - uses: actions/checkout@v2
+        with:
+          fetch-depth: "0"
+      - name: Setup Python
+        # skip python setup if running with docker
+        if: ${{ matrix.test-case != 'test-docker' }}
+        uses: actions/setup-python@v2
+        with:
+          python-version: "${{ matrix.python-version }}"
+      - name: Parse Python Version
+        id: python-semver
+        run: |
+          echo "::set-output name=major:$(echo ${{ matrix.python-version }} | cut -d '.' -f 1)"
+          echo "::set-output name=minor:$(echo ${{ matrix.python-version }} | cut -d '.' -f 2)"
+      - uses: actions/cache@v3
+        name: Check Proj Lib Pre-Built in Cache
+        id: cache-proj
+        with:
+          # note: '22' is v8, '21' is v7
+          path: /tmp/proj-8.2.1/install
+          key: ${{ runner.os }}-python${{ matrix.python-version }}-proj
+      - name: Install Dependencies
+        # skip python setup if running with docker
+        if: ${{ matrix.test-case != 'test-docker' }}
+        # install package and dependencies directly,
+        # skip sys/conda setup to use active python
+        run: make setup-pyessv-archive install-dev version
+      - name: Display Packages
+        # skip python setup if running with docker
+        if: ${{ matrix.test-case != 'test-docker' }}
+        run: pip freeze
+      #- name: Setup Environment Variables
+      #  uses: c-py/action-dotenv-to-setenv@v2
+      #  with:
+      #    env-file: ./ci/weaver.env
+      - name: Display Environment Variables
+        run: |
+          hash -r
+          env | sort
+      - name: Run Tests
+        run: make ${{ matrix.test-case }}
+      - name: Upload coverage report
+        uses: codecov/codecov-action@v2
+        if: ${{ success() && matrix.test-case == 'test-coverage-only' }}
+        with:
+          token: ${{ secrets.CODECOV_TOKEN }}
+          files: ./reports/coverage.xml
+          fail_ci_if_error: true
+          verbose: true
+
+#  deploy-docker:
+#    needs: tests
+#    if: ${{ success() && (contains(github.ref, 'refs/tags') || github.ref == 'refs/heads/master') }}
+#    runs-on: ubuntu-latest
+#    steps:
+#      - uses: actions/checkout@v2
+#        with:
+#          fetch-depth: "0"
+#      - name: Get Tag Version
+#        id: version
+#        shell: bash
+#        run: |
+#          if [[ "${GITHUB_REF}" == "refs/heads/master" ]]; then
+#            echo "::set-output name=TAG_VERSION::latest"
+#          else
+#            echo "::set-output name=TAG_VERSION::${GITHUB_REF##*/}"
+#          fi
+#      - name: Build Docker
+#        run: |
+#          make DOCKER_REPO=pavics/weaver APP_VERSION=${{ steps.version.outputs.TAG_VERSION }} docker-info docker-build
+#      - name: Login to DockerHub
+#        uses: docker/login-action@v1
+#        with:
+#          username: ${{ secrets.DOCKERHUB_USERNAME }}
+#          password: ${{ secrets.DOCKERHUB_TOKEN }}
+#      - name: Push to DockerHub
+#        run: |
+#          make DOCKER_REPO=pavics/weaver APP_VERSION=${{ steps.version.outputs.TAG_VERSION }} docker-push

.gitignore

@@ -1,7 +1,21 @@
-*.pyc
-STACpopulator.egg-info/
+## IDE
+.idea/
 .vscode/
+
+## Environment
+.conda/
+.env*
+*.env
 .venv/
 jupyter/
-.idea
-.vscode
+
+## Tests
+.coverage
+.pytest_cache
+reports
+
+## Caches
+**/__pycache__/
+STACpopulator.egg-info/
+build
+*.pyc

CHANGES.md

@@ -0,0 +1,23 @@
+# Changes
+
+## [Unreleased](https://github.com/crim-ca/stac-populator) (latest)
+
+* Add `LICENSE` file.
+* Add `bump-my-version` with `make version` and `make VERSION=<...> bump` utilities to self-update release versions.
+* Add more metadata to `pyproject.toml`.
+* Adjust `README.md` with updated references and release version indicators.
+* Add `CHANGES.md` to record version updates.
+* Add `dev` dependencies to `pyproject.toml` for testing the package (install with `pip install ".[dev]"`).
+* Add GitHub CI tests.
+* Remove `requirements.txt` in favor of all dependencies combined in `pyproject.toml`.
+* Add test to validate STAC Collection and Item contain `source` with expected THREDDS format.
+* Fix broken tests and invalid imports.
+
+## [0.1.0](https://github.com/crim-ca/stac-populator/tree/0.1.0) (2023-11-08)
+
+
+* Refactor of `CMIP6_UofT` with more robust parsing strategies and STAC Item generation from THREDDS NCML metadata.
+
+## [0.0.1](https://github.com/crim-ca/stac-populator/tree/0.0.1) (2023-08-22)
+
+* Initial release with implementation of `CMIP6_UofT`.

LICENSE

@@ -0,0 +1,21 @@
+MIT License
+
+Copyright (c) 2013-2014 Computer Research Institute of Montreal (CRIM)
+
+Permission is hereby granted, free of charge, to any person obtaining a copy
+of this software and associated documentation files (the "Software"), to deal
+in the Software without restriction, including without limitation the rights
+to use, copy, modify, merge, publish, distribute, sublicense, and/or sell
+copies of the Software, and to permit persons to whom the Software is
+furnished to do so, subject to the following conditions:
+
+The above copyright notice and this permission notice shall be included in all
+copies or substantial portions of the Software.
+
+THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS OR
+IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF MERCHANTABILITY,
+FITNESS FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT. IN NO EVENT SHALL THE
+AUTHORS OR COPYRIGHT HOLDERS BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER
+LIABILITY, WHETHER IN AN ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING FROM,
+OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS IN THE
+SOFTWARE.

Makefile

@@ -1,24 +1,79 @@
-IMP_DIR = STACpopulator/implementations
-STAC_HOST = http://localhost:8880/stac
+MAKEFILE_NAME := $(word $(words $(MAKEFILE_LIST)),$(MAKEFILE_LIST))
+# Include custom config if it is available
+-include Makefile.config
+APP_ROOT    := $(abspath $(lastword $(MAKEFILE_NAME))/..)
+APP_NAME    := STACpopulator
+APP_VERSION ?= 0.1.0
+
+DOCKER_COMPOSE_FILES := -f "$(APP_ROOT)/docker/docker-compose.yml"
+DOCKER_TAG := ghcr.io/crim-ca/stac-populator:$(APP_VERSION)
+
+IMP_DIR := $(APP_NAME)/implementations
+STAC_HOST ?= http://localhost:8880/stac
 # CATALOG = https://pavics.ouranos.ca/twitcher/ows/proxy/thredds/catalog/birdhouse/testdata/xclim/cmip6/catalog.html
 CATALOG = https://daccs.cs.toronto.edu/twitcher/ows/proxy/thredds/catalog/datasets/CMIP6/catalog.html
 # CATALOG = https://daccs.cs.toronto.edu/twitcher/ows/proxy/thredds/catalog/datasets/CMIP6/CMIP/NOAA-GFDL/catalog.html
 # CATALOG = https://daccs.cs.toronto.edu/twitcher/ows/proxy/thredds/catalog/datasets/CMIP6/CMIP/AS-RCEC/catalog.html
 
-testcmip6:
+## -- Testing targets -------------------------------------------------------------------------------------------- ##
+
+setup-pyessv-archive:
+	git clone "https://github.com/ES-DOC/pyessv-archive" ~/.esdoc/pyessv-archive
+
+test-cmip6:
 	python $(IMP_DIR)/CMIP6_UofT/add_CMIP6.py $(STAC_HOST) $(CATALOG)
 
-delcmip6:
+del-cmip6:
 	curl --location --request DELETE '$(STAC_HOST)/collections/CMIP6_UofT'
 	@echo ""
 
-starthost:
-	docker compose up
+docker-start:
+	docker compose $(DOCKER_COMPOSE_FILES) up
+starthost: docker-start
 
-stophost:
-	docker compose down
+docker-stop:
+	docker compose $(DOCKER_COMPOSE_FILES) down
+stophost: docker-stop
+
+docker-build:
+	docker build "$(APP_ROOT)" -f "$(APP_ROOT)/docker/Dockerfile" -t "$(DOCKER_TAG)"
 
 del_docker_volume: stophost
 	docker volume rm stac-populator_stac-db
 
 resethost: del_docker_volume starthost
+
+install:
+	pip install "$(APP_ROOT)"
+
+install-dev:
+	pip install "$(APP_ROOT)[dev]"
+
+test-unit:
+	pytest "$(APP_ROOT)"
+
+test-cov:
+	pytest "$(APP_ROOT)" --cov="$(APP_NAME)" --cov-report=term --cov-report=html
+
+## -- Versioning targets -------------------------------------------------------------------------------------------- ##
+
+# Bumpversion 'dry' config
+# if 'dry' is specified as target, any bumpversion call using 'BUMP_XARGS' will not apply changes
+BUMP_TOOL := bump-my-version
+BUMP_XARGS ?= --verbose --allow-dirty
+ifeq ($(filter dry, $(MAKECMDGOALS)), dry)
+	BUMP_XARGS := $(BUMP_XARGS) --dry-run
+endif
+.PHONY: dry
+dry: pyproject.toml		## run 'bump' target without applying changes (dry-run) [make VERSION=<x.y.z> bump dry]
+	@-echo > /dev/null
+
+.PHONY: bump
+bump:  ## bump version using VERSION specified as user input [make VERSION=<x.y.z> bump]
+	@-echo "Updating package version ..."
+	@[ "${VERSION}" ] || ( echo ">> 'VERSION' is not set"; exit 1 )
+	@-bash -c '$(CONDA_CMD) $(BUMP_TOOL) $(BUMP_XARGS) --new-version "${VERSION}" patch;'
+
+.PHONY: version
+version:	## display current version
+	@-echo "$(APP_NAME) version: $(APP_VERSION)"