From a871bc5a83808af51ba0f27a45ddf798228ce350 Mon Sep 17 00:00:00 2001 From: Damien Ayers Date: Fri, 29 Nov 2024 09:45:45 +1100 Subject: [PATCH 01/14] Replace setup.py with pyproject.toml --- pyproject.toml | 133 ++++++++++++++++++++++++++++++++++++++- setup.cfg | 6 -- setup.py | 165 ------------------------------------------------- 3 files changed, 131 insertions(+), 173 deletions(-) delete mode 100755 setup.py diff --git a/pyproject.toml b/pyproject.toml index 4f0f416502..3a021d6285 100644 --- a/pyproject.toml +++ b/pyproject.toml @@ -1,6 +1,135 @@ -# pyproject.toml +[project] +name = 'datacube' +dynamic = ['version'] +requires-python = ">=3.10" +readme = "README.rst" +license = {text = "Apache License 2.0"} +description = "An analysis environment for satellite and other earth observation data" +classifiers = [ + "Development Status :: 4 - Beta", + "Intended Audience :: Developers", + "Intended Audience :: Science/Research", + "License :: OSI Approved :: Apache Software License", + "Natural Language :: English", + "Operating System :: MacOS :: MacOS X", + "Operating System :: POSIX", + "Operating System :: POSIX :: BSD", + "Operating System :: POSIX :: Linux", + "Operating System :: Microsoft :: Windows", + "Programming Language :: Python", + "Programming Language :: Python :: 3", + "Programming Language :: Python :: 3.10", + "Programming Language :: Python :: 3.11", + "Programming Language :: Python :: 3.12", + "Topic :: Scientific/Engineering :: GIS", + "Topic :: Scientific/Engineering :: Information Analysis", +] +dependencies = [ + 'affine', + 'antimeridian', + 'attrs>=18.1', + 'pyproj>=2.5', + 'shapely>=2.0', + 'cachetools', + 'click>=5.0', + 'cloudpickle>=0.4', + 'dask[array]<2024.11.0', # Dask versions from 2024.11 cause problems with numpy2 + 'distributed<2024.11.0', + 'jsonschema>=4.18', # New reference resolution API + 'numpy>=1.26.0', + 'lark', + 'pandas', + 'python-dateutil', + 'pyyaml', + 'rasterio>=1.3.11', # Deleted Numpy version in 1.3.10 + 'ruamel.yaml', + 'sqlalchemy>=2.0', # GeoAlchemy2 requires >=1.4. + 'GeoAlchemy2', + "alembic", + 'toolz', + 'xarray>=0.9', # >0.9 fixes most problems with `crs` attributes being lost + 'packaging', + 'odc-geo>=0.4.8', + 'deprecat', +] + +[project.urls] +Homepage = "https://opendatacube.org/" +Documentation = "https://datacube-core.readthedocs.io/" +Source = "https://github.com/opendatacube/datacube-core/" +Issues = "https://github.com/opendatacube/datacube-core/issues" +Chat = "https://discord.com/invite/4hhBQVas5U" + +[project.optional-dependencies] +performance = ['ciso8601', 'bottleneck'] +distributed = ['distributed', 'dask[distributed]'] +s3 = ['boto3', 'botocore'] +cf = ['compliance-checker>=4.0.0'] +netcdf = ['netcdf4'] +postgres = ['psycopg2'] + +[dependency-groups] +dev = [ + 'hypothesis', + 'pycodestyle', + 'pylint', + 'pytest', + 'pytest-cov', + 'pytest-timeout', + 'pytest-httpserver', + 'moto<5.0', # 5.0 will require changes to some tests. +] +types = [ + "mypy", + "types-affine", + "types-cachetools", + "types-jsonschema", + "types-psutil", + "types-python-dateutil", + "types-PyYAML", + "types-redis", + "types-setuptools", + "boto3-stubs", + "botocore-stubs" +] +doc = [ + 'Sphinx', + 'sphinx_autodoc_typehints', # Propagate mypy info into docs + 'sphinx-click', + 'recommonmark', + 'autodocsumm', + 'beautifulsoup4', + 'nbsphinx', + 'pydata-sphinx-theme==0.9.0', +] + +[project.scripts] +datacube = 'datacube.scripts.cli_app:cli' +datacube-search = 'datacube.scripts.search_tool:cli' +datacube-worker = 'datacube.execution.worker:main' + + [build-system] -requires = ["setuptools>=42", "wheel", "setuptools_scm[toml]>=3.4"] +requires = ["setuptools>=69", "setuptools_scm[toml]>=3.4"] +# v61 Add support for pyproject.toml, and include-package-data defaults to try when using pyproject.toml +# v69 includes py.typed files by default +build-backend = "setuptools.build_meta" + +[project.entry-points."datacube.plugins.io.read"] +netcdf = "datacube.drivers.netcdf.driver:reader_driver_init" + +[project.entry-points."datacube.plugins.io.write"] +netcdf = 'datacube.drivers.netcdf.driver:writer_driver_init' + +[project.entry-points."datacube.plugins.io.index"] +postgres = 'datacube.index.postgres.index:index_driver_init' +null = 'datacube.index.null.index:index_driver_init' +memory = 'datacube.index.memory.index:index_driver_init' +postgis = 'datacube.index.postgis.index:index_driver_init' + +[tool.setuptools.packages.find] +exclude = ['integration_tests', 'tests', 'examples', 'docs'] +include = ['datacube*'] [tool.setuptools_scm] write_to = "datacube/_version.py" diff --git a/setup.cfg b/setup.cfg index 7dce40ae36..4ba60d7ba6 100644 --- a/setup.cfg +++ b/setup.cfg @@ -1,9 +1,3 @@ -[aliases] -test=pytest - -[bdist_wheel] -universal=1 - [flake8] max-line-length = 120 ignore = diff --git a/setup.py b/setup.py deleted file mode 100755 index d74448a1d0..0000000000 --- a/setup.py +++ /dev/null @@ -1,165 +0,0 @@ -#!/usr/bin/env python - -from setuptools import setup, find_packages - -tests_require = [ - 'hypothesis', - 'pycodestyle', - 'pylint', - 'pytest', - 'pytest-cov', - 'pytest-timeout', - 'pytest-httpserver', - 'moto<5.0', # 5.0 will require changes to some tests. - 'psycopg2', -] - -types_require = [ - "mypy", - "types-affine", - "types-cachetools", - "types-jsonschema", - "types-psutil", - "types-python-dateutil", - "types-PyYAML", - "types-redis", - "types-setuptools", - "boto3-stubs", - "botocore-stubs" -] - -doc_require = [ - 'Sphinx', - 'sphinx_autodoc_typehints', # Propagate mypy info into docs - 'sphinx-click', - 'recommonmark', - 'autodocsumm', - 'beautifulsoup4', - 'nbsphinx', - 'pydata-sphinx-theme==0.9.0', -] - -extras_require = { - 'performance': ['ciso8601', 'bottleneck'], - 'distributed': ['distributed', 'dask[distributed]'], - 'doc': doc_require, - 's3': ['boto3', 'botocore'], - 'test': tests_require, - "types": types_require, - 'cf': ['compliance-checker>=4.0.0'], - 'netcdf': ['netcdf4'], - 'postgres': ['psycopg2'], -} - -extras_require['dev'] = sorted(set(sum([extras_require[k] for k in [ - 'test', - 'types', - 'doc', - 'performance', - 's3', - 'distributed', - 'netcdf', - 'postgres', -]], []))) - -# An 'all' option, following ipython naming conventions. -extras_require['all'] = sorted(set(sum(extras_require.values(), []))) - -extra_plugins = dict(read=[], write=[], index=[]) - -setup( - name='datacube', - python_requires='>=3.10.0', - - url='https://github.com/opendatacube/datacube-core', - author='Open Data Cube', - maintainer='Open Data Cube', - maintainer_email='', - description='An analysis environment for satellite and other earth observation data', - long_description=open('README.rst').read(), - long_description_content_type='text/x-rst', - license='Apache License 2.0', - classifiers=[ - "Development Status :: 4 - Beta", - "Intended Audience :: Developers", - "Intended Audience :: Science/Research", - "License :: OSI Approved :: Apache Software License", - "Natural Language :: English", - "Operating System :: MacOS :: MacOS X", - "Operating System :: POSIX", - "Operating System :: POSIX :: BSD", - "Operating System :: POSIX :: Linux", - "Operating System :: Microsoft :: Windows", - "Programming Language :: Python", - "Programming Language :: Python :: 3", - "Programming Language :: Python :: 3.10", - "Programming Language :: Python :: 3.11", - "Programming Language :: Python :: 3.12", - "Topic :: Scientific/Engineering :: GIS", - "Topic :: Scientific/Engineering :: Information Analysis", - ], - - packages=find_packages( - exclude=('tests', 'tests.*', - 'integration_tests', 'integration_tests.*') - ), - package_data={ - '': ['*.yaml', '*/*.yaml'], - 'datacube': ['py.typed'], - 'datacube.drivers.postgis': ['alembic.ini', 'alembic/*.py*', 'alembic/README', 'alembic/versions/*.py'], - }, - scripts=[], - install_requires=[ - 'affine', - 'antimeridian', - 'attrs>=18.1', - 'pyproj>=2.5', - 'shapely>=2.0', - 'cachetools', - 'click>=5.0', - 'cloudpickle>=0.4', - 'dask[array]<2024.11.0', # Dask versions from 2024.11 cause problems with numpy2 - 'distributed<2024.11.0', - 'jsonschema>=4.18', # New reference resolution API - 'numpy>=1.26.0', - 'lark', - 'pandas', - 'python-dateutil', - 'pyyaml', - 'rasterio>=1.3.11', # Deleted Numpy version in 1.3.10 - 'ruamel.yaml', - 'sqlalchemy>=2.0', # GeoAlchemy2 requires >=1.4. - 'GeoAlchemy2', - "alembic", - 'toolz', - 'xarray>=0.9', # >0.9 fixes most problems with `crs` attributes being lost - 'packaging', - 'odc-geo>=0.4.8', - 'deprecat', - ], - extras_require=extras_require, - tests_require=tests_require, - - entry_points={ - 'console_scripts': [ - 'datacube = datacube.scripts.cli_app:cli', - 'datacube-search = datacube.scripts.search_tool:cli', - 'datacube-worker = datacube.execution.worker:main', - ], - 'datacube.plugins.io.read': [ - 'netcdf = datacube.drivers.netcdf.driver:reader_driver_init', - *extra_plugins['read'], - ], - 'datacube.plugins.io.write': [ - 'netcdf = datacube.drivers.netcdf.driver:writer_driver_init', - *extra_plugins['write'], - ], - 'datacube.plugins.index': [ - 'postgres = datacube.index.postgres.index:index_driver_init', - 'null = datacube.index.null.index:index_driver_init', - 'memory = datacube.index.memory.index:index_driver_init', - 'postgis = datacube.index.postgis.index:index_driver_init', - *extra_plugins['index'], - ], - }, -) From bd0a74d08708ab838d4e7823ecb9d92de7de31e7 Mon Sep 17 00:00:00 2001 From: Damien Ayers Date: Fri, 29 Nov 2024 12:08:07 +1100 Subject: [PATCH 02/14] Use PEP 735 dependency groups while linting in CI --- .github/workflows/lint.yaml | 71 ++++++++----------------------------- 1 file changed, 14 insertions(+), 57 deletions(-) diff --git a/.github/workflows/lint.yaml b/.github/workflows/lint.yaml index 336f322d59..15df346e49 100644 --- a/.github/workflows/lint.yaml +++ b/.github/workflows/lint.yaml @@ -15,72 +15,29 @@ on: - '!contrib/**' jobs: - pylint: + lint: runs-on: ubuntu-latest strategy: matrix: - python-version: ["3.10"] - name: Pylint - steps: - - name: checkout git - uses: actions/checkout@v4 - with: - fetch-depth: 0 - - name: Setup conda - uses: s-weigand/setup-conda@v1 - with: - update-conda: true - python-version: ${{ matrix.python-version }} - conda-channels: anaconda, conda-forge - - name: run pylint - run: | - sudo apt-get remove python3-openssl - pip install --upgrade -e '.[test]' - pip install pylint>3.2 - pylint -j 2 --reports no datacube - + python-version: ["3.12"] + lint-command: + - "pylint --j 2 --reports no datacube" + - "--group types mypy datacube" + - "pycodestyle tests integration_tests examples --max-line-length 120" - mypy: - runs-on: ubuntu-latest - strategy: - matrix: - python-version: ["3.10"] - name: MyPy + name: Code Lint steps: - name: checkout git uses: actions/checkout@v4 with: fetch-depth: 0 - - name: Setup conda - uses: s-weigand/setup-conda@v1 + - name: Install uv + uses: astral-sh/setup-uv@v2 with: - update-conda: true - python-version: ${{ matrix.python-version }} - conda-channels: anaconda, conda-forge - - name: run mypy + version: 0.5.5 + enable-cache: 'true' + cache-suffix: ${{ matrix.python-version }} + - name: Run ${{ matrix.lint-command }} run: | - sudo apt-get remove python3-openssl - pip install --upgrade -e '.[types]' - mypy datacube - + uv run ${{ matrix.lint-command }} - pycodestyle: - name: pycodestyle - runs-on: ubuntu-latest - strategy: - matrix: - python-version: ["3.10"] - steps: - - name: Checkout - uses: actions/checkout@v4 - - name: Setup conda - uses: s-weigand/setup-conda@v1 - with: - update-conda: true - python-version: ${{ matrix.python-version }} - conda-channels: anaconda, conda-forge - - name: Run pycodestyle - run: | - sudo apt-get remove python3-openssl - pip install --upgrade -e '.[test]' - pycodestyle tests integration_tests examples --max-line-length 120 From bd12792b85825b75f39177536dc8dd517c1aceba Mon Sep 17 00:00:00 2001 From: Damien Ayers Date: Fri, 29 Nov 2024 12:08:07 +1100 Subject: [PATCH 03/14] Update whats_new.rst --- docs/about/whats_new.rst | 1 + 1 file changed, 1 insertion(+) diff --git a/docs/about/whats_new.rst b/docs/about/whats_new.rst index 8dfb82ea86..02b5ac2403 100644 --- a/docs/about/whats_new.rst +++ b/docs/about/whats_new.rst @@ -19,6 +19,7 @@ v1.9.next * Fix broken documentation build (:pull:`1668`) * Rename `DatasetType` to `Product` in all the tests (:pull:`1671`) * Documentation updates for 1.9 release (:pull:`1664`) +* Move package metadata from `setup.py` to `pyproject.toml` (:pull:`1670`) v1.9.0-rc11 (28th October 2024) =============================== From 4692933a7359f43b7c735c8adf0f07eb9294f64d Mon Sep 17 00:00:00 2001 From: Damien Ayers Date: Mon, 2 Dec 2024 12:07:10 +1100 Subject: [PATCH 04/14] Fix typo in pyproject.toml index entry_points --- pyproject.toml | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/pyproject.toml b/pyproject.toml index 3a021d6285..069da4b2d0 100644 --- a/pyproject.toml +++ b/pyproject.toml @@ -121,7 +121,7 @@ netcdf = "datacube.drivers.netcdf.driver:reader_driver_init" [project.entry-points."datacube.plugins.io.write"] netcdf = 'datacube.drivers.netcdf.driver:writer_driver_init' -[project.entry-points."datacube.plugins.io.index"] +[project.entry-points."datacube.plugins.index"] postgres = 'datacube.index.postgres.index:index_driver_init' null = 'datacube.index.null.index:index_driver_init' memory = 'datacube.index.memory.index:index_driver_init' From bd85eba38e064461417065fa3ac17428f28783b1 Mon Sep 17 00:00:00 2001 From: "pre-commit-ci[bot]" <66853113+pre-commit-ci[bot]@users.noreply.github.com> Date: Sun, 1 Dec 2024 23:28:02 +0000 Subject: [PATCH 05/14] CI: Cancel in progress and don't double run on PRs --- .github/workflows/lint.yaml | 26 +++++++++++++++----------- 1 file changed, 15 insertions(+), 11 deletions(-) diff --git a/.github/workflows/lint.yaml b/.github/workflows/lint.yaml index 15df346e49..c24b70d7d8 100644 --- a/.github/workflows/lint.yaml +++ b/.github/workflows/lint.yaml @@ -1,18 +1,23 @@ name: lint + on: + push: + branches: + - main pull_request: - paths: - - '**' + branches: + - main + - develop-1.9 + + +concurrency: + # Here the group is defined by the head_ref of the PR + group: ${{ github.head_ref }} + # Here we specify that we'll cancel any "in progress" workflow of the same group. Thus if we push, ammend a commit and push + # again the previous workflow will be cancelled, thus saving us github action build minutes and avoid any conflicts + cancel-in-progress: true - push: - paths: - - '**' - - '!.github/**' - - '.github/workflows/lint.yaml' - - '!docker/**' - - '!docs/**' - - '!contrib/**' jobs: lint: @@ -40,4 +45,3 @@ jobs: - name: Run ${{ matrix.lint-command }} run: | uv run ${{ matrix.lint-command }} - From eb2b781d3878456d8776bade384d4bfeee33fa3b Mon Sep 17 00:00:00 2001 From: Damien Ayers Date: Mon, 2 Dec 2024 12:55:56 +1100 Subject: [PATCH 06/14] More efficient CI --- .github/workflows/doc-qa.yaml | 26 +---- .github/workflows/lint.yaml | 47 -------- .github/workflows/main.yml | 165 +++++++++++------------------ .github/workflows/rtd-preview.yaml | 21 ++++ 4 files changed, 87 insertions(+), 172 deletions(-) delete mode 100644 .github/workflows/lint.yaml create mode 100644 .github/workflows/rtd-preview.yaml diff --git a/.github/workflows/doc-qa.yaml b/.github/workflows/doc-qa.yaml index 233c7a988d..7dea4db283 100644 --- a/.github/workflows/doc-qa.yaml +++ b/.github/workflows/doc-qa.yaml @@ -2,24 +2,11 @@ name: Doc QA on: # Triggers the workflow on pull request events for the main branch pull_request_target: - types: - - opened - -permissions: - pull-requests: write + branches: + - develop + - develop-1.9 jobs: - pyspell: - runs-on: ubuntu-latest - steps: - # Spellcheck - - uses: actions/checkout@v4 - with: - fetch-depth: 0 - - - uses: igsekor/pyspelling-any@v1.0.4 - name: Spellcheck - doctor-rst: name: Lint (DOCtor-RST) runs-on: ubuntu-latest @@ -44,10 +31,3 @@ jobs: uses: docker://oskarstark/doctor-rst:1.23.0 with: args: --short --error-format=github --cache-file=/github/workspace/.cache/doctor-rst.cache - - documentation-preview: - runs-on: ubuntu-latest - steps: - - uses: readthedocs/actions/preview@v1 - with: - project-slug: "datacube-core" diff --git a/.github/workflows/lint.yaml b/.github/workflows/lint.yaml deleted file mode 100644 index c24b70d7d8..0000000000 --- a/.github/workflows/lint.yaml +++ /dev/null @@ -1,47 +0,0 @@ -name: lint - - -on: - push: - branches: - - main - pull_request: - branches: - - main - - develop-1.9 - - -concurrency: - # Here the group is defined by the head_ref of the PR - group: ${{ github.head_ref }} - # Here we specify that we'll cancel any "in progress" workflow of the same group. Thus if we push, ammend a commit and push - # again the previous workflow will be cancelled, thus saving us github action build minutes and avoid any conflicts - cancel-in-progress: true - - -jobs: - lint: - runs-on: ubuntu-latest - strategy: - matrix: - python-version: ["3.12"] - lint-command: - - "pylint --j 2 --reports no datacube" - - "--group types mypy datacube" - - "pycodestyle tests integration_tests examples --max-line-length 120" - - name: Code Lint - steps: - - name: checkout git - uses: actions/checkout@v4 - with: - fetch-depth: 0 - - name: Install uv - uses: astral-sh/setup-uv@v2 - with: - version: 0.5.5 - enable-cache: 'true' - cache-suffix: ${{ matrix.python-version }} - - name: Run ${{ matrix.lint-command }} - run: | - uv run ${{ matrix.lint-command }} diff --git a/.github/workflows/main.yml b/.github/workflows/main.yml index 610239306d..492968f2f1 100644 --- a/.github/workflows/main.yml +++ b/.github/workflows/main.yml @@ -1,34 +1,66 @@ name: build on: - pull_request: - paths: - - '**' - push: - paths: - - '**' - - '!docs/**' - - '!contrib/**' + branches: + - main + pull_request: + branches: + - develop + - develop-1.9 - workflow_dispatch: -env: - DOCKER_USER: gadockersvc - DOCKER_IMAGE: opendatacube/datacube-tests:latest1.9 - # NB Restore standard image name when merge into develop - # DOCKER_IMAGE: opendatacube/datacube-tests:latest +concurrency: + # Here the group is defined by the head_ref of the PR + group: ${{ github.head_ref }} + # Here we specify that we'll cancel any "in progress" workflow of the same group. Thus if we push, ammend a commit and push + # again the previous workflow will be cancelled, thus saving us github action build minutes and avoid any conflicts + cancel-in-progress: true jobs: + lint: + runs-on: ubuntu-latest + strategy: + matrix: + python-version: ["3.12"] + lint-command: + - "pylint --j 2 --reports no datacube" + - "--group types mypy datacube" + - "pycodestyle tests integration_tests examples --max-line-length 120" + + name: Code Lint + steps: + - name: checkout git + uses: actions/checkout@v4 + with: + fetch-depth: 0 + - name: Install uv + uses: astral-sh/setup-uv@v2 + with: + version: 0.5.5 + enable-cache: 'true' + cache-suffix: ${{ matrix.python-version }} + - name: Run ${{ matrix.lint-command }} + run: | + uv run ${{ matrix.lint-command }} + main: runs-on: ubuntu-latest + needs: ['lint'] steps: - uses: actions/checkout@v4 with: fetch-depth: 0 + - name: Install uv + uses: astral-sh/setup-uv@v2 + with: + version: 0.5.5 + enable-cache: 'true' + cache-suffix: 3.12 + - name: Config id: cfg run: | @@ -40,102 +72,35 @@ jobs: echo "push_pypi=yes" >> $GITHUB_OUTPUT fi - - uses: dorny/paths-filter@v3 - id: changes - if: | - github.event_name == 'push' - with: - filters: | - docker: - - 'docker/**' - - name: Pull Docker - if: steps.changes.outputs.docker == 'false' - run: | - docker pull "${{ env.DOCKER_IMAGE }}" - - - name: Set up Docker Buildx - if: steps.changes.outputs.docker == 'true' - uses: docker/setup-buildx-action@v3 - - - name: Cache Docker layers - if: steps.changes.outputs.docker == 'true' - uses: pat-s/always-upload-cache@v3.0.11 - with: - path: /tmp/.buildx-cache - key: ${{ runner.os }}-buildx-${{ github.sha }} - restore-keys: | - ${{ runner.os }}-buildx- - - - name: DockerHub Login - id: login - if: | - github.event_name == 'push' - && github.ref == 'refs/heads/develop' - uses: docker/login-action@v3 - with: - username: ${{ env.DOCKER_USER }} - password: ${{ secrets.GADOCKERSVC_PASSWORD }} - - - name: Build Docker - uses: docker/build-push-action@v6 - with: - file: docker/Dockerfile - context: . - tags: ${{ env.DOCKER_IMAGE }} - load: true - - name: Verify and Run Tests run: | - echo "Verify that twine is installed" - docker run --rm ${{ env.DOCKER_IMAGE }} twine --version - echo "Run tests" - cat < Date: Mon, 2 Dec 2024 13:25:45 +1100 Subject: [PATCH 07/14] netCDF is still required in the tests --- .github/workflows/main.yml | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/.github/workflows/main.yml b/.github/workflows/main.yml index 492968f2f1..ed06f65a92 100644 --- a/.github/workflows/main.yml +++ b/.github/workflows/main.yml @@ -75,7 +75,7 @@ jobs: - name: Verify and Run Tests run: | echo "Run tests" - uv sync + uv sync --all-extras source .venv/bin/activate uv pip install -e ./tests/drivers/fail_drivers --no-deps uv pip install -e ./examples/io_plugin --no-deps From 773e7f3ff157b44df4b7191d96db5042fcb9875c Mon Sep 17 00:00:00 2001 From: Damien Ayers Date: Mon, 2 Dec 2024 13:25:45 +1100 Subject: [PATCH 08/14] Streamline GitHub Actions Runs - Don't double run (For both PR and Push to any branch) - Cancel if a new Run starts --- .github/workflows/doc-qa.yaml | 16 ++++++++++++++-- .github/workflows/main.yml | 3 +++ 2 files changed, 17 insertions(+), 2 deletions(-) diff --git a/.github/workflows/doc-qa.yaml b/.github/workflows/doc-qa.yaml index 7dea4db283..a772eee9f9 100644 --- a/.github/workflows/doc-qa.yaml +++ b/.github/workflows/doc-qa.yaml @@ -1,10 +1,22 @@ name: Doc QA + on: - # Triggers the workflow on pull request events for the main branch - pull_request_target: + push: branches: + - main - develop - develop-1.9 + pull_request: + branches: + - develop + - develop-1.9 + +concurrency: + # Here the group is defined by the head_ref of the PR + group: ${{ github.head_ref }} + # Here we specify that we'll cancel any "in progress" workflow of the same group. Thus if we push, ammend a commit and push + # again the previous workflow will be cancelled, thus saving us github action build minutes and avoid any conflicts + cancel-in-progress: true jobs: doctor-rst: diff --git a/.github/workflows/main.yml b/.github/workflows/main.yml index ed06f65a92..64300a712d 100644 --- a/.github/workflows/main.yml +++ b/.github/workflows/main.yml @@ -4,6 +4,8 @@ on: push: branches: - main + - develop + - develop-1.9 pull_request: branches: - develop @@ -60,6 +62,7 @@ jobs: version: 0.5.5 enable-cache: 'true' cache-suffix: 3.12 + python-version: 3.12 - name: Config id: cfg From 7b1d1d51e2411715762facf4ddf8f1eb9e45de12 Mon Sep 17 00:00:00 2001 From: Damien Ayers Date: Mon, 2 Dec 2024 13:25:45 +1100 Subject: [PATCH 09/14] Don't double run conda smoke test CI --- .github/workflows/test-conda-build.yml | 20 +++++++++++++++----- 1 file changed, 15 insertions(+), 5 deletions(-) diff --git a/.github/workflows/test-conda-build.yml b/.github/workflows/test-conda-build.yml index 6413b61aed..3e92697b30 100644 --- a/.github/workflows/test-conda-build.yml +++ b/.github/workflows/test-conda-build.yml @@ -1,12 +1,22 @@ name: Conda Smoke Test + on: + push: + branches: + - main + - develop + - develop-1.9 pull_request: - paths: - - '**' + branches: + - develop + - develop-1.9 - push: - paths: - - '**' +concurrency: + # Here the group is defined by the head_ref of the PR + group: ${{ github.head_ref }} + # Here we specify that we'll cancel any "in progress" workflow of the same group. Thus if we push, ammend a commit and push + # again the previous workflow will be cancelled, thus saving us github action build minutes and avoid any conflicts + cancel-in-progress: true jobs: check-conda: From 29144660361bc5d2cdc2ba0e0767ad62d0ef6025 Mon Sep 17 00:00:00 2001 From: Damien Ayers Date: Mon, 2 Dec 2024 13:25:45 +1100 Subject: [PATCH 10/14] We need postgres running in CI --- .github/workflows/main.yml | 58 ++++++++++++++++++++++++++++++++++++-- 1 file changed, 55 insertions(+), 3 deletions(-) diff --git a/.github/workflows/main.yml b/.github/workflows/main.yml index 64300a712d..f46362661b 100644 --- a/.github/workflows/main.yml +++ b/.github/workflows/main.yml @@ -50,7 +50,24 @@ jobs: main: runs-on: ubuntu-latest needs: ['lint'] - + services: + # Label used to access the service container + postgres: + # Docker Hub image + image: postgis/postgis:16-3.5 + # Provide the password for postgres + env: + POSTGRES_PASSWORD: postgres + # POSTGRES_DB: pgintegration + # Set health checks to wait until postgres has started + options: >- + --health-cmd pg_isready + --health-interval 10s + --health-timeout 5s + --health-retries 5 + ports: + # Maps tcp port 5432 on service container to the host + - 5432:5432 steps: - uses: actions/checkout@v4 with: @@ -75,7 +92,39 @@ jobs: echo "push_pypi=yes" >> $GITHUB_OUTPUT fi - - name: Verify and Run Tests + - name: Setup test Database + run: | + createdb -h localhost -U postgres pgintegration + createdb -h localhost -U postgres pgisintegration + env: + PGHOST: localhost + PGUSER: postgres + PGPASSWORD: postgres + + + - name: Setup test environment + run: | + uv sync --all-extras + source .venv/bin/activate + uv pip install -e ./tests/drivers/fail_drivers --no-deps + uv pip install -e ./examples/io_plugin --no-deps + + - name: Run Unit and Doc Tests + run: | + echo "Run tests" + uv sync --all-extras + source .venv/bin/activate + uv pip install -e ./tests/drivers/fail_drivers --no-deps + uv pip install -e ./examples/io_plugin --no-deps + pytest -r a \ + --cov datacube \ + --cov-report=xml \ + --doctest-ignore-import-errors \ + --durations=5 \ + datacube \ + tests + + - name: Run Integration Tests run: | echo "Run tests" uv sync --all-extras @@ -90,7 +139,10 @@ jobs: datacube \ tests \ integration_tests - + env: + PGHOST: localhost + PGUSER: postgres + PGPASSWORD: postgres - name: Build Packages run: | From 8e34a4119c272bf9295bf14a38ab9fc5fce3995e Mon Sep 17 00:00:00 2001 From: Damien Ayers Date: Mon, 2 Dec 2024 14:53:46 +1100 Subject: [PATCH 11/14] Fix GHA workflow concurrency I'd set only a single workflow per branch, this makes it one workflow per workflow+branch. --- .github/workflows/doc-qa.yaml | 2 +- .github/workflows/main.yml | 2 +- .github/workflows/test-conda-build.yml | 2 +- 3 files changed, 3 insertions(+), 3 deletions(-) diff --git a/.github/workflows/doc-qa.yaml b/.github/workflows/doc-qa.yaml index a772eee9f9..4781c9776b 100644 --- a/.github/workflows/doc-qa.yaml +++ b/.github/workflows/doc-qa.yaml @@ -13,7 +13,7 @@ on: concurrency: # Here the group is defined by the head_ref of the PR - group: ${{ github.head_ref }} + group: ${{ github.workflow }}-${{ github.head_ref }} # Here we specify that we'll cancel any "in progress" workflow of the same group. Thus if we push, ammend a commit and push # again the previous workflow will be cancelled, thus saving us github action build minutes and avoid any conflicts cancel-in-progress: true diff --git a/.github/workflows/main.yml b/.github/workflows/main.yml index f46362661b..905cc416a2 100644 --- a/.github/workflows/main.yml +++ b/.github/workflows/main.yml @@ -14,7 +14,7 @@ on: concurrency: # Here the group is defined by the head_ref of the PR - group: ${{ github.head_ref }} + group: ${{ github.workflow }}-${{ github.head_ref }} # Here we specify that we'll cancel any "in progress" workflow of the same group. Thus if we push, ammend a commit and push # again the previous workflow will be cancelled, thus saving us github action build minutes and avoid any conflicts cancel-in-progress: true diff --git a/.github/workflows/test-conda-build.yml b/.github/workflows/test-conda-build.yml index 3e92697b30..babbb68eaf 100644 --- a/.github/workflows/test-conda-build.yml +++ b/.github/workflows/test-conda-build.yml @@ -13,7 +13,7 @@ on: concurrency: # Here the group is defined by the head_ref of the PR - group: ${{ github.head_ref }} + group: ${{ github.workflow }}-${{ github.head_ref }} # Here we specify that we'll cancel any "in progress" workflow of the same group. Thus if we push, ammend a commit and push # again the previous workflow will be cancelled, thus saving us github action build minutes and avoid any conflicts cancel-in-progress: true From 3870f51fc88bcfa8bad32619382bb3877757d115 Mon Sep 17 00:00:00 2001 From: Damien Ayers Date: Mon, 2 Dec 2024 14:53:46 +1100 Subject: [PATCH 12/14] Try and fix CI DB tests --- .github/workflows/main.yml | 16 +++++----------- 1 file changed, 5 insertions(+), 11 deletions(-) diff --git a/.github/workflows/main.yml b/.github/workflows/main.yml index 905cc416a2..567981525e 100644 --- a/.github/workflows/main.yml +++ b/.github/workflows/main.yml @@ -58,7 +58,6 @@ jobs: # Provide the password for postgres env: POSTGRES_PASSWORD: postgres - # POSTGRES_DB: pgintegration # Set health checks to wait until postgres has started options: >- --health-cmd pg_isready @@ -81,6 +80,10 @@ jobs: cache-suffix: 3.12 python-version: 3.12 + - uses: actions/setup-python@v5 + with: + python-version: '3.12' + - name: Config id: cfg run: | @@ -111,11 +114,7 @@ jobs: - name: Run Unit and Doc Tests run: | - echo "Run tests" - uv sync --all-extras source .venv/bin/activate - uv pip install -e ./tests/drivers/fail_drivers --no-deps - uv pip install -e ./examples/io_plugin --no-deps pytest -r a \ --cov datacube \ --cov-report=xml \ @@ -126,23 +125,18 @@ jobs: - name: Run Integration Tests run: | - echo "Run tests" - uv sync --all-extras source .venv/bin/activate - uv pip install -e ./tests/drivers/fail_drivers --no-deps - uv pip install -e ./examples/io_plugin --no-deps pytest -r a \ --cov datacube \ --cov-report=xml \ --doctest-ignore-import-errors \ --durations=5 \ - datacube \ - tests \ integration_tests env: PGHOST: localhost PGUSER: postgres PGPASSWORD: postgres + DB_USERNAME: postgres - name: Build Packages run: | From be2f8ca1bf9bb8db437e8b199c361679b33b98af Mon Sep 17 00:00:00 2001 From: Damien Ayers Date: Tue, 3 Dec 2024 10:07:42 +1100 Subject: [PATCH 13/14] More fix attempts (segfaults!?) Try more python versions --- .github/workflows/main.yml | 253 +++++++++++++------------ .github/workflows/test-conda-build.yml | 55 ------ 2 files changed, 127 insertions(+), 181 deletions(-) delete mode 100644 .github/workflows/test-conda-build.yml diff --git a/.github/workflows/main.yml b/.github/workflows/main.yml index 567981525e..74079189ac 100644 --- a/.github/workflows/main.yml +++ b/.github/workflows/main.yml @@ -11,7 +11,6 @@ on: - develop - develop-1.9 - concurrency: # Here the group is defined by the head_ref of the PR group: ${{ github.workflow }}-${{ github.head_ref }} @@ -19,7 +18,6 @@ concurrency: # again the previous workflow will be cancelled, thus saving us github action build minutes and avoid any conflicts cancel-in-progress: true - jobs: lint: runs-on: ubuntu-latest @@ -41,7 +39,7 @@ jobs: uses: astral-sh/setup-uv@v2 with: version: 0.5.5 - enable-cache: 'true' + enable-cache: "true" cache-suffix: ${{ matrix.python-version }} - name: Run ${{ matrix.lint-command }} run: | @@ -49,7 +47,11 @@ jobs: main: runs-on: ubuntu-latest - needs: ['lint'] + needs: ["lint"] + strategy: + fail-fast: false + matrix: + python-version: ["3.10", "3.11", "3.12", "3.13"] services: # Label used to access the service container postgres: @@ -68,125 +70,124 @@ jobs: # Maps tcp port 5432 on service container to the host - 5432:5432 steps: - - uses: actions/checkout@v4 - with: - fetch-depth: 0 - - - name: Install uv - uses: astral-sh/setup-uv@v2 - with: - version: 0.5.5 - enable-cache: 'true' - cache-suffix: 3.12 - python-version: 3.12 - - - uses: actions/setup-python@v5 - with: - python-version: '3.12' - - - name: Config - id: cfg - run: | - push_test_pypi=no - push_pypi=no - - if [[ "${GITHUB_REF}" =~ refs/tags/.* ]]; then - echo "push_test_pypi=yes" >> $GITHUB_OUTPUT - echo "push_pypi=yes" >> $GITHUB_OUTPUT - fi - - - name: Setup test Database - run: | - createdb -h localhost -U postgres pgintegration - createdb -h localhost -U postgres pgisintegration - env: - PGHOST: localhost - PGUSER: postgres - PGPASSWORD: postgres - - - - name: Setup test environment - run: | - uv sync --all-extras - source .venv/bin/activate - uv pip install -e ./tests/drivers/fail_drivers --no-deps - uv pip install -e ./examples/io_plugin --no-deps - - - name: Run Unit and Doc Tests - run: | - source .venv/bin/activate - pytest -r a \ - --cov datacube \ - --cov-report=xml \ - --doctest-ignore-import-errors \ - --durations=5 \ - datacube \ - tests - - - name: Run Integration Tests - run: | - source .venv/bin/activate - pytest -r a \ - --cov datacube \ - --cov-report=xml \ - --doctest-ignore-import-errors \ - --durations=5 \ - integration_tests - env: - PGHOST: localhost - PGUSER: postgres - PGPASSWORD: postgres - DB_USERNAME: postgres - - - name: Build Packages - run: | - uv run --with build python-build - ls -lh ./dist/ - uv run --with twine twine check ./dist/* - - - name: Publish to Test PyPi - if: | - steps.cfg.outputs.push_test_pypi == 'yes' - run: | - if [ -n "${TWINE_PASSWORD}" ]; then - uv run --with twine twine upload \ - --verbose \ - --non-interactive \ - --disable-progress-bar \ - --username=__token__ \ - --password=${TWINE_PASSWORD} \ - --repository-url=${TWINE_REPOSITORY_URL} \ - --skip-existing dist/* || true - else - echo "Skipping upload as 'TestPyPiToken' is not set" - fi - env: - TWINE_PASSWORD: ${{ secrets.TestPyPiToken }} - TWINE_REPOSITORY_URL: 'https://test.pypi.org/legacy/' - - - name: Publish to PyPi - if: | - github.event_name == 'push' - && steps.cfg.outputs.push_pypi == 'yes' - run: | - if [ -n "${TWINE_PASSWORD}" ]; then - uv run --with twine twine upload \ - --verbose \ - --non-interactive \ - --disable-progress-bar \ - --username=__token__ \ - --password=${TWINE_PASSWORD} \ - --skip-existing dist/* - else - echo "Skipping upload as 'PyPiToken' is not set" - fi - - env: - TWINE_PASSWORD: ${{ secrets.PyPiToken }} - - - name: Upload coverage to Codecov - uses: codecov/codecov-action@v4 - with: - file: ./coverage.xml - fail_ci_if_error: false - token: ${{ secrets.CODECOV_TOKEN }} + - uses: actions/checkout@v4 + with: + fetch-depth: 0 + + - name: Install uv + uses: astral-sh/setup-uv@v2 + with: + version: 0.5.5 + enable-cache: "true" + cache-suffix: ${{ matrix.python-version }} + python-version: ${{ matrix.python-version }} + + - uses: actions/setup-python@v5 + with: + python-version: ${{ matrix.python-version }} + + - name: Config + id: cfg + run: | + push_test_pypi=no + push_pypi=no + + if [[ "${GITHUB_REF}" =~ refs/tags/.* ]]; then + echo "push_test_pypi=yes" >> $GITHUB_OUTPUT + echo "push_pypi=yes" >> $GITHUB_OUTPUT + fi + + - name: Setup test Database + run: | + createdb -h localhost -U postgres pgintegration + createdb -h localhost -U postgres pgisintegration + env: + PGHOST: localhost + PGUSER: postgres + PGPASSWORD: postgres + + - name: Setup test environment + run: | + uv sync --all-extras + source .venv/bin/activate + uv pip install -e ./tests/drivers/fail_drivers --no-deps + uv pip install -e ./examples/io_plugin --no-deps + + - name: Run Unit and Doc Tests + run: | + source .venv/bin/activate + pytest -r a \ + --cov datacube \ + --cov-report=xml \ + --doctest-ignore-import-errors \ + --durations=5 \ + datacube \ + tests + + - name: Run Integration Tests + run: | + source .venv/bin/activate + pytest -r a \ + --cov datacube \ + --cov-report=xml \ + --doctest-ignore-import-errors \ + --durations=5 \ + integration_tests + env: + PGHOST: localhost + PGUSER: postgres + PGPASSWORD: postgres + DB_USERNAME: postgres + + - name: Build Packages + run: | + uv run --with build python-build + ls -lh ./dist/ + uv run --with twine twine check ./dist/* + + - name: Publish to Test PyPi + if: | + steps.cfg.outputs.push_test_pypi == 'yes' + run: | + if [ -n "${TWINE_PASSWORD}" ]; then + uv run --with twine twine upload \ + --verbose \ + --non-interactive \ + --disable-progress-bar \ + --username=__token__ \ + --password=${TWINE_PASSWORD} \ + --repository-url=${TWINE_REPOSITORY_URL} \ + --skip-existing dist/* || true + else + echo "Skipping upload as 'TestPyPiToken' is not set" + fi + env: + TWINE_PASSWORD: ${{ secrets.TestPyPiToken }} + TWINE_REPOSITORY_URL: "https://test.pypi.org/legacy/" + + - name: Publish to PyPi + if: | + github.event_name == 'push' + && steps.cfg.outputs.push_pypi == 'yes' + run: | + if [ -n "${TWINE_PASSWORD}" ]; then + uv run --with twine twine upload \ + --verbose \ + --non-interactive \ + --disable-progress-bar \ + --username=__token__ \ + --password=${TWINE_PASSWORD} \ + --skip-existing dist/* + else + echo "Skipping upload as 'PyPiToken' is not set" + fi + + env: + TWINE_PASSWORD: ${{ secrets.PyPiToken }} + + - name: Upload coverage to Codecov + uses: codecov/codecov-action@v4 + with: + file: ./coverage.xml + fail_ci_if_error: false + token: ${{ secrets.CODECOV_TOKEN }} diff --git a/.github/workflows/test-conda-build.yml b/.github/workflows/test-conda-build.yml deleted file mode 100644 index babbb68eaf..0000000000 --- a/.github/workflows/test-conda-build.yml +++ /dev/null @@ -1,55 +0,0 @@ -name: Conda Smoke Test - -on: - push: - branches: - - main - - develop - - develop-1.9 - pull_request: - branches: - - develop - - develop-1.9 - -concurrency: - # Here the group is defined by the head_ref of the PR - group: ${{ github.workflow }}-${{ github.head_ref }} - # Here we specify that we'll cancel any "in progress" workflow of the same group. Thus if we push, ammend a commit and push - # again the previous workflow will be cancelled, thus saving us github action build minutes and avoid any conflicts - cancel-in-progress: true - -jobs: - check-conda: - runs-on: ${{ matrix.os }} - strategy: - matrix: - os: ["ubuntu-latest", "macos-latest", "windows-latest"] - python-version: ["3.10", "3.11", "3.12"] - - steps: - - uses: actions/checkout@v4 - - - name: Cache conda - uses: actions/cache@v4 - env: - # Increase this value to reset cache if setup.py has not changed - CACHE_NUMBER: 0 - with: - path: ~/conda_pkgs_dir - key: - ${{ runner.os }}-conda-${{ env.CACHE_NUMBER }}-${{ - hashFiles('conda-environment.yml') }} - - - uses: conda-incubator/setup-miniconda@v3 - with: - environment-file: conda-environment.yml - auto-update-conda: true - python-version: ${{ matrix.python-version}} - - - name: Bash - shell: bash -l {0} - run: | - pip install . - python --version - datacube --version - conda env export From 37cfc3740b3b4ae71ddb69ea25a5e6e80819d20f Mon Sep 17 00:00:00 2001 From: Damien Ayers Date: Wed, 4 Dec 2024 13:08:08 +1100 Subject: [PATCH 14/14] Try compiling some python packages --- .github/workflows/main.yml | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/.github/workflows/main.yml b/.github/workflows/main.yml index 74079189ac..f49762005c 100644 --- a/.github/workflows/main.yml +++ b/.github/workflows/main.yml @@ -108,7 +108,7 @@ jobs: - name: Setup test environment run: | - uv sync --all-extras + uv sync --all-extras --no-binary-package rasterio --no-binary-package fiona --no-binary-package shapely --no-binary-package cf-units --no-binary-package netCDF4 source .venv/bin/activate uv pip install -e ./tests/drivers/fail_drivers --no-deps uv pip install -e ./examples/io_plugin --no-deps