From 338af8836a4291c7d6d1f224e680eab989e9e907 Mon Sep 17 00:00:00 2001 From: Alex Leith Date: Fri, 7 Jan 2022 11:17:47 +1100 Subject: [PATCH] Refactor Docker requirements to use pip-compile and simplify things (#131) * Refactor Docker requirements to use pip-compile and simplify things * Formatting * Fix editable installs --- Dockerfile | 13 +- constraints.txt | 140 ----------- datacube_alchemist/worker.py | 8 +- requirements.in | 22 ++ requirements.txt | 450 +++++++++++++++++++++++++++++++++-- setup.cfg | 7 +- 6 files changed, 469 insertions(+), 171 deletions(-) delete mode 100644 constraints.txt create mode 100644 requirements.in diff --git a/Dockerfile b/Dockerfile index a0077c5..249b1fd 100644 --- a/Dockerfile +++ b/Dockerfile @@ -1,4 +1,4 @@ -FROM osgeo/gdal:ubuntu-small-3.3.3 +FROM osgeo/gdal:ubuntu-small-3.4.1 ENV DEBIAN_FRONTEND=noninteractive \ LC_ALL=C.UTF-8 \ @@ -29,8 +29,8 @@ RUN echo "Environment is: $ENVIRONMENT" # Pip installation RUN mkdir -p /conf -COPY requirements.txt constraints.txt /conf/ -RUN pip install -r /conf/requirements.txt -c /conf/constraints.txt +COPY requirements.txt /conf/ +RUN pip install -r /conf/requirements.txt # Set up a nice workdir and add the live code ENV APPDIR=/code @@ -41,11 +41,12 @@ ADD . $APPDIR # These ENVIRONMENT flags make this a bit complex, but basically, if we are in dev # then we want to link the source (with the -e flag) and if we're in prod, we # want to delete the stuff in the /code folder to keep it simple. +# no-use-pep517 because of this https://github.com/pypa/pip/issues/7953 RUN if [ "$ENVIRONMENT" = "deployment" ] ; then\ - pip install -c /code/constraints.txt . ; \ - rm -rf /code/* ; \ + pip install . ; \ + rm -rf $APPDIR/* ; \ else \ - pip install -c /code/constraints.txt --editable .[$ENVIRONMENT] ; \ + pip install --no-use-pep517 --editable ".[$ENVIRONMENT]" ; \ fi RUN pip freeze diff --git a/constraints.txt b/constraints.txt deleted file mode 100644 index 23aea89..0000000 --- a/constraints.txt +++ /dev/null @@ -1,140 +0,0 @@ -affine==2.3.0 -aiobotocore==1.3.3 -aiohttp==3.7.4.post0 -aioitertools==0.8.0 -appdirs==1.4.4 -async-timeout==3.0.1 -attrs==21.2.0 -awscli==1.19.106 -black==21.7b0 -boltons==21.0.0 -boto3==1.17.106 -botocore==1.20.106 -cachetools==4.2.2 -cattrs==1.0.0 -certifi==2021.5.30 -cffi==1.14.6 -cftime==1.5.0 -chardet==4.0.0 -ciso8601==2.2.0 -click==8.0.1 -click-plugins==1.1.1 -cligj==0.7.2 -cloudpickle==1.6.0 -colorama==0.4.3 -coverage==5.5 -cryptography==3.4.7 -cycler==0.10.0 -dask==2021.8.0 -dask-image==0.6.0 -datacube==1.8.5 -DAWG==0.8.0 -defusedxml==0.7.1 -digitalearthau==20180116+437.g805c962 -distributed==2021.8.0 -docutils==0.15.2 -eodatasets3==0.22.0 -ephem==4.0.0.2 -fc==1.3.6 -fsspec==2021.7.0 -GDAL==3.3.1 -greenlet==1.1.1 -h5py==3.3.0 -hdstats==0.2.1 -HeapDict==1.0.1 -idna==2.10 -imageio==2.9.0 -importlib-metadata==4.6.4 -iniconfig==1.1.1 -itsdangerous==2.0.1 -Jinja2==3.0.1 -jmespath==0.10.0 -joblib==1.0.1 -jsonschema==3.2.0 -kiwisolver==1.3.1 -lark-parser==0.11.3 -locket==0.2.1 -lxml==4.6.3 -MarkupSafe==2.0.1 -matplotlib==3.4.3 -more-itertools==8.8.0 -moto==2.2.2 -msgpack==1.0.2 -multidict==5.1.0 -mypy-extensions==0.4.3 -netCDF4==1.5.7 -networkx==2.6.2 -nrt-predict==0.1.2 -numexpr==2.7.3 -numpy==1.21.2 -odc-aio==0.2.1.dev2263 -odc-algo==0.2.1.dev2263 -odc-apps-cloud==0.2.1.dev2263 -odc-apps-dc-tools==0.2.1.dev2263 -odc-aws==0.2.1.dev2263 -odc-stac==0.2.0a2 -odc-io==0.2.1.dev2263 -odc-ppt==0.2.1.dev2263 -packaging==21.0 -pandas==1.3.2 -partd==1.2.0 -pathspec==0.9.0 -Pillow==8.3.1 -PIMS==0.5 -pluggy==0.13.1 -psutil==5.8.0 -psycopg2==2.9.1 -py==1.10.0 -pyasn1==0.4.8 -pycparser==2.20 -pydash==5.0.2 -pyparsing==2.4.7 -pyproj==3.1.0 -pyrsistent==0.18.0 -pystac==1.1.0 -pystac-client==0.2.0 -pytest==6.2.4 -pytest-cov==2.12.1 -python-dateutil==2.8.2 -python-rapidjson==1.4 -pytz==2021.1 -PyWavelets==1.1.1 -PyYAML==5.4.1 -rasterio==1.2.6 -regex==2021.8.3 -requests==2.25.1 -requests-cache==0.7.3 -responses==0.13.4 -rsa==4.7.2 -ruamel.yaml==0.17.10 -ruamel.yaml.clib==0.2.6 -s3fs==2021.7.0 -s3transfer==0.4.2 -scikit-image==0.18.2 -scikit-learn==0.24.2 -scipy==1.7.1 -Shapely==1.7.1 -six==1.16.0 -slicerator==1.0.0 -snuggs==1.4.7 -sortedcontainers==2.4.0 -SQLAlchemy==1.4.22 -structlog==21.1.0 -tblib==1.7.0 -threadpoolctl==2.2.0 -tifffile==2021.8.8 -toml==0.10.2 -tomli==1.2.1 -toolz==0.11.1 -tornado==6.1 -typing-extensions==3.10.0.0 -url-normalize==1.4.3 -urllib3==1.26.6 -Werkzeug==2.0.1 -wofs==1.6.5 -wrapt==1.12.1 -xarray==0.16.1 -xmltodict==0.12.0 -yarl==1.6.3 -zict==2.0.0 -zipp==3.5.0 \ No newline at end of file diff --git a/datacube_alchemist/worker.py b/datacube_alchemist/worker.py index dd3dee3..97b8662 100644 --- a/datacube_alchemist/worker.py +++ b/datacube_alchemist/worker.py @@ -6,7 +6,7 @@ import tempfile from datetime import datetime from pathlib import Path -from typing import Iterable, Type, Union, Mapping +from typing import Iterable, Mapping, Type, Union import cattr import datacube @@ -17,13 +17,13 @@ import yaml from datacube.model import Dataset from datacube.testutils.io import native_geobox, native_load +from datacube.utils.aws import configure_s3_access from datacube.virtual import Transformation from eodatasets3.assemble import DatasetAssembler +from odc.apps.dc_tools._docs import odc_uuid +from odc.apps.dc_tools._stac import stac_transform from odc.aws import s3_url_parse from odc.aws.queue import get_messages, get_queue -from odc.index import odc_uuid -from datacube.utils.aws import configure_s3_access -from odc.stac.transform import stac_transform from datacube_alchemist import __version__ from datacube_alchemist._utils import ( diff --git a/requirements.in b/requirements.in new file mode 100644 index 0000000..450c4e4 --- /dev/null +++ b/requirements.in @@ -0,0 +1,22 @@ +--extra-index-url="https://packages.dea.ga.gov.au" + +datacube[s3]==1.8.6 +aiobotocore[awscli, boto3]==2.0.1 +fc==1.3.6 +kombu==4.6.11 +fsspec +hdstats +nrt-predict==0.1.2 +numpy +odc-algo +odc-apps-cloud +odc-apps-dc-tools +wofs==1.6.5 +# We MUST pin xarray, because otherwise masking doesn't work properly +xarray==0.16.1 +eodatasets3 +h5py + +--no-binary rasterio +--no-binary shapely +--no-binary fiona diff --git a/requirements.txt b/requirements.txt index fe022ba..bc30116 100644 --- a/requirements.txt +++ b/requirements.txt @@ -1,23 +1,439 @@ ---extra-index-url="https://packages.dea.ga.gov.au" +# +# This file is autogenerated by pip-compile with python 3.9 +# To update, run: +# +# pip-compile --output-file=requirements.txt requirements.in +# +--extra-index-url https://packages.dea.ga.gov.au +--no-binary fiona +--no-binary rasterio +--no-binary shapely -datacube -aiobotocore[awscli]==1.3.3 +affine==2.3.0 + # via + # datacube + # eodatasets3 + # odc-algo + # rasterio +aiobotocore[awscli,boto3]==2.0.1 + # via + # -r requirements.in + # odc-cloud +aiohttp==3.8.1 + # via aiobotocore +aioitertools==0.8.0 + # via aiobotocore +aiosignal==1.2.0 + # via aiohttp +amqp==2.6.1 + # via kombu +appdirs==1.4.4 + # via requests-cache +async-timeout==4.0.2 + # via aiohttp +attrs==21.4.0 + # via + # aiohttp + # cattrs + # digitalearthau + # eodatasets3 + # jsonschema + # rasterio + # requests-cache +awscli==1.21.8 + # via aiobotocore +billiard==3.6.4.0 + # via celery +boltons==21.0.0 + # via + # digitalearthau + # eodatasets3 +boto3==1.19.8 + # via + # aiobotocore + # datacube + # odc-cloud +botocore==1.22.8 + # via + # aiobotocore + # awscli + # boto3 + # datacube + # eodatasets3 + # odc-cloud + # s3transfer +cachetools==5.0.0 + # via datacube +cattrs==1.10.0 + # via + # eodatasets3 + # requests-cache +celery==4.4.7 + # via datacube +certifi==2021.10.8 + # via + # pyproj + # rasterio + # requests +cftime==1.5.1.1 + # via netcdf4 +charset-normalizer==2.0.10 + # via + # aiohttp + # requests +ciso8601==2.2.0 + # via eodatasets3 +click==8.0.3 + # via + # click-plugins + # cligj + # datacube + # digitalearthau + # distributed + # eodatasets3 + # fc + # odc-apps-cloud + # odc-apps-dc-tools + # rasterio +click-plugins==1.1.1 + # via rasterio +cligj==0.7.2 + # via rasterio +cloudpickle==2.0.0 + # via + # dask + # datacube + # distributed +colorama==0.4.3 + # via + # awscli + # digitalearthau +dask[array]==2021.12.0 + # via + # dask-image + # datacube + # distributed + # odc-algo +dask-image==2021.12.0 + # via odc-algo +datacube[celery,s3]==1.8.6 + # via + # -r requirements.in + # digitalearthau + # eodatasets3 + # fc + # odc-algo + # odc-apps-dc-tools + # wofs +dawg==0.8.0 + # via digitalearthau +defusedxml==0.7.1 + # via eodatasets3 +deprecated==1.2.13 + # via redis +digitalearthau==20180116+437.g805c962 + # via fc +distributed==2021.12.0 + # via + # datacube + # odc-algo +docutils==0.15.2 + # via awscli +eodatasets3==0.24.0 + # via + # -r requirements.in + # digitalearthau +ephem==4.1.3 + # via wofs fc==1.3.6 -fsspec -hdstats + # via -r requirements.in +frozenlist==1.2.0 + # via + # aiohttp + # aiosignal +fsspec==2021.11.1 + # via + # -r requirements.in + # dask + # odc-apps-dc-tools +gdal==3.4.1 + # via + # digitalearthau + # nrt-predict +greenlet==1.1.2 + # via sqlalchemy +h5py==3.6.0 + # via -r requirements.in +hdstats==0.2.1 + # via -r requirements.in +heapdict==1.0.1 + # via zict +idna==3.3 + # via + # requests + # yarl +imageio==2.13.5 + # via scikit-image +jinja2==3.0.3 + # via distributed +jmespath==0.10.0 + # via + # boto3 + # botocore +joblib==1.1.0 + # via + # nrt-predict + # scikit-learn +jsonschema==4.3.3 + # via + # datacube + # eodatasets3 +kombu==4.6.11 + # via + # -r requirements.in + # celery + # datacube +lark-parser==0.12.0 + # via datacube +locket==0.2.1 + # via partd +lxml==4.7.1 + # via digitalearthau +markupsafe==2.0.1 + # via jinja2 +msgpack==1.0.3 + # via distributed +multidict==5.2.0 + # via + # aiohttp + # yarl +netcdf4==1.5.8 + # via datacube +networkx==2.6.3 + # via scikit-image nrt-predict==0.1.2 -numpy -odc-algo -odc-apps-cloud -odc-apps-dc-tools -odc-aws -odc-stac + # via -r requirements.in +numexpr==2.8.1 + # via + # fc + # odc-algo +numpy==1.22.0 + # via + # -r requirements.in + # cftime + # dask + # dask-image + # datacube + # eodatasets3 + # fc + # h5py + # hdstats + # imageio + # netcdf4 + # nrt-predict + # numexpr + # odc-algo + # pandas + # pims + # pywavelets + # rasterio + # scikit-image + # scikit-learn + # scipy + # snuggs + # tifffile + # xarray +odc-algo==0.2.2 + # via -r requirements.in +odc-apps-cloud==0.2.1 + # via -r requirements.in +odc-apps-dc-tools==0.2.3 + # via -r requirements.in +odc-cloud[async]==0.2.1 + # via + # odc-apps-cloud + # odc-apps-dc-tools +odc-io==0.2.1 + # via + # odc-apps-cloud + # odc-apps-dc-tools +packaging==21.3 + # via + # dask + # numexpr + # redis + # scikit-image +pandas==1.3.5 + # via + # datacube + # fc + # xarray +partd==1.2.0 + # via dask +pillow==9.0.0 + # via + # imageio + # scikit-image +pims==0.5 + # via dask-image +psutil==5.9.0 + # via + # distributed + # nrt-predict +psycopg2==2.9.3 + # via datacube +pyasn1==0.4.8 + # via rsa +pydash==5.1.0 + # via digitalearthau +pyparsing==3.0.6 + # via + # packaging + # snuggs +pyproj==3.3.0 + # via + # datacube + # eodatasets3 +pyrsistent==0.18.0 + # via jsonschema +pystac==1.2.0 + # via + # eodatasets3 + # odc-apps-dc-tools + # pystac-client + # rio-stac +pystac-client==0.3.1 + # via odc-apps-dc-tools +python-dateutil==2.8.2 + # via + # botocore + # datacube + # digitalearthau + # pandas + # pystac +python-rapidjson==1.5 + # via eodatasets3 +pytz==2021.3 + # via + # celery + # pandas +pywavelets==1.2.0 + # via scikit-image +pyyaml==5.4.1 + # via + # awscli + # dask + # datacube + # distributed + # nrt-predict + # odc-apps-dc-tools +rasterio==1.2.10 + # via + # datacube + # eodatasets3 + # odc-algo + # rio-stac +redis==4.1.0 + # via datacube +requests==2.27.1 + # via + # nrt-predict + # pystac-client + # requests-cache +requests-cache==0.9.0 + # via eodatasets3 +rio-stac==0.3.2 + # via odc-apps-dc-tools +rsa==4.7.2 + # via awscli +ruamel.yaml==0.17.20 + # via eodatasets3 +ruamel.yaml.clib==0.2.6 + # via ruamel.yaml +s3transfer==0.5.0 + # via + # awscli + # boto3 +scikit-image==0.19.1 + # via + # nrt-predict + # odc-algo +scikit-learn==1.0.2 + # via nrt-predict +scipy==1.7.3 + # via + # dask-image + # eodatasets3 + # hdstats + # scikit-image + # scikit-learn + # wofs +shapely==1.8.0 + # via + # datacube + # eodatasets3 +six==1.16.0 + # via + # pims + # python-dateutil + # slicerator + # url-normalize +slicerator==1.0.0 + # via pims +snuggs==1.4.7 + # via rasterio +sortedcontainers==2.4.0 + # via distributed +sqlalchemy==1.4.29 + # via datacube +structlog==21.5.0 + # via + # digitalearthau + # eodatasets3 +tblib==1.7.0 + # via distributed +threadpoolctl==3.0.0 + # via scikit-learn +tifffile==2021.11.2 + # via scikit-image +toolz==0.11.2 + # via + # dask + # datacube + # distributed + # odc-algo + # odc-apps-dc-tools + # partd +tornado==6.1 + # via distributed +url-normalize==1.4.3 + # via requests-cache +urllib3==1.26.7 + # via + # botocore + # requests + # requests-cache +vine==1.3.0 + # via + # amqp + # celery wofs==1.6.5 -# We MUST pin xarray, because otherwise masking doesn't work properly + # via -r requirements.in +wrapt==1.13.3 + # via + # aiobotocore + # deprecated xarray==0.16.1 -eodatasets3 -h5py + # via + # -r requirements.in + # datacube + # eodatasets3 + # odc-algo + # wofs +yarl==1.7.2 + # via aiohttp +zict==2.0.0 + # via distributed ---no-binary rasterio ---no-binary shapely ---no-binary fiona +# The following packages are considered to be unsafe in a requirements file: +# setuptools diff --git a/setup.cfg b/setup.cfg index 11592a8..0f64b6c 100644 --- a/setup.cfg +++ b/setup.cfg @@ -25,18 +25,17 @@ install_requires = attrs>=18.1 cattrs==1.0.0;python_version<'3.7' structlog - boto3 dask distributed fsspec s3fs jsonschema requests - odc-cloud odc-algo - odc-stac + odc-apps-dc-tools + boto3 awscli - xarray<=0.16.1 + xarray==0.16.1 [options.packages.find] exclude = tests