Refactor Docker requirements to use pip-compile and simplify things (#…

…131)

* Refactor Docker requirements to use pip-compile and simplify things

* Formatting

* Fix editable installs

Dockerfile

@@ -1,4 +1,4 @@
-FROM osgeo/gdal:ubuntu-small-3.3.3
+FROM osgeo/gdal:ubuntu-small-3.4.1
 
 ENV DEBIAN_FRONTEND=noninteractive \
     LC_ALL=C.UTF-8 \
@@ -29,8 +29,8 @@ RUN echo "Environment is: $ENVIRONMENT"
 
 # Pip installation
 RUN mkdir -p /conf
-COPY requirements.txt constraints.txt /conf/
-RUN pip install -r /conf/requirements.txt -c /conf/constraints.txt
+COPY requirements.txt /conf/
+RUN pip install -r /conf/requirements.txt
 
 # Set up a nice workdir and add the live code
 ENV APPDIR=/code
@@ -41,11 +41,12 @@ ADD . $APPDIR
 # These ENVIRONMENT flags make this a bit complex, but basically, if we are in dev
 # then we want to link the source (with the -e flag) and if we're in prod, we
 # want to delete the stuff in the /code folder to keep it simple.
+# no-use-pep517 because of this https://github.com/pypa/pip/issues/7953
 RUN if [ "$ENVIRONMENT" = "deployment" ] ; then\
-        pip install -c /code/constraints.txt . ; \
-        rm -rf /code/* ; \
+        pip install . ; \
+        rm -rf $APPDIR/* ; \
     else \
-        pip install -c /code/constraints.txt --editable .[$ENVIRONMENT] ; \
+        pip install --no-use-pep517 --editable ".[$ENVIRONMENT]" ; \
     fi
 
 RUN pip freeze

datacube_alchemist/worker.py

@@ -6,7 +6,7 @@
 import tempfile
 from datetime import datetime
 from pathlib import Path
-from typing import Iterable, Type, Union, Mapping
+from typing import Iterable, Mapping, Type, Union
 
 import cattr
 import datacube
@@ -17,13 +17,13 @@
 import yaml
 from datacube.model import Dataset
 from datacube.testutils.io import native_geobox, native_load
+from datacube.utils.aws import configure_s3_access
 from datacube.virtual import Transformation
 from eodatasets3.assemble import DatasetAssembler
+from odc.apps.dc_tools._docs import odc_uuid
+from odc.apps.dc_tools._stac import stac_transform
 from odc.aws import s3_url_parse
 from odc.aws.queue import get_messages, get_queue
-from odc.index import odc_uuid
-from datacube.utils.aws import configure_s3_access
-from odc.stac.transform import stac_transform
 
 from datacube_alchemist import __version__
 from datacube_alchemist._utils import (

requirements.in

@@ -0,0 +1,22 @@
+--extra-index-url="https://packages.dea.ga.gov.au"
+
+datacube[s3]==1.8.6
+aiobotocore[awscli, boto3]==2.0.1
+fc==1.3.6
+kombu==4.6.11
+fsspec
+hdstats
+nrt-predict==0.1.2
+numpy
+odc-algo
+odc-apps-cloud
+odc-apps-dc-tools
+wofs==1.6.5
+# We MUST pin xarray, because otherwise masking doesn't work properly
+xarray==0.16.1
+eodatasets3
+h5py
+
+--no-binary rasterio
+--no-binary shapely
+--no-binary fiona