diff --git a/deployment/bin/deploy b/deployment/bin/deploy index d3a2b386..af776d1f 100755 --- a/deployment/bin/deploy +++ b/deployment/bin/deploy @@ -60,9 +60,6 @@ fi require_env "IMAGE_TAG" require_env "GIT_COMMIT" -require_env "ARM_CLIENT_ID" -require_env "ARM_TENANT_ID" -require_env "ARM_USE_OIDC" # Directory for rendered values and templates CONF_DIR='/opt/conf' diff --git a/docker-compose.yml b/docker-compose.yml index 63790237..31cef50f 100644 --- a/docker-compose.yml +++ b/docker-compose.yml @@ -101,7 +101,7 @@ services: azurite: container_name: pcapis-azurite - image: mcr.microsoft.com/azure-storage/azurite:3.29.0 + image: mcr.microsoft.com/azure-storage/azurite:3.30.0 hostname: azurite command: "azurite --silent --blobHost 0.0.0.0 --queueHost 0.0.0.0 --tableHost 0.0.0.0 -l /workspace" diff --git a/pccommon/requirements.txt b/pccommon/requirements.txt index d4992e27..17770920 100644 --- a/pccommon/requirements.txt +++ b/pccommon/requirements.txt @@ -13,12 +13,14 @@ azure-core==1.30.1 # azure-data-tables # azure-identity # azure-storage-blob - # msrest -azure-data-tables==12.4.0 - # via pccommon (pccommon/setup.py) -azure-identity==1.7.1 + # opencensus-ext-azure +azure-data-tables==12.5.0 # via pccommon (pccommon/setup.py) -azure-storage-blob==12.19.1 +azure-identity==1.16.0 + # via + # opencensus-ext-azure + # pccommon (pccommon/setup.py) +azure-storage-blob==12.20.0 # via pccommon (pccommon/setup.py) beautifulsoup4==4.12.3 # via html-sanitizer @@ -27,9 +29,7 @@ cachetools==5.3.3 # google-auth # pccommon (pccommon/setup.py) certifi==2024.2.2 - # via - # msrest - # requests + # via requests cffi==1.16.0 # via cryptography charset-normalizer==3.3.2 @@ -40,8 +40,6 @@ cryptography==42.0.5 # azure-storage-blob # msal # pyjwt -deprecated==1.2.14 - # via redis exceptiongroup==1.2.0 # via anyio fastapi==0.90.1 @@ -52,47 +50,46 @@ google-auth==2.29.0 # via google-api-core googleapis-common-protos==1.63.0 # via google-api-core -html-sanitizer==2.4.0 +html-sanitizer==2.4.4 # via pccommon (pccommon/setup.py) idna==3.7 # via # anyio # pccommon (pccommon/setup.py) # requests + # yarl isodate==0.6.1 # via + # azure-data-tables # azure-storage-blob - # msrest -lxml[html-clean]==5.2.1 +lxml==5.2.1 # via # html-sanitizer # lxml-html-clean lxml-html-clean==0.1.0 - # via pccommon (pccommon/setup.py) + # via + # html-sanitizer + # pccommon (pccommon/setup.py) msal==1.28.0 # via # azure-identity # msal-extensions msal-extensions==0.3.1 # via azure-identity -msrest==0.7.1 - # via azure-data-tables -oauthlib==3.2.2 - # via requests-oauthlib +multidict==6.0.5 + # via yarl opencensus==0.11.4 # via # opencensus-ext-azure # opencensus-ext-logging opencensus-context==0.1.3 # via opencensus -opencensus-ext-azure==1.0.8 +opencensus-ext-azure==1.1.13 # via pccommon (pccommon/setup.py) -opencensus-ext-logging==0.1.0 +opencensus-ext-logging==0.1.1 # via pccommon (pccommon/setup.py) -orjson==3.9.15 +orjson==3.10.4 # via pccommon (pccommon/setup.py) -packaging==24.0 - # via redis portalocker==2.8.2 # via msal-extensions proto-plus==1.23.0 @@ -120,24 +117,19 @@ pyhumps==3.5.3 # via pccommon (pccommon/setup.py) pyjwt[crypto]==2.8.0 # via msal -redis==4.2.0rc1 +redis==4.6.0 # via pccommon (pccommon/setup.py) requests==2.31.0 # via # azure-core # google-api-core # msal - # msrest # opencensus-ext-azure - # requests-oauthlib -requests-oauthlib==2.0.0 - # via msrest rsa==4.9 # via google-auth six==1.16.0 # via # azure-core - # azure-identity # isodate # opencensus sniffio==1.3.1 @@ -154,13 +146,13 @@ typing-extensions==4.10.0 # via # anyio # azure-core + # azure-data-tables # azure-storage-blob # pydantic - # redis # starlette urllib3==2.2.1 # via # pccommon (pccommon/setup.py) # requests -wrapt==1.16.0 - # via deprecated +yarl==1.9.4 + # via azure-data-tables diff --git a/pccommon/setup.py b/pccommon/setup.py index cbcc29d4..9e12dcbe 100644 --- a/pccommon/setup.py +++ b/pccommon/setup.py @@ -6,26 +6,26 @@ inst_reqs = [ "fastapi==0.90.1", "starlette>=0.22.0,<0.23.0", - "opencensus-ext-azure==1.0.8", - "opencensus-ext-logging==0.1.0", - "orjson>=3.9.15", - "azure-identity==1.7.1", - "azure-data-tables==12.4.0", - "azure-storage-blob>=12.19.1", - "pydantic>=1.9, <2.0.0", + "opencensus-ext-azure==1.1.13", + "opencensus-ext-logging==0.1.1", + "orjson>=3.10.4", + "azure-identity==1.16.0", + "azure-data-tables==12.5.0", + "azure-storage-blob>=12.20.0", + "pydantic>=1.10, <2.0.0", "cachetools~=5.3", "types-cachetools==4.2.9", "pyhumps==3.5.3", - "redis==4.2.0-rc1", + "redis==4.6.0", "idna>=3.7.0", - "html-sanitizer==2.4", + "html-sanitizer==2.4.4", # Soon available as lxml[html_clean] "lxml_html_clean==0.1.0", "urllib3>=1.26.18", ] extra_reqs = { - "test": ["pytest", "pytest-asyncio", "types-redis"], + "test": ["pytest", "pytest-asyncio", "types-redis", "types-requests"], "dev": ["pytest", "pytest-asyncio", "types-redis"], } diff --git a/pcstac/Dockerfile b/pcstac/Dockerfile index a206280c..aa800380 100644 --- a/pcstac/Dockerfile +++ b/pcstac/Dockerfile @@ -1,7 +1,7 @@ -FROM python:3.9-slim +FROM mcr.microsoft.com/cbl-mariner/base/python:3.9 -RUN apt-get update && \ - apt-get install -y build-essential git +RUN tdnf install -y ca-certificates build-essential \ + && tdnf clean all ENV CURL_CA_BUNDLE /etc/ssl/certs/ca-certificates.crt diff --git a/pcstac/requirements-server.txt b/pcstac/requirements-server.txt index 99f95af8..e4113bfa 100644 --- a/pcstac/requirements-server.txt +++ b/pcstac/requirements-server.txt @@ -56,7 +56,7 @@ iso8601==1.1.0 # via stac-fastapi-types lark==0.12.0 # via pygeofilter -orjson==3.9.15 +orjson==3.10.4 # via # pcstac (pcstac/setup.py) # pypgstac @@ -87,7 +87,7 @@ pypgstac[psycopg]==0.7.10 # via # pcstac (pcstac/setup.py) # stac-fastapi-pgstac -pystac==1.9.0 +pystac==1.10.1 # via # pcstac (pcstac/setup.py) # stac-fastapi-types diff --git a/pcstac/setup.py b/pcstac/setup.py index 70e60308..37839d38 100644 --- a/pcstac/setup.py +++ b/pcstac/setup.py @@ -9,10 +9,10 @@ "stac-fastapi.extensions==2.4.8", "stac-fastapi.pgstac==2.4.9", "stac-fastapi.types==2.4.8", - "orjson>=3.9.15", + "orjson==3.10.4", # Required due to some imports related to pypgstac CLI usage in startup script "pypgstac[psycopg]~=0.7", - "pystac>=1.9", + "pystac==1.10.1", ] extra_reqs = { diff --git a/pctiler/Dockerfile b/pctiler/Dockerfile index caca68f5..10deaa29 100644 --- a/pctiler/Dockerfile +++ b/pctiler/Dockerfile @@ -1,4 +1,7 @@ -FROM python:3.9-slim +FROM mcr.microsoft.com/cbl-mariner/base/python:3.9 + +RUN tdnf install -y ca-certificates build-essential \ + && tdnf clean all WORKDIR /opt/src @@ -7,14 +10,14 @@ COPY pctiler /opt/src/pctiler # Install the local modules in the new environment RUN --mount=type=cache,target=/root/.cache \ - /bin/sh -c "python -m pip install -U 'setuptools>=65.5.1'" + /bin/sh -c "python3 -m pip install -U 'setuptools>=65.5.1'" # The order of these pip installs is important :( RUN --mount=type=cache,target=/root/.cache \ - /bin/sh -c "python -m pip install -r ./pccommon/requirements.txt" + /bin/sh -c "python3 -m pip install -r ./pccommon/requirements.txt" RUN --mount=type=cache,target=/root/.cache \ - /bin/sh -c "python -m pip install -r ./pctiler/requirements-server.txt" + /bin/sh -c "python3 -m pip install -r ./pctiler/requirements-server.txt" RUN --mount=type=cache,target=/root/.cache \ - /bin/sh -c "python -m pip install --no-deps -e ./pccommon -e ./pctiler[server]" + /bin/sh -c "python3 -m pip install --no-deps -e ./pccommon -e ./pctiler[server]" # GDAL config ENV GDAL_CACHEMAX 200 diff --git a/pctiler/Dockerfile.dev b/pctiler/Dockerfile.dev index 12f95fbe..39fad6e8 100644 --- a/pctiler/Dockerfile.dev +++ b/pctiler/Dockerfile.dev @@ -1,6 +1,6 @@ FROM pc-apis-tiler -RUN curl -sL https://aka.ms/InstallAzureCLIDeb | bash +RUN tdnf install azure-cli -y COPY requirements-dev.txt requirements-dev.txt diff --git a/pctiler/requirements-dev.txt b/pctiler/requirements-dev.txt index d3b5ff7d..f4d2b734 100644 --- a/pctiler/requirements-dev.txt +++ b/pctiler/requirements-dev.txt @@ -18,11 +18,11 @@ attrs==23.2.0 # morecantile # rasterio # rio-tiler -boto3==1.34.71 +boto3==1.34.123 # via # pctiler (pctiler/setup.py) # rio-tiler -botocore==1.34.71 +botocore==1.34.123 # via # boto3 # pctiler (pctiler/setup.py) @@ -62,6 +62,8 @@ cogeo-mosaic==5.0.0 # via titiler-mosaic color-operations==0.1.3 # via rio-tiler +contourpy==1.2.1 + # via matplotlib cycler==0.12.1 # via matplotlib exceptiongroup==1.2.0 @@ -70,6 +72,8 @@ fastapi==0.91.0 # via # titiler-core # titiler-pgstac +fonttools==4.53.0 + # via matplotlib geojson-pydantic==0.4.2 # via # pctiler (pctiler/setup.py) @@ -90,7 +94,11 @@ idna==3.7 # httpx # pctiler (pctiler/setup.py) # requests -jinja2==3.0.3 +importlib-metadata==7.1.0 + # via rasterio +importlib-resources==6.4.0 + # via matplotlib +jinja2==3.1.4 # via # pctiler (pctiler/setup.py) # titiler-core @@ -102,7 +110,7 @@ kiwisolver==1.4.5 # via matplotlib markupsafe==2.1.5 # via jinja2 -matplotlib==3.4.3 +matplotlib==3.9.0 # via pctiler (pctiler/setup.py) mercantile==1.2.1 # via supermercado @@ -115,6 +123,7 @@ numexpr==2.9.0 numpy==1.26.4 # via # color-operations + # contourpy # matplotlib # numexpr # rasterio @@ -123,8 +132,10 @@ numpy==1.26.4 # snuggs # supermercado # titiler-core -orjson==3.9.15 +orjson==3.10.4 # via pctiler (pctiler/setup.py) +packaging==24.1 + # via matplotlib pillow==10.3.0 # via # matplotlib @@ -154,7 +165,7 @@ pyparsing==3.1.2 # snuggs pyproj==3.6.1 # via morecantile -pystac==1.7.1 +pystac==1.10.1 # via # pctiler (pctiler/setup.py) # planetary-computer @@ -172,7 +183,7 @@ python-dotenv==1.0.1 # via pydantic pytz==2024.1 # via planetary-computer -rasterio==1.3.9 +rasterio==1.3.10 # via # cogeo-mosaic # pctiler (pctiler/setup.py) @@ -235,6 +246,10 @@ urllib3==1.26.18 # via # botocore # requests +zipp==3.19.2 + # via + # importlib-metadata + # importlib-resources # The following packages are considered to be unsafe in a requirements file: # setuptools diff --git a/pctiler/requirements-server.txt b/pctiler/requirements-server.txt index c16bc70e..e7f828e8 100644 --- a/pctiler/requirements-server.txt +++ b/pctiler/requirements-server.txt @@ -21,11 +21,11 @@ attrs==23.2.0 # morecantile # rasterio # rio-tiler -boto3==1.34.71 +boto3==1.34.123 # via # pctiler (pctiler/setup.py) # rio-tiler -botocore==1.34.71 +botocore==1.34.123 # via # boto3 # pctiler (pctiler/setup.py) @@ -66,6 +66,8 @@ cogeo-mosaic==5.0.0 # via titiler-mosaic color-operations==0.1.3 # via rio-tiler +contourpy==1.2.1 + # via matplotlib cycler==0.12.1 # via matplotlib exceptiongroup==1.2.0 @@ -74,6 +76,8 @@ fastapi==0.91.0 # via # titiler-core # titiler-pgstac +fonttools==4.53.0 + # via matplotlib geojson-pydantic==0.4.2 # via # pctiler (pctiler/setup.py) @@ -98,7 +102,11 @@ idna==3.7 # httpx # pctiler (pctiler/setup.py) # requests -jinja2==3.0.3 +importlib-metadata==7.1.0 + # via rasterio +importlib-resources==6.4.0 + # via matplotlib +jinja2==3.1.4 # via # pctiler (pctiler/setup.py) # titiler-core @@ -110,7 +118,7 @@ kiwisolver==1.4.5 # via matplotlib markupsafe==2.1.5 # via jinja2 -matplotlib==3.4.3 +matplotlib==3.9.0 # via pctiler (pctiler/setup.py) mercantile==1.2.1 # via supermercado @@ -123,6 +131,7 @@ numexpr==2.9.0 numpy==1.26.4 # via # color-operations + # contourpy # matplotlib # numexpr # rasterio @@ -131,8 +140,10 @@ numpy==1.26.4 # snuggs # supermercado # titiler-core -orjson==3.9.15 +orjson==3.10.4 # via pctiler (pctiler/setup.py) +packaging==24.1 + # via matplotlib pillow==10.3.0 # via # matplotlib @@ -162,7 +173,7 @@ pyparsing==3.1.2 # snuggs pyproj==3.6.1 # via morecantile -pystac==1.7.1 +pystac==1.10.1 # via # pctiler (pctiler/setup.py) # planetary-computer @@ -184,7 +195,7 @@ pytz==2024.1 # via planetary-computer pyyaml==6.0.1 # via uvicorn -rasterio==1.3.9 +rasterio==1.3.10 # via # cogeo-mosaic # pctiler (pctiler/setup.py) @@ -251,6 +262,10 @@ watchgod==0.8.2 # via uvicorn websockets==12.0 # via uvicorn +zipp==3.19.2 + # via + # importlib-metadata + # importlib-resources # The following packages are considered to be unsafe in a requirements file: # setuptools diff --git a/pctiler/setup.py b/pctiler/setup.py index e2a03567..824f7d84 100644 --- a/pctiler/setup.py +++ b/pctiler/setup.py @@ -6,15 +6,15 @@ # Runtime requirements, see environment.yaml inst_reqs: List[str] = [ "geojson-pydantic==0.4.2", - "jinja2==3.0.3", - "pystac==1.7.1", + "jinja2==3.1.4", + "pystac==1.10.1", "planetary-computer==0.4.9", - "rasterio==1.3.9", + "rasterio==1.3.10", "titiler.core==0.10.2", "titiler.mosaic==0.10.2", "pillow==10.3.0", - "boto3==1.34.71", - "botocore==1.34.71", + "boto3==1.34.123", + "botocore==1.34.123", "pydantic==1.10.14", "idna>=3.7.0", # titiler-pgstac @@ -22,9 +22,9 @@ "titiler.pgstac==0.2.4", # colormap dependencies - "matplotlib==3.4.3", + "matplotlib==3.9.0", - "orjson==3.9.15", + "orjson==3.10.4", "importlib_resources>=1.1.0;python_version<'3.9'", ] diff --git a/scripts/bin/test-common b/scripts/bin/test-common index 8b1648de..d0bb5a1d 100755 --- a/scripts/bin/test-common +++ b/scripts/bin/test-common @@ -28,5 +28,5 @@ if [ "${BASH_SOURCE[0]}" = "${0}" ]; then flake8 pccommon/pccommon pccommon/tests echo "Running unit tests for common..." - python -m pytest pccommon/tests + python3 -m pytest pccommon/tests fi diff --git a/scripts/bin/test-stac b/scripts/bin/test-stac index 1e5b0d26..41982cc3 100755 --- a/scripts/bin/test-stac +++ b/scripts/bin/test-stac @@ -28,6 +28,6 @@ if [ "${BASH_SOURCE[0]}" = "${0}" ]; then flake8 pcstac/pcstac pcstac/tests echo "Running unit tests for stac..." - python -m pytest pcstac/tests + python3 -m pytest pcstac/tests fi diff --git a/scripts/bin/test-tiler b/scripts/bin/test-tiler index 5bc76aba..c3a6372f 100755 --- a/scripts/bin/test-tiler +++ b/scripts/bin/test-tiler @@ -44,6 +44,6 @@ if [ "${BASH_SOURCE[0]}" = "${0}" ]; then flake8 pctiler/pctiler pctiler/tests echo "Running unit tests for tiler..." - python -m pytest pctiler/tests ${INTEGRATION:+$INTEGRATION} + python3 -m pytest pctiler/tests ${INTEGRATION:+$INTEGRATION} fi diff --git a/scripts/setup b/scripts/setup index 88804290..31bcc3c1 100755 --- a/scripts/setup +++ b/scripts/setup @@ -31,7 +31,7 @@ if [ "${BASH_SOURCE[0]}" = "${0}" ]; then -f docker-compose.yml \ run --rm \ stac \ - python /opt/src/pcstac/tests/loadtestdata.py + python3 /opt/src/pcstac/tests/loadtestdata.py echo "Setting up azurite..." @@ -40,7 +40,7 @@ if [ "${BASH_SOURCE[0]}" = "${0}" ]; then -f docker-compose.dev.yml \ run --rm \ stac-dev \ - python /opt/src/scripts/bin/setup_azurite.py + python3 /opt/src/scripts/bin/setup_azurite.py echo "Done."