Merge pull request #103 from fredliporace/dev

Update to python3.9; synth canary runtime update

.github/workflows/ci.yml

@@ -12,7 +12,7 @@ jobs:
     runs-on: ubuntu-20.04
     strategy:
       matrix:
-        python-version: [3.7.9]
+        python-version: [3.9.13]
 
     steps:
 

.pre-commit-config.yaml

@@ -3,13 +3,13 @@ repos:
     rev: 19.10b0
     hooks:
       - id: black
-        language_version: python3.7
+        language_version: python3.9
         args: ["--safe"]
         additional_dependencies:
           # To avoid
           # ImportError: cannot import name '_unicodefun' from 'click'
-          #   (/home/liporace/.cache/pre-commit/repoesurte4p/py_env-python3.7/
-          #   lib/python3.7/site-packages/click/__init__.py)
+          #   (/home/liporace/.cache/pre-commit/repoesurte4p/py_env-python3.9/
+          #   lib/python3.9/site-packages/click/__init__.py)
           - "click==7.1.2"
 
   - repo: https://github.com/PyCQA/isort
@@ -19,7 +19,7 @@ repos:
     hooks:
       - id: isort
         args: ["--profile", "black"]
-        language_version: python3.7
+        language_version: python3.9
 
   #- repo: https://github.com/PyCQA/pylint
   #  rev: pylint-2.6.0
@@ -32,7 +32,7 @@ repos:
         # Reference to .pylintrc to make sure it is being used.
         entry: pylint --rcfile=.pylintrc # added for local repo option
         # This is used for remote repo
-        #language_version: python3.7
+        #language_version: python3.9
         language: python
         types: [python]
         args: [
@@ -48,7 +48,7 @@ repos:
   #   rev: 5.1.1
   #   hooks:
   #     - id: pydocstyle
-  #       language_version: python3.7
+  #       language_version: python3.9
   #       args: [
   #           # Check for docstring presence only
   #           "--select=D1",
@@ -57,9 +57,10 @@ repos:
   #         ]
 
   - repo: https://github.com/pre-commit/mirrors-mypy
-    rev: v0.770
+    # rev: v0.770
+    rev: v0.790
     hooks:
       - id: mypy
-        language_version: python3.7
+        language_version: python3.9
         #args are defined in mypy.ini
         #args: ["--no-strict-optional", "--ignore-missing-imports"]

README.md

@@ -215,10 +215,12 @@ $ pytest
 [act](https://github.com/nektos/act) may be used to test github actions locally. At the project's root directory:
 
 ```bash
-$ act -j tests
-$ act -r -j tests # To keep docker container's state
+$ act --env-file < /dev/null -j tests
+$ act --env-filr < /dev/null -r -j tests # To keep docker container's state
 ```
 
+The `env-file` option is used to make sure that the `.env` file is used only for CDK and not docker.
+
 # References
 
 The mechanism to include new items into the archive as soon as they are ingested is described in [this AWS blog post](https://aws.amazon.com/blogs/publicsector/keeping-a-spatiotemporal-asset-catalog-stac-up-to-date-with-sns-sqs/).

cbers2stac/dlq_replay/code.py

@@ -26,7 +26,7 @@ def gather_messages(queue):  # pylint: disable=missing-docstring
 
 def send_messages(messages, queue):  # pylint: disable=missing-docstring
     entries = [
-        dict(Id=str(i + 1), MessageBody=message.body)
+        {"Id": str(i + 1), "MessageBody": message.body}
         for i, message in enumerate(messages)
     ]
 

cbers2stac/process_new_scene_queue/code.py

@@ -253,7 +253,7 @@ def catalog_update_request(table_name: str, stac_item_key: str):
     )
 
 
-def process_trigger(
+def process_trigger(  # pylint: disable=too-many-arguments
     *,
     stac_bucket: str,
     cog_pds_meta_pds: Dict[str, str],
@@ -319,7 +319,7 @@ def process_trigger(
                 )
 
 
-def process_queue(
+def process_queue(  # pylint: disable=too-many-arguments
     *,
     stac_bucket: str,
     cog_pds_meta_pds: Dict[str, str],

constraints.txt

@@ -1,75 +1,71 @@
-astroid==2.11.7
-attrs==21.4.0
-aws-cdk-lib==2.111.0
-aws-cdk.asset-awscli-v1==2.2.201
+annotated-types==0.6.0
+astroid==3.0.3
+attrs==23.2.0
+aws-cdk-lib==2.129.0
+aws-cdk.asset-awscli-v1==2.2.202
 aws-cdk.asset-kubectl-v20==2.1.2
 aws-cdk.asset-node-proxy-agent-v6==2.0.1
 aws-requests-auth==0.4.3
-awscli==1.25.48
-awscli-local==0.20
-boto3==1.24.48
-botocore==1.27.48
-cattrs==22.1.0
-certifi==2022.6.15
-cfgv==3.3.1
-charset-normalizer==2.1.0
+awscli==1.32.47
+awscli-local==0.22.0
+boto3==1.34.47
+botocore==1.34.47
+cattrs==23.2.3
+certifi==2024.2.2
+cfgv==3.4.0
+charset-normalizer==3.3.2
 colorama==0.4.4
 constructs==10.3.0
-coverage==6.4.3
+coverage==7.4.2
 decorator==5.1.1
-dill==0.3.5.1
-distlib==0.3.5
-docker==5.0.3
+dill==0.3.8
+distlib==0.3.8
+docker==7.0.0
 docutils==0.16
 elasticsearch==7.13.4
-elasticsearch-dsl==7.4.0
-exceptiongroup==1.0.0rc8
-filelock==3.8.0
-identify==2.5.3
-idna==3.3
+elasticsearch-dsl==7.4.1
+exceptiongroup==1.2.0
+filelock==3.13.1
+identify==2.5.35
+idna==3.6
 importlib-metadata==1.7.0
-importlib-resources==5.12.0
-iniconfig==1.1.1
-isort==5.10.1
+importlib-resources==6.1.1
+iniconfig==2.0.0
+isort==5.13.2
 jmespath==1.0.1
-jsii==1.91.0
+jsii==1.94.0
 jsonschema==3.2.0
-lazy-object-proxy==1.7.1
-localstack-client==1.36
+localstack-client==2.5
 mccabe==0.7.0
-nodeenv==1.7.0
-packaging==21.3
-platformdirs==2.5.2
-pluggy==1.0.0
-pre-commit==2.20.0
+nodeenv==1.8.0
+packaging==23.2
+platformdirs==4.2.0
+pluggy==1.4.0
+pre-commit==3.6.2
 publication==0.0.3
 py==1.11.0
-pyasn1==0.4.8
+pyasn1==0.5.1
 pydantic==1.9.1
-pylint==2.14.5
-pyparsing==3.0.9
-pyrsistent==0.18.1
+pydantic_core==2.16.2
+pylint==3.0.3
+pyrsistent==0.20.0
 pystac==0.5.6
-pytest==7.1.2
-pytest-cov==3.0.0
+pytest==8.0.1
+pytest-cov==4.1.0
 python-dateutil==2.8.2
-python-dotenv==0.20.0
-PyYAML==5.4.1
-requests==2.28.1
+python-dotenv==1.0.1
+PyYAML==6.0.1
+requests==2.31.0
 retry==0.9.2
 rsa==4.7.2
-s3transfer==0.6.0
+s3transfer==0.10.0
 six==1.16.0
-toml==0.10.2
 tomli==2.0.1
-tomlkit==0.11.3
-tox==3.25.1
-typed-ast==1.5.4
+tomlkit==0.12.3
+tox==3.28.0
 typeguard==2.13.3
-typing_extensions==4.3.0
-urllib3==1.26.11
+typing_extensions==4.9.0
+urllib3==1.26.18
 utm==0.7.0
-virtualenv==20.16.2
-websocket-client==1.3.3
-wrapt==1.14.1
-zipp==3.8.1
+virtualenv==20.25.1
+zipp==3.17.0

pip-on-lambdas.sh

@@ -7,4 +7,6 @@ echo $PATH
 export PATH=$(p=$(echo $PATH | tr ":" "\n" | grep -v "\"" | tr "\n" ":"); echo ${p%:})
 echo $PATH
 
+find ./cbers2stac -type f -name 'requirements.txt' -execdir sh -c "git ls-files --others | xargs rm" \;
+find ./cbers2stac -type f -name 'requirements.txt' -execdir sh -c "git ls-files --others --directory | xargs rm -rf" \;
 find ./cbers2stac -type f -name 'requirements.txt' -execdir pip install -r {} -t .  \;

setup.py

@@ -34,7 +34,11 @@
         # Used in process_new_scene_queue lambda.
         "utm",
     ],
-    "deploy": ["pydantic[dotenv]", "aws-cdk-lib>=2.0.0", "constructs>=10.0.0",],
+    "deploy": [
+        "pydantic[dotenv]<=1.9.1",
+        "aws-cdk-lib>=2.129.0",
+        "constructs>=10.0.0",
+    ],
 }
 
 ENTRY_POINTS = """
@@ -45,11 +49,11 @@
 setup(
     name="cbers-2-stac",
     version="0.0.0",
-    description=u"",
+    description="STAC service for CBERS and Amazonia data on AWS",
     long_description=long_description,
     long_description_content_type="text/markdown",
-    python_requires="==3.7.9",
-    author=u"Frederico Liporace (AMS Kepler)",
+    python_requires="==3.9.13",
+    author="Frederico Liporace (AMS Kepler)",
     author_email="[email protected]",
     url="https://github.com/fredliporace/cbers-2-stac",
     packages=find_packages(exclude=["tests*"]),

stack/app.py

@@ -38,6 +38,7 @@ class CBERS2STACStack(Stack):
     """CBERS2STACStack"""
 
     lambdas_env_: Dict[str, str] = {}
+    python_runtime_ = aws_lambda.Runtime.PYTHON_3_9
 
     def create_queue(self, **kwargs: Any) -> sqs.Queue:
         """
@@ -364,7 +365,7 @@ def create_all_layers(self) -> None:
             self,
             "common_layer",
             code=common_layer_asset,
-            compatible_runtimes=[aws_lambda.Runtime.PYTHON_3_7],
+            compatible_runtimes=[self.python_runtime_],
             description="Common utils",
         )
 
@@ -378,7 +379,7 @@ def create_all_lambdas(self) -> None:
                 path="cbers2stac/process_new_scene_queue", exclude=["*~"]
             ),
             handler="code.handler",
-            runtime=aws_lambda.Runtime.PYTHON_3_7,
+            runtime=self.python_runtime_,
             environment={
                 **self.lambdas_env_,
                 **{
@@ -406,7 +407,7 @@ def create_all_lambdas(self) -> None:
                 path="cbers2stac/generate_catalog_levels_to_be_updated", exclude=["*~"]
             ),
             handler="code.handler",
-            runtime=aws_lambda.Runtime.PYTHON_3_7,
+            runtime=self.python_runtime_,
             environment={
                 **self.lambdas_env_,
                 **{
@@ -427,7 +428,7 @@ def create_all_lambdas(self) -> None:
                 path="cbers2stac/update_catalog_tree", exclude=["*~"]
             ),
             handler="code.trigger_handler",
-            runtime=aws_lambda.Runtime.PYTHON_3_7,
+            runtime=self.python_runtime_,
             environment={**self.lambdas_env_,},
             timeout=Duration.seconds(55),
             dead_letter_queue=self.queues_["dead_letter_queue"],
@@ -446,7 +447,7 @@ def create_all_lambdas(self) -> None:
                 path="cbers2stac/populate_reconcile_queue", exclude=["*~"]
             ),
             handler="code.handler",
-            runtime=aws_lambda.Runtime.PYTHON_3_7,
+            runtime=self.python_runtime_,
             environment={
                 **self.lambdas_env_,
                 **{"RECONCILE_QUEUE": self.queues_["reconcile_queue"].queue_url},
@@ -463,7 +464,7 @@ def create_all_lambdas(self) -> None:
                 path="cbers2stac/consume_reconcile_queue", exclude=["*~"]
             ),
             handler="code.handler",
-            runtime=aws_lambda.Runtime.PYTHON_3_7,
+            runtime=self.python_runtime_,
             environment={
                 **self.lambdas_env_,
                 **{"NEW_SCENES_QUEUE": self.queues_["new_scenes_queue"].queue_url},
@@ -510,7 +511,7 @@ def create_all_lambdas(self) -> None:
                     path="cbers2stac/elasticsearch", exclude=["*~"]
                 ),
                 handler="es.create_stac_index_handler",
-                runtime=aws_lambda.Runtime.PYTHON_3_7,
+                runtime=self.python_runtime_,
                 environment={**self.lambdas_env_,},
                 layers=[self.layers_["common_layer"]],
                 timeout=Duration.seconds(30),
@@ -524,7 +525,7 @@ def create_all_lambdas(self) -> None:
                     path="cbers2stac/elasticsearch", exclude=["*~"]
                 ),
                 handler="es.create_documents_handler",
-                runtime=aws_lambda.Runtime.PYTHON_3_7,
+                runtime=self.python_runtime_,
                 environment={
                     **self.lambdas_env_,
                     **{"ES_STRIPPED": "YES", "BULK_CALLS": "1", "BULK_SIZE": "10"},
@@ -548,7 +549,7 @@ def create_all_lambdas(self) -> None:
                     path="cbers2stac/reindex_stac_items", exclude=["*~"]
                 ),
                 handler="code.consume_stac_reconcile_queue_handler",
-                runtime=aws_lambda.Runtime.PYTHON_3_7,
+                runtime=self.python_runtime_,
                 environment=self.lambdas_env_,
                 layers=[self.layers_["common_layer"]],
                 timeout=Duration.seconds(900),
@@ -566,7 +567,7 @@ def create_all_lambdas(self) -> None:
                     path="cbers2stac/reindex_stac_items", exclude=["*~"]
                 ),
                 handler="code.populate_stac_reconcile_queue_handler",
-                runtime=aws_lambda.Runtime.PYTHON_3_7,
+                runtime=self.python_runtime_,
                 environment={**self.lambdas_env_,},
                 timeout=Duration.seconds(300),
                 dead_letter_queue=self.queues_["dead_letter_queue"],
@@ -588,7 +589,7 @@ def create_api_lambdas(self) -> None:
                 path="cbers2stac/stac_endpoint", exclude=["*~"]
             ),
             handler="code.handler",
-            runtime=aws_lambda.Runtime.PYTHON_3_7,
+            runtime=self.python_runtime_,
             environment={**self.lambdas_env_,},
             layers=[self.layers_["common_layer"]],
             timeout=Duration.seconds(30),
@@ -601,7 +602,7 @@ def create_api_lambdas(self) -> None:
                 path="cbers2stac/elasticsearch", exclude=["*~"]
             ),
             handler="es.stac_search_endpoint_handler",
-            runtime=aws_lambda.Runtime.PYTHON_3_7,
+            runtime=self.python_runtime_,
             environment={**self.lambdas_env_,},
             layers=[self.layers_["common_layer"]],
             timeout=Duration.seconds(55),
@@ -644,7 +645,7 @@ def create_api_gateway(self) -> None:
             self,
             "SearchEndpointCanary",
             schedule=synthetics.Schedule.rate(Duration.hours(1)),
-            runtime=synthetics.Runtime.SYNTHETICS_PYTHON_SELENIUM_1_0,
+            runtime=synthetics.Runtime.SYNTHETICS_PYTHON_SELENIUM_2_1,
             test=synthetics.Test.custom(
                 code=synthetics.Code.from_asset("cbers2stac/canary", exclude=["*~"]),
                 handler="api_canary.handler",