Merge pull request #99 from fredliporace/dev

Migration from us-east-1 to us-west-2
fredliporace · Feb 16, 2024 · 9d45790 · 9d45790
2 parents d9c2454 + 2e9c09f
commit 9d45790
Show file tree

Hide file tree

Showing 17 changed files with 92 additions and 222 deletions.
diff --git a/README.md b/README.md
@@ -87,10 +87,10 @@ cbers2stac-prod.stacitemtopicoutput = arn:aws:sns:us-east-1:...:...
 
 Empty static stac catalogs and collections are created when the stack is deployed. Note that when these files are updated and a new deploy is executed to an already populated stac bucket the [deployment may fail](https://github.com/fredliporace/cbers-2-stac/issues/88).
 
-In order to overcome that you may set `STACK_DEPLOY_STATIC_CATALOG_STRUCTURE` to `false` in `.env` and manually copy the static files. From a prompy in the `stack/static_catalog_structure` dir:
+In order to overcome that you may set `STACK_DEPLOY_STATIC_CATALOG_STRUCTURE` to `false` in `.env` and manually copy the static files. From a prompt in the `stack/static_catalog_structure` dir:
 
 ```bash
-$ aws s3 cp ./ s3://cbers-stac-1-0-0 --recursive
+$ aws s3 cp ./ s3://[REPLACE_WITH_STAC_BUCKET_NAME] --recursive
 ```
 
 Replace `cbers-stac-1-0-0` with the given stac bucket name. The collections initially do not contain the links to the children, this is updated when a new scene is inserted at any level.

diff --git a/cbers2stac/layers/common/cbers_2_stac.py b/cbers2stac/layers/common/cbers_2_stac.py
@@ -366,7 +366,7 @@ def build_stac_item_keys(cbers_am, buckets):
     ][cbers_am["sensor"]]["summaries"]["gsd"][0]
 
     # Links
-    meta_prefix = f"https://s3.amazonaws.com/{buckets['metadata']}/"
+    meta_prefix = f"https://{buckets['metadata']}.s3.amazonaws.com/"
     main_prefix = f"s3://{buckets['cog']}/"
     stac_prefix = f"https://{buckets['stac']}.s3.amazonaws.com/"
     # https://s3.amazonaws.com/cbers-meta-pds/CBERS4/MUX/066/096/CBERS_4_MUX_20170522_066_096_L2/CBERS_4_MUX_20170522_066_096.jpg

diff --git a/stack/app.py b/stack/app.py
@@ -138,41 +138,41 @@ def create_all_queues(self) -> None:
                 max_receive_count=1, queue=self.queues_["process_new_scenes_queue_dlq"]
             ),
         )
-        # Add subscriptions for each CB4 camera
-        sns.Topic.from_topic_arn(
-            self,
-            id="CB4MUX",
-            topic_arn="arn:aws:sns:us-east-1:599544552497:NewCB4MUXQuicklook",
-        ).add_subscription(
-            sns_subscriptions.SqsSubscription(self.queues_["new_scenes_queue"])
-        )
-        sns.Topic.from_topic_arn(
-            self,
-            id="CB4AWFI",
-            topic_arn="arn:aws:sns:us-east-1:599544552497:NewCB4AWFIQuicklook",
-        ).add_subscription(
-            sns_subscriptions.SqsSubscription(self.queues_["new_scenes_queue"])
-        )
-        sns.Topic.from_topic_arn(
-            self,
-            id="CB4PAN10M",
-            topic_arn="arn:aws:sns:us-east-1:599544552497:NewCB4PAN10MQuicklook",
-        ).add_subscription(
-            sns_subscriptions.SqsSubscription(self.queues_["new_scenes_queue"])
-        )
-        sns.Topic.from_topic_arn(
-            self,
-            id="CBPAN5M",
-            topic_arn="arn:aws:sns:us-east-1:599544552497:NewCB4PAN5MQuicklook",
-        ).add_subscription(
-            sns_subscriptions.SqsSubscription(self.queues_["new_scenes_queue"])
-        )
+        # # Add subscriptions for each CB4 camera (old ingestion system)
+        # sns.Topic.from_topic_arn(
+        #     self,
+        #     id="CB4MUX",
+        #     topic_arn="arn:aws:sns:us-east-1:599544552497:NewCB4MUXQuicklook",
+        # ).add_subscription(
+        #     sns_subscriptions.SqsSubscription(self.queues_["new_scenes_queue"])
+        # )
+        # sns.Topic.from_topic_arn(
+        #     self,
+        #     id="CB4AWFI",
+        #     topic_arn="arn:aws:sns:us-east-1:599544552497:NewCB4AWFIQuicklook",
+        # ).add_subscription(
+        #     sns_subscriptions.SqsSubscription(self.queues_["new_scenes_queue"])
+        # )
+        # sns.Topic.from_topic_arn(
+        #     self,
+        #     id="CB4PAN10M",
+        #     topic_arn="arn:aws:sns:us-east-1:599544552497:NewCB4PAN10MQuicklook",
+        # ).add_subscription(
+        #     sns_subscriptions.SqsSubscription(self.queues_["new_scenes_queue"])
+        # )
+        # sns.Topic.from_topic_arn(
+        #     self,
+        #     id="CBPAN5M",
+        #     topic_arn="arn:aws:sns:us-east-1:599544552497:NewCB4PAN5MQuicklook",
+        # ).add_subscription(
+        #     sns_subscriptions.SqsSubscription(self.queues_["new_scenes_queue"])
+        # )
         # Subscription for CB4 (all cameras), CB4A (all cameras) and Amazonia1
         # These are from the INPE catalog that creates quicklooks as .png
         for topic_arn in settings.topics:
             sns.Topic.from_topic_arn(
                 self,
-                id=f"CB4A-AM1-{topic_arn.rsplit(':', maxsplit=1)[-1]}",
+                id=f"CB-AM-{topic_arn.rsplit(':', maxsplit=1)[-1]}",
                 topic_arn=topic_arn,
             ).add_subscription(
                 sns_subscriptions.SqsSubscription(self.queues_["new_scenes_queue"])
@@ -399,15 +399,6 @@ def create_all_lambdas(self) -> None:
         self.lambdas_["process_new_scene_lambda"].add_event_source(
             SqsEventSource(queue=self.queues_["new_scenes_queue"], batch_size=10)
         )
-        # See comment below on using from_bucket_name to
-        # create a CDK bucket
-        read_cbers_pds_permissions = iam.PolicyStatement(
-            actions=["s3:ListObjectsV2", "s3:ListBucket", "s3:Get*"],
-            resources=["arn:aws:s3:::cbers-pds", "arn:aws:s3:::cbers-pds/*",],
-        )
-        self.lambdas_["process_new_scene_lambda"].add_to_role_policy(
-            read_cbers_pds_permissions
-        )
 
         self.create_lambda(
             id="generate_catalog_levels_to_be_updated_lambda",
@@ -488,22 +479,25 @@ def create_all_lambdas(self) -> None:
         )
 
         # I'm using the bucket ARNs directly here just to make sure that I don't
-        # mess with the cbers-pds bucket... creating it from_bucket_name should
+        # mess with the pds buckets... creating it from_bucket_name should
         # be safe but I'll not take my chances
         # cbers_pds_bucket = s3.Bucket.from_bucket_name(self, "cbers-pds", "cbers-pds")
-        for pds_bucket in ["cbers-pds", "amazonia-pds"]:
-            list_pds_permissions = iam.PolicyStatement(
-                actions=["s3:ListObjectsV2", "s3:ListBucket"],
+        for pds_bucket in ["brazil-eosats"]:
+            pds_permissions = iam.PolicyStatement(
+                actions=["s3:ListObjectsV2", "s3:ListBucket", "s3:Get*"],
                 resources=[
                     f"arn:aws:s3:::{pds_bucket}",
                     f"arn:aws:s3:::{pds_bucket}/*",
                 ],
             )
             self.lambdas_["populate_reconcile_queue_lambda"].add_to_role_policy(
-                list_pds_permissions
+                pds_permissions
             )
             self.lambdas_["consume_reconcile_queue_lambda"].add_to_role_policy(
-                list_pds_permissions
+                pds_permissions
+            )
+            self.lambdas_["process_new_scene_lambda"].add_to_role_policy(
+                pds_permissions
             )
 
         # Section with lambdas used to support STAC API. Specific lambdas integrated
@@ -773,6 +767,11 @@ def __init__(
             "stac_working_bucket",
             bucket_name=settings.stac_bucket_name,
             auto_delete_objects=True,
+            block_public_access=s3.BlockPublicAccess(
+                block_public_policy=False,
+                block_public_acls=False,
+                restrict_public_buckets=False,
+            ),
             removal_policy=core.RemovalPolicy.DESTROY,
         )
         self.lambdas_env_.update({"STAC_BUCKET": stac_working_bucket.bucket_name})

diff --git a/stack/config.py b/stack/config.py
@@ -27,11 +27,7 @@ class StackSettings(pydantic.BaseSettings):  # pylint: disable=too-few-public-me
     es_volume_size: int
     es_data_nodes: int
 
-    topics: Optional[List[str]] = [
-        "arn:aws:sns:us-east-1:599544552497:NewCB4Quicklook",
-        "arn:aws:sns:us-east-1:599544552497:NewCB4AQuicklook",
-        "arn:aws:sns:us-east-1:599544552497:NewAMQuicklook",
-    ]
+    topics: List[str]
 
     deploy_static_catalog_structure: Optional[bool] = True
 

diff --git a/test/cbers_am_2_stac_test.py b/test/cbers_am_2_stac_test.py
@@ -298,7 +298,7 @@ def test_build_awfi_stac_item_keys():
 
     assert (
         smeta["assets"]["thumbnail"]["href"]
-        == "https://s3.amazonaws.com/cbers-meta-pds/CBERS4/AWFI/167/123/"
+        == "https://cbers-meta-pds.s3.amazonaws.com/CBERS4/AWFI/167/123/"
         "CBERS_4_AWFI_20170409_167_123_L4/CBERS_4_AWFI_20170409_167_123.jpg"
     )
     assert (
@@ -382,7 +382,7 @@ def test_build_mux_stac_item_keys():
 
     assert (
         smeta["assets"]["thumbnail"]["href"]
-        == "https://s3.amazonaws.com/cbers-meta-pds/CBERS4/MUX/090/084/"
+        == "https://cbers-meta-pds.s3.amazonaws.com/CBERS4/MUX/090/084/"
         "CBERS_4_MUX_20170528_090_084_L2/CBERS_4_MUX_20170528_090_084.jpg"
     )
     assert (
@@ -462,7 +462,7 @@ def test_build_pan10_stac_item_keys():
 
     assert (
         smeta["assets"]["thumbnail"]["href"]
-        == "https://s3.amazonaws.com/cbers-meta-pds/CBERS4/PAN10M/180/125/"
+        == "https://cbers-meta-pds.s3.amazonaws.com/CBERS4/PAN10M/180/125/"
         "CBERS_4_PAN10M_20190201_180_125_L2/CBERS_4_PAN10M_20190201_180_125.jpg"
     )
     assert (
@@ -538,7 +538,7 @@ def test_build_pan5_stac_item_keys():
     assert len(smeta["assets"]) == 3
     assert (
         smeta["assets"]["thumbnail"]["href"]
-        == "https://s3.amazonaws.com/cbers-meta-pds/CBERS4/PAN5M/219/050/"
+        == "https://cbers-meta-pds.s3.amazonaws.com/CBERS4/PAN5M/219/050/"
         "CBERS_4_PAN5M_20161009_219_050_L2/CBERS_4_PAN5M_20161009_219_050.jpg"
     )
     assert (
@@ -593,7 +593,7 @@ def test_build_cb4a_wfi_stac_item_keys():
 
     assert (
         smeta["assets"]["thumbnail"]["href"]
-        == "https://s3.amazonaws.com/cbers-meta-pds/CBERS4A/WFI/221/156/"
+        == "https://cbers-meta-pds.s3.amazonaws.com/CBERS4A/WFI/221/156/"
         "CBERS_4A_WFI_20200801_221_156_L4/CBERS_4A_WFI_20200801_221_156.png"
     )
     assert (
@@ -645,7 +645,7 @@ def test_build_am1_wfi_stac_item_keys():
     # assets
     assert (
         smeta["assets"]["thumbnail"]["href"]
-        == "https://s3.amazonaws.com/cbers-meta-pds/AMAZONIA1/WFI/036/018/"
+        == "https://cbers-meta-pds.s3.amazonaws.com/AMAZONIA1/WFI/036/018/"
         "AMAZONIA_1_WFI_20220811_036_018_L4/AMAZONIA_1_WFI_20220811_036_018.png"
     )
     assert (
@@ -694,7 +694,7 @@ def test_build_am1_wfi_stac_item_keys():
     # assets
     assert (
         smeta["assets"]["thumbnail"]["href"]
-        == "https://s3.amazonaws.com/cbers-meta-pds/AMAZONIA1/WFI/033/018/"
+        == "https://cbers-meta-pds.s3.amazonaws.com/AMAZONIA1/WFI/033/018/"
         "AMAZONIA_1_WFI_20220810_033_018_L4/AMAZONIA_1_WFI_20220810_033_018.png"
     )
     assert (
@@ -739,20 +739,20 @@ def test_convert_inpe_to_stac(tmp_path):  # pylint: disable=too-many-statements
     resam1wfileft = diff_files(ref_output_filename, output_filename)
 
     # MUX, CB4
-    output_filename = tmp_path / "CBERS_4_MUX_20170528_090_084_L2.json"
-    ref_output_filename = "test/fixtures/ref_CBERS_4_MUX_20170528_090_084_L2.json"
-    convert_inpe_to_stac(
-        inpe_metadata_filename="test/fixtures/CBERS_4_MUX_20170528"
-        "_090_084_L2_BAND6.xml",
-        stac_metadata_filename=output_filename,
-        buckets=buckets,
-    )
-    jsv.validate(output_filename)
-    rescb4mux = diff_files(ref_output_filename, output_filename)
+    # output_filename = tmp_path / "CBERS_4_MUX_20170528_090_084_L2.json"
+    # ref_output_filename = "test/fixtures/ref_CBERS_4_MUX_20170528_090_084_L2.json"
+    # convert_inpe_to_stac(
+    #     inpe_metadata_filename="test/fixtures/CBERS_4_MUX_20170528"
+    #     "_090_084_L2_BAND6.xml",
+    #     stac_metadata_filename=output_filename,
+    #     buckets=buckets,
+    # )
+    # jsv.validate(output_filename)
+    # rescb4mux = diff_files(ref_output_filename, output_filename)
 
     # MUX, CB4 with thumbnail extension override
     output_filename = tmp_path / "CBERS_4_MUX_20170528_090_084_L2.json"
-    ref_output_filename = "test/fixtures/ref_CBERS_4_MUX_20170528_090_084_L2_PNG.json"
+    ref_output_filename = "test/fixtures/ref_CBERS_4_MUX_20170528_090_084_L2.json"
     convert_inpe_to_stac(
         inpe_metadata_filename="test/fixtures/CBERS_4_MUX_20170528"
         "_090_084_L2_BAND6.xml",

diff --git a/test/diff_update_references.sh b/test/diff_update_references.sh
@@ -1,21 +1,25 @@
 #!/bin/bash
 
-declare -a items=("CBERS_4_MUX_20170528_090_084_L2.json"
-                  "CBERS_4_AWFI_20170409_167_123_L4.json"
-                  "CBERS_4_PAN10M_20190201_180_125_L2.json"
-                  "CBERS_4_PAN5M_20161009_219_050_L2.json"
-                  "CBERS_4_PAN10M_NOGAIN.json"
-                  "CBERS_4A_MUX_20200808_201_137_L4.json"
-                  "CBERS_4A_WPM_20200730_209_139_L4.json"
-                 )
+tmpdir="/tmp/pytest-of-liporace/pytest-current/test_convert_inpe_to_staccurrent/"
+declare -a items=(
+    "AMAZONIA_1_WFI_20220811_036_018_L4.json"
+    "AMAZONIA_1_WFI_20220810_033_018_L4.json"
+    "CBERS_4_MUX_20170528_090_084_L2.json"
+    "CBERS_4_AWFI_20170409_167_123_L4.json"
+    "CBERS_4_PAN10M_20190201_180_125_L2.json"
+    "CBERS_4_PAN5M_20161009_219_050_L2.json"
+    "CBERS_4_PAN10M_NOGAIN.json"
+    "CBERS_4A_MUX_20200808_201_137_L4.json"
+    "CBERS_4A_WPM_20200730_209_139_L4.json"
+)
 
 for item in ${items[@]}; do
-    echo $item
-    diff --context=3 fixtures/ref_$item $item
+    echo $tmpdir$item
+    diff --context=3 fixtures/ref_$item $tmpdir$item
     echo "Update?"
     select yn in "Yes" "No"; do
         case $yn in
-            Yes ) cp $item fixtures/ref_$item; break;;
+            Yes ) cp $tmpdir$item fixtures/ref_$item; break;;
             No ) break;;
         esac
     done

diff --git a/test/fixtures/ref_AMAZONIA_1_WFI_20220810_033_018_L4.json b/test/fixtures/ref_AMAZONIA_1_WFI_20220810_033_018_L4.json
@@ -77,7 +77,7 @@
   ],
   "assets": {
     "thumbnail": {
-      "href": "https://s3.amazonaws.com/cbers-meta-pds/AMAZONIA1/WFI/033/018/AMAZONIA_1_WFI_20220810_033_018_L4/AMAZONIA_1_WFI_20220810_033_018.png",
+      "href": "https://cbers-meta-pds.s3.amazonaws.com/AMAZONIA1/WFI/033/018/AMAZONIA_1_WFI_20220810_033_018_L4/AMAZONIA_1_WFI_20220810_033_018.png",
       "type": "image/png"
     },
     "metadata": {

diff --git a/test/fixtures/ref_AMAZONIA_1_WFI_20220811_036_018_L4.json b/test/fixtures/ref_AMAZONIA_1_WFI_20220811_036_018_L4.json
@@ -77,7 +77,7 @@
   ],
   "assets": {
     "thumbnail": {
-      "href": "https://s3.amazonaws.com/cbers-meta-pds/AMAZONIA1/WFI/036/018/AMAZONIA_1_WFI_20220811_036_018_L4/AMAZONIA_1_WFI_20220811_036_018.png",
+      "href": "https://cbers-meta-pds.s3.amazonaws.com/AMAZONIA1/WFI/036/018/AMAZONIA_1_WFI_20220811_036_018_L4/AMAZONIA_1_WFI_20220811_036_018.png",
       "type": "image/png"
     },
     "metadata": {

diff --git a/test/fixtures/ref_CBERS_4A_MUX_20200808_201_137_L4.json b/test/fixtures/ref_CBERS_4A_MUX_20200808_201_137_L4.json
@@ -77,7 +77,7 @@
   ],
   "assets": {
     "thumbnail": {
-      "href": "https://s3.amazonaws.com/cbers-meta-pds/CBERS4A/MUX/201/137/CBERS_4A_MUX_20200808_201_137_L4/CBERS_4A_MUX_20200808_201_137.png",
+      "href": "https://cbers-meta-pds.s3.amazonaws.com/CBERS4A/MUX/201/137/CBERS_4A_MUX_20200808_201_137_L4/CBERS_4A_MUX_20200808_201_137.png",
       "type": "image/png"
     },
     "metadata": {

diff --git a/test/fixtures/ref_CBERS_4A_WPM_20200730_209_139_L4.json b/test/fixtures/ref_CBERS_4A_WPM_20200730_209_139_L4.json
@@ -77,7 +77,7 @@
   ],
   "assets": {
     "thumbnail": {
-      "href": "https://s3.amazonaws.com/cbers-meta-pds/CBERS4A/WPM/209/139/CBERS_4A_WPM_20200730_209_139_L4/CBERS_4A_WPM_20200730_209_139.png",
+      "href": "https://cbers-meta-pds.s3.amazonaws.com/CBERS4A/WPM/209/139/CBERS_4A_WPM_20200730_209_139_L4/CBERS_4A_WPM_20200730_209_139.png",
       "type": "image/png"
     },
     "metadata": {

diff --git a/test/fixtures/ref_CBERS_4_AWFI_20170409_167_123_L4.json b/test/fixtures/ref_CBERS_4_AWFI_20170409_167_123_L4.json
@@ -77,7 +77,7 @@
   ],
   "assets": {
     "thumbnail": {
-      "href": "https://s3.amazonaws.com/cbers-meta-pds/CBERS4/AWFI/167/123/CBERS_4_AWFI_20170409_167_123_L4/CBERS_4_AWFI_20170409_167_123.jpg",
+      "href": "https://cbers-meta-pds.s3.amazonaws.com/CBERS4/AWFI/167/123/CBERS_4_AWFI_20170409_167_123_L4/CBERS_4_AWFI_20170409_167_123.jpg",
       "type": "image/jpeg"
     },
     "metadata": {

diff --git a/test/fixtures/ref_CBERS_4_MUX_20170528_090_084_L2.json b/test/fixtures/ref_CBERS_4_MUX_20170528_090_084_L2.json
@@ -77,8 +77,8 @@
   ],
   "assets": {
     "thumbnail": {
-      "href": "https://s3.amazonaws.com/cbers-meta-pds/CBERS4/MUX/090/084/CBERS_4_MUX_20170528_090_084_L2/CBERS_4_MUX_20170528_090_084.jpg",
-      "type": "image/jpeg"
+      "href": "https://cbers-meta-pds.s3.amazonaws.com/CBERS4/MUX/090/084/CBERS_4_MUX_20170528_090_084_L2/CBERS_4_MUX_20170528_090_084.png",
+      "type": "image/png"
     },
     "metadata": {
       "href": "s3://cbers-pds/CBERS4/MUX/090/084/CBERS_4_MUX_20170528_090_084_L2/CBERS_4_MUX_20170528_090_084_L2_BAND6.xml",