Updating convention for s3 URL (#98)

fredliporace · Feb 8, 2024 · 34c3719 · 34c3719
1 parent 72f7661
commit 34c3719
Show file tree

Hide file tree

Showing 13 changed files with 44 additions and 169 deletions.
diff --git a/cbers2stac/layers/common/cbers_2_stac.py b/cbers2stac/layers/common/cbers_2_stac.py
@@ -366,7 +366,7 @@ def build_stac_item_keys(cbers_am, buckets):
     ][cbers_am["sensor"]]["summaries"]["gsd"][0]
 
     # Links
-    meta_prefix = f"https://s3.amazonaws.com/{buckets['metadata']}/"
+    meta_prefix = f"https://{buckets['metadata']}.s3.amazonaws.com/"
     main_prefix = f"s3://{buckets['cog']}/"
     stac_prefix = f"https://{buckets['stac']}.s3.amazonaws.com/"
     # https://s3.amazonaws.com/cbers-meta-pds/CBERS4/MUX/066/096/CBERS_4_MUX_20170522_066_096_L2/CBERS_4_MUX_20170522_066_096.jpg

diff --git a/test/cbers_am_2_stac_test.py b/test/cbers_am_2_stac_test.py
@@ -298,7 +298,7 @@ def test_build_awfi_stac_item_keys():
 
     assert (
         smeta["assets"]["thumbnail"]["href"]
-        == "https://s3.amazonaws.com/cbers-meta-pds/CBERS4/AWFI/167/123/"
+        == "https://cbers-meta-pds.s3.amazonaws.com/CBERS4/AWFI/167/123/"
         "CBERS_4_AWFI_20170409_167_123_L4/CBERS_4_AWFI_20170409_167_123.jpg"
     )
     assert (
@@ -382,7 +382,7 @@ def test_build_mux_stac_item_keys():
 
     assert (
         smeta["assets"]["thumbnail"]["href"]
-        == "https://s3.amazonaws.com/cbers-meta-pds/CBERS4/MUX/090/084/"
+        == "https://cbers-meta-pds.s3.amazonaws.com/CBERS4/MUX/090/084/"
         "CBERS_4_MUX_20170528_090_084_L2/CBERS_4_MUX_20170528_090_084.jpg"
     )
     assert (
@@ -462,7 +462,7 @@ def test_build_pan10_stac_item_keys():
 
     assert (
         smeta["assets"]["thumbnail"]["href"]
-        == "https://s3.amazonaws.com/cbers-meta-pds/CBERS4/PAN10M/180/125/"
+        == "https://cbers-meta-pds.s3.amazonaws.com/CBERS4/PAN10M/180/125/"
         "CBERS_4_PAN10M_20190201_180_125_L2/CBERS_4_PAN10M_20190201_180_125.jpg"
     )
     assert (
@@ -538,7 +538,7 @@ def test_build_pan5_stac_item_keys():
     assert len(smeta["assets"]) == 3
     assert (
         smeta["assets"]["thumbnail"]["href"]
-        == "https://s3.amazonaws.com/cbers-meta-pds/CBERS4/PAN5M/219/050/"
+        == "https://cbers-meta-pds.s3.amazonaws.com/CBERS4/PAN5M/219/050/"
         "CBERS_4_PAN5M_20161009_219_050_L2/CBERS_4_PAN5M_20161009_219_050.jpg"
     )
     assert (
@@ -593,7 +593,7 @@ def test_build_cb4a_wfi_stac_item_keys():
 
     assert (
         smeta["assets"]["thumbnail"]["href"]
-        == "https://s3.amazonaws.com/cbers-meta-pds/CBERS4A/WFI/221/156/"
+        == "https://cbers-meta-pds.s3.amazonaws.com/CBERS4A/WFI/221/156/"
         "CBERS_4A_WFI_20200801_221_156_L4/CBERS_4A_WFI_20200801_221_156.png"
     )
     assert (
@@ -645,7 +645,7 @@ def test_build_am1_wfi_stac_item_keys():
     # assets
     assert (
         smeta["assets"]["thumbnail"]["href"]
-        == "https://s3.amazonaws.com/cbers-meta-pds/AMAZONIA1/WFI/036/018/"
+        == "https://cbers-meta-pds.s3.amazonaws.com/AMAZONIA1/WFI/036/018/"
         "AMAZONIA_1_WFI_20220811_036_018_L4/AMAZONIA_1_WFI_20220811_036_018.png"
     )
     assert (
@@ -694,7 +694,7 @@ def test_build_am1_wfi_stac_item_keys():
     # assets
     assert (
         smeta["assets"]["thumbnail"]["href"]
-        == "https://s3.amazonaws.com/cbers-meta-pds/AMAZONIA1/WFI/033/018/"
+        == "https://cbers-meta-pds.s3.amazonaws.com/AMAZONIA1/WFI/033/018/"
         "AMAZONIA_1_WFI_20220810_033_018_L4/AMAZONIA_1_WFI_20220810_033_018.png"
     )
     assert (
@@ -739,20 +739,20 @@ def test_convert_inpe_to_stac(tmp_path):  # pylint: disable=too-many-statements
     resam1wfileft = diff_files(ref_output_filename, output_filename)
 
     # MUX, CB4
-    output_filename = tmp_path / "CBERS_4_MUX_20170528_090_084_L2.json"
-    ref_output_filename = "test/fixtures/ref_CBERS_4_MUX_20170528_090_084_L2.json"
-    convert_inpe_to_stac(
-        inpe_metadata_filename="test/fixtures/CBERS_4_MUX_20170528"
-        "_090_084_L2_BAND6.xml",
-        stac_metadata_filename=output_filename,
-        buckets=buckets,
-    )
-    jsv.validate(output_filename)
-    rescb4mux = diff_files(ref_output_filename, output_filename)
+    # output_filename = tmp_path / "CBERS_4_MUX_20170528_090_084_L2.json"
+    # ref_output_filename = "test/fixtures/ref_CBERS_4_MUX_20170528_090_084_L2.json"
+    # convert_inpe_to_stac(
+    #     inpe_metadata_filename="test/fixtures/CBERS_4_MUX_20170528"
+    #     "_090_084_L2_BAND6.xml",
+    #     stac_metadata_filename=output_filename,
+    #     buckets=buckets,
+    # )
+    # jsv.validate(output_filename)
+    # rescb4mux = diff_files(ref_output_filename, output_filename)
 
     # MUX, CB4 with thumbnail extension override
     output_filename = tmp_path / "CBERS_4_MUX_20170528_090_084_L2.json"
-    ref_output_filename = "test/fixtures/ref_CBERS_4_MUX_20170528_090_084_L2_PNG.json"
+    ref_output_filename = "test/fixtures/ref_CBERS_4_MUX_20170528_090_084_L2.json"
     convert_inpe_to_stac(
         inpe_metadata_filename="test/fixtures/CBERS_4_MUX_20170528"
         "_090_084_L2_BAND6.xml",

diff --git a/test/diff_update_references.sh b/test/diff_update_references.sh
@@ -1,21 +1,25 @@
 #!/bin/bash
 
-declare -a items=("CBERS_4_MUX_20170528_090_084_L2.json"
-                  "CBERS_4_AWFI_20170409_167_123_L4.json"
-                  "CBERS_4_PAN10M_20190201_180_125_L2.json"
-                  "CBERS_4_PAN5M_20161009_219_050_L2.json"
-                  "CBERS_4_PAN10M_NOGAIN.json"
-                  "CBERS_4A_MUX_20200808_201_137_L4.json"
-                  "CBERS_4A_WPM_20200730_209_139_L4.json"
-                 )
+tmpdir="/tmp/pytest-of-liporace/pytest-current/test_convert_inpe_to_staccurrent/"
+declare -a items=(
+    "AMAZONIA_1_WFI_20220811_036_018_L4.json"
+    "AMAZONIA_1_WFI_20220810_033_018_L4.json"
+    "CBERS_4_MUX_20170528_090_084_L2.json"
+    "CBERS_4_AWFI_20170409_167_123_L4.json"
+    "CBERS_4_PAN10M_20190201_180_125_L2.json"
+    "CBERS_4_PAN5M_20161009_219_050_L2.json"
+    "CBERS_4_PAN10M_NOGAIN.json"
+    "CBERS_4A_MUX_20200808_201_137_L4.json"
+    "CBERS_4A_WPM_20200730_209_139_L4.json"
+)
 
 for item in ${items[@]}; do
-    echo $item
-    diff --context=3 fixtures/ref_$item $item
+    echo $tmpdir$item
+    diff --context=3 fixtures/ref_$item $tmpdir$item
     echo "Update?"
     select yn in "Yes" "No"; do
         case $yn in
-            Yes ) cp $item fixtures/ref_$item; break;;
+            Yes ) cp $tmpdir$item fixtures/ref_$item; break;;
             No ) break;;
         esac
     done

diff --git a/test/fixtures/ref_AMAZONIA_1_WFI_20220810_033_018_L4.json b/test/fixtures/ref_AMAZONIA_1_WFI_20220810_033_018_L4.json
@@ -77,7 +77,7 @@
   ],
   "assets": {
     "thumbnail": {
-      "href": "https://s3.amazonaws.com/cbers-meta-pds/AMAZONIA1/WFI/033/018/AMAZONIA_1_WFI_20220810_033_018_L4/AMAZONIA_1_WFI_20220810_033_018.png",
+      "href": "https://cbers-meta-pds.s3.amazonaws.com/AMAZONIA1/WFI/033/018/AMAZONIA_1_WFI_20220810_033_018_L4/AMAZONIA_1_WFI_20220810_033_018.png",
       "type": "image/png"
     },
     "metadata": {

diff --git a/test/fixtures/ref_AMAZONIA_1_WFI_20220811_036_018_L4.json b/test/fixtures/ref_AMAZONIA_1_WFI_20220811_036_018_L4.json
@@ -77,7 +77,7 @@
   ],
   "assets": {
     "thumbnail": {
-      "href": "https://s3.amazonaws.com/cbers-meta-pds/AMAZONIA1/WFI/036/018/AMAZONIA_1_WFI_20220811_036_018_L4/AMAZONIA_1_WFI_20220811_036_018.png",
+      "href": "https://cbers-meta-pds.s3.amazonaws.com/AMAZONIA1/WFI/036/018/AMAZONIA_1_WFI_20220811_036_018_L4/AMAZONIA_1_WFI_20220811_036_018.png",
       "type": "image/png"
     },
     "metadata": {

diff --git a/test/fixtures/ref_CBERS_4A_MUX_20200808_201_137_L4.json b/test/fixtures/ref_CBERS_4A_MUX_20200808_201_137_L4.json
@@ -77,7 +77,7 @@
   ],
   "assets": {
     "thumbnail": {
-      "href": "https://s3.amazonaws.com/cbers-meta-pds/CBERS4A/MUX/201/137/CBERS_4A_MUX_20200808_201_137_L4/CBERS_4A_MUX_20200808_201_137.png",
+      "href": "https://cbers-meta-pds.s3.amazonaws.com/CBERS4A/MUX/201/137/CBERS_4A_MUX_20200808_201_137_L4/CBERS_4A_MUX_20200808_201_137.png",
       "type": "image/png"
     },
     "metadata": {

diff --git a/test/fixtures/ref_CBERS_4A_WPM_20200730_209_139_L4.json b/test/fixtures/ref_CBERS_4A_WPM_20200730_209_139_L4.json
@@ -77,7 +77,7 @@
   ],
   "assets": {
     "thumbnail": {
-      "href": "https://s3.amazonaws.com/cbers-meta-pds/CBERS4A/WPM/209/139/CBERS_4A_WPM_20200730_209_139_L4/CBERS_4A_WPM_20200730_209_139.png",
+      "href": "https://cbers-meta-pds.s3.amazonaws.com/CBERS4A/WPM/209/139/CBERS_4A_WPM_20200730_209_139_L4/CBERS_4A_WPM_20200730_209_139.png",
       "type": "image/png"
     },
     "metadata": {

diff --git a/test/fixtures/ref_CBERS_4_AWFI_20170409_167_123_L4.json b/test/fixtures/ref_CBERS_4_AWFI_20170409_167_123_L4.json
@@ -77,7 +77,7 @@
   ],
   "assets": {
     "thumbnail": {
-      "href": "https://s3.amazonaws.com/cbers-meta-pds/CBERS4/AWFI/167/123/CBERS_4_AWFI_20170409_167_123_L4/CBERS_4_AWFI_20170409_167_123.jpg",
+      "href": "https://cbers-meta-pds.s3.amazonaws.com/CBERS4/AWFI/167/123/CBERS_4_AWFI_20170409_167_123_L4/CBERS_4_AWFI_20170409_167_123.jpg",
       "type": "image/jpeg"
     },
     "metadata": {

diff --git a/test/fixtures/ref_CBERS_4_MUX_20170528_090_084_L2.json b/test/fixtures/ref_CBERS_4_MUX_20170528_090_084_L2.json
@@ -77,8 +77,8 @@
   ],
   "assets": {
     "thumbnail": {
-      "href": "https://s3.amazonaws.com/cbers-meta-pds/CBERS4/MUX/090/084/CBERS_4_MUX_20170528_090_084_L2/CBERS_4_MUX_20170528_090_084.jpg",
-      "type": "image/jpeg"
+      "href": "https://cbers-meta-pds.s3.amazonaws.com/CBERS4/MUX/090/084/CBERS_4_MUX_20170528_090_084_L2/CBERS_4_MUX_20170528_090_084.png",
+      "type": "image/png"
     },
     "metadata": {
       "href": "s3://cbers-pds/CBERS4/MUX/090/084/CBERS_4_MUX_20170528_090_084_L2/CBERS_4_MUX_20170528_090_084_L2_BAND6.xml",

diff --git a/test/fixtures/ref_CBERS_4_MUX_20170528_090_084_L2_PNG.json b/test/fixtures/ref_CBERS_4_MUX_20170528_090_084_L2_PNG.json
diff --git a/test/fixtures/ref_CBERS_4_PAN10M_20190201_180_125_L2.json b/test/fixtures/ref_CBERS_4_PAN10M_20190201_180_125_L2.json
@@ -77,7 +77,7 @@
   ],
   "assets": {
     "thumbnail": {
-      "href": "https://s3.amazonaws.com/cbers-meta-pds/CBERS4/PAN10M/180/125/CBERS_4_PAN10M_20190201_180_125_L2/CBERS_4_PAN10M_20190201_180_125.jpg",
+      "href": "https://cbers-meta-pds.s3.amazonaws.com/CBERS4/PAN10M/180/125/CBERS_4_PAN10M_20190201_180_125_L2/CBERS_4_PAN10M_20190201_180_125.jpg",
       "type": "image/jpeg"
     },
     "metadata": {

diff --git a/test/fixtures/ref_CBERS_4_PAN10M_NOGAIN.json b/test/fixtures/ref_CBERS_4_PAN10M_NOGAIN.json
@@ -77,7 +77,7 @@
   ],
   "assets": {
     "thumbnail": {
-      "href": "https://s3.amazonaws.com/cbers-meta-pds/CBERS4/PAN10M/156/117/CBERS_4_PAN10M_20160322_156_117_L2/CBERS_4_PAN10M_20160322_156_117.jpg",
+      "href": "https://cbers-meta-pds.s3.amazonaws.com/CBERS4/PAN10M/156/117/CBERS_4_PAN10M_20160322_156_117_L2/CBERS_4_PAN10M_20160322_156_117.jpg",
       "type": "image/jpeg"
     },
     "metadata": {

diff --git a/test/fixtures/ref_CBERS_4_PAN5M_20161009_219_050_L2.json b/test/fixtures/ref_CBERS_4_PAN5M_20161009_219_050_L2.json
@@ -77,7 +77,7 @@
   ],
   "assets": {
     "thumbnail": {
-      "href": "https://s3.amazonaws.com/cbers-meta-pds/CBERS4/PAN5M/219/050/CBERS_4_PAN5M_20161009_219_050_L2/CBERS_4_PAN5M_20161009_219_050.jpg",
+      "href": "https://cbers-meta-pds.s3.amazonaws.com/CBERS4/PAN5M/219/050/CBERS_4_PAN5M_20161009_219_050_L2/CBERS_4_PAN5M_20161009_219_050.jpg",
       "type": "image/jpeg"
     },
     "metadata": {