Update support for multipart uploads to non-AWS S3 endpoints

ohsu-comp-bio · Dec 6, 2023 · 4db98a9 · 4db98a9
1 parent b9c6efb
commit 4db98a9
Show file tree

Hide file tree

Showing 2 changed files with 14 additions and 9 deletions.
diff --git a/fence/blueprints/data/indexd.py b/fence/blueprints/data/indexd.py
@@ -349,12 +349,13 @@ def init_multipart_upload(key, expires_in=None, bucket=None):
         Returns:
             uploadId(str)
         """
-        bucket = bucket or flask.current_app.config["DATA_UPLOAD_BUCKET"]
         if not bucket:
-            raise InternalError(
-                "fence not configured with data upload bucket; can't create signed URL"
-            )
-
+            try:
+                bucket = flask.current_app.config["DATA_UPLOAD_BUCKET"]
+            except KeyError:
+                raise InternalError(
+                    "fence not configured with data upload bucket; can't create signed URL"
+                )
         s3_url = "s3://{}/{}".format(bucket, key)
         return S3IndexedFileLocation(s3_url).init_multipart_upload(expires_in)
 

diff --git a/fence/blueprints/data/multipart_upload.py b/fence/blueprints/data/multipart_upload.py
@@ -39,7 +39,6 @@ def initialize_multipart_upload(bucket_name, key, credentials):
         aws_secret_access_key=credentials["aws_secret_access_key"],
         aws_session_token=credentials.get("aws_session_token"),
     )
-
     s3client = None
     if url:
         s3client = session.client("s3", endpoint_url=url)
@@ -94,7 +93,6 @@ def complete_multipart_upload(bucket_name, key, credentials, uploadId, parts):
         aws_secret_access_key=credentials["aws_secret_access_key"],
         aws_session_token=credentials.get("aws_session_token"),
     )
-
     s3client = None
     if url:
         s3client = session.client("s3", endpoint_url=url)
@@ -147,7 +145,12 @@ def generate_presigned_url_for_uploading_part(
     )
     bucket = s3_buckets.get(bucket_name)
 
-    if s3_buckets.get("endpoint_url"):
+    s3_buckets = get_value(
+        config, "S3_BUCKETS", InternalError("S3_BUCKETS not configured")
+    )
+    bucket = s3_buckets.get(bucket_name)
+
+    if bucket.get("endpoint_url"):
         url = bucket["endpoint_url"].strip("/") + "/{}/{}".format(
             bucket_name, key.strip("/")
         )
@@ -156,9 +159,10 @@ def generate_presigned_url_for_uploading_part(
     additional_signed_qs = {"partNumber": str(partNumber), "uploadId": uploadId}
 
     try:
-        return generate_aws_presigned_url(
+        presigned_url = generate_aws_presigned_url(
             url, "PUT", credentials, "s3", region, expires, additional_signed_qs
         )
+        return presigned_url
     except Exception as e:
         raise InternalError(
             "Can not generate presigned url for part number {} of key {}. Detail {}".format(