Merge branch 'master' into feat/s3-delete-files

Author: lbeckman314

fence/blueprints/data/blueprint.py

@@ -134,6 +134,8 @@ def upload_data_file():
     authz = params.get("authz")
     uploader = None
 
+    guid = params.get("guid")
+
     if authz:
         # if requesting an authz field, using new authorization method which doesn't
         # rely on uploader field, so clear it out
@@ -165,7 +167,10 @@ def upload_data_file():
         )
 
     blank_index = BlankIndex(
-        file_name=params["file_name"], authz=params.get("authz"), uploader=uploader
+        file_name=params["file_name"],
+        authz=authz,
+        uploader=uploader,
+        guid=guid,
     )
     default_expires_in = flask.current_app.config.get("MAX_PRESIGNED_URL_TTL", 3600)
 
@@ -199,16 +204,16 @@ def upload_data_file():
 def init_multipart_upload():
     """
     Initialize a multipart upload request
-
-    NOTE This endpoint does not currently accept a `bucket` parameter like
-    `POST /upload` and `GET /upload/<GUID>` do.
     """
     params = flask.request.get_json()
     if not params:
         raise UserError("wrong Content-Type; expected application/json")
     if "file_name" not in params:
         raise UserError("missing required argument `file_name`")
-    blank_index = BlankIndex(file_name=params["file_name"])
+
+    guid = params.get("guid")
+
+    blank_index = BlankIndex(file_name=params["file_name"], guid=guid)
 
     default_expires_in = flask.current_app.config.get("MAX_PRESIGNED_URL_TTL", 3600)
     expires_in = get_valid_expiration(

fence/blueprints/data/indexd.py

@@ -258,7 +258,12 @@ class BlankIndex(object):
     """
 
     def __init__(
-        self, uploader=None, file_name=None, logger_=None, guid=None, authz=None
+        self,
+        uploader=None,
+        file_name=None,
+        logger_=None,
+        guid=None,
+        authz=None,
     ):
         self.logger = logger_ or logger
         self.indexd = (
@@ -279,8 +284,9 @@ def __init__(
         self.file_name = file_name
         self.authz = authz
 
-        # if a guid is not provided, this will create a blank record for you
-        self.guid = guid or self.index_document["did"]
+        self.guid = guid
+        # .index_document is a cached property with code below, it creates/retrieves the actual record and this line updates the stored GUID to the returned record
+        self.guid = self.index_document["did"]
 
     @cached_property
     def index_document(self):
@@ -292,6 +298,20 @@ def index_document(self):
                 response from indexd (the contents of the record), containing ``guid``
                 and ``url``
         """
+
+        if self.guid:
+            index_url = self.indexd.rstrip("/") + "/index/" + self.guid
+            indexd_response = requests.get(index_url)
+            if indexd_response.status_code == 200:
+                document = indexd_response.json()
+                self.logger.info(f"Record with {self.guid} id found in Indexd.")
+                return document
+            else:
+                raise NotFound(f"No indexed document found with id {self.guid}")
+
+        return self._create_blank_record()
+
+    def _create_blank_record(self):
         index_url = self.indexd.rstrip("/") + "/index/blank/"
         params = {"uploader": self.uploader, "file_name": self.file_name}
 

openapis/swagger.yaml

@@ -577,6 +577,8 @@ paths:
         Previous authorization check requires a more general, global upload permission:
         "file_upload" on "/data_file" resource. When "authz" is *not* provided, this
         endpoint will check for that permission for your user.
+
+        Accepts a "guid" field in the request body. If "guid" is provided, it checks indexd for an existing record. If not found, it raises a 404.
       security:
         - OAuth2:
           - user
@@ -602,6 +604,8 @@ paths:
                   url:
                     type: string
                     description: the presigned URL usable for data upload
+        404:
+          description: Record with <guid> not found.
   '/data/upload/{file_id}':
     get:
       tags:
@@ -678,7 +682,10 @@ paths:
         flow, Fence needs to provide a list of endpoints for supporting multipart upload presigned url
         This is the first step on the API side for the multipart upload presigned url. This endpoint
         causes fence to make a request to indexd to create a new, blank index record, and returns
-        the GUID for this new record and an uploadId for multipart upload presigned url
+        the GUID for this new record and an uploadId for multipart upload presigned url.
+
+
+        Accepts a "guid" field in the request body. If "guid" is provided, it checks indexd for an existing record. If not found, it raises a 404.
       security:
         - OAuth2:
           - user
@@ -704,6 +711,8 @@ paths:
                   uploadId:
                     type: string
                     description: the uploadId for multipart upload presigned URL usable for data upload
+        404:
+          description: Record with <guid> not found.
 
   '/multipart/upload':
     post:
@@ -1698,6 +1707,10 @@ components:
           description: >-
             the requested bucket to upload to.
             If not provided, defaults to the configured DATA_UPLOAD_BUCKET.
+        guid:
+          type: string
+          required: false
+          description: GUID to be assigned to the object
         expires_in:
           type: integer
           description: optional integer specifying the presigned URL lifetime
@@ -1708,6 +1721,7 @@ components:
           description: requested authorization resources to be set on the
             resulting indexed record. You must have proper authorization to set this
       example:
+        guid: "123456abcd"
         file_name: "my_file.bam"
         bucket: "bucket1"
         expires_in: 1200
@@ -1720,6 +1734,10 @@ components:
           type: string
           required: true
           description: the file name to use for this upload
+        guid:
+          type: string
+          required: false
+          description: GUID to be assigned to the object
         expires_in:
           type: integer
           description: optional integer specifying the presigned URL lifetime