update 3.24.6.1

huaweicloud · Jul 26, 2024 · 7d25d15 · 7d25d15
1 parent 0da1595
commit 7d25d15
Show file tree

Hide file tree

Showing 7 changed files with 51 additions and 15 deletions.
diff --git a/README.md b/README.md
@@ -1,3 +1,10 @@
+Version 3.24.6.1
+
+New Features:
+
+1. ObsClient.uploadFile Supported check data integrity by crc64.
+
+------------------------------------------------------------------------------------------------- 
 Version 3.24.6
 
 New Features:

diff --git a/README_CN.md b/README_CN.md
@@ -1,4 +1,11 @@
-Version 3.24.6
+Version 3.24.6.1
+
+新特性：
+
+1. 支持crc64校验(ObsClient.uploadFile)
+
+-------------------------------------------------------------------------------------------------
+Version 3.24.6
 
 新特性：
 

diff --git a/src/obs/client.py b/src/obs/client.py
@@ -2488,15 +2488,15 @@ def _clear_virtual_bucket(self, staged, bucketName1, bucketName2, bucketAlias):
     def uploadFile(self, bucketName, objectKey, uploadFile, partSize=9 * 1024 * 1024,
                    taskNum=1, enableCheckpoint=False, checkpointFile=None,
                    checkSum=False, metadata=None, progressCallback=None, headers=None,
-                   extensionHeaders=None, encoding_type=None):
+                   extensionHeaders=None, encoding_type=None, isAttachCrc64=False):
         self.log_client.log(INFO, 'enter resume upload file...')
         self._assert_not_null(bucketName, 'bucketName is empty')
         self._assert_not_null(objectKey, 'objectKey is empty')
         self._assert_not_null(uploadFile, 'uploadFile is empty')
 
         return _resume_upload(bucketName, objectKey, uploadFile, partSize, taskNum, enableCheckpoint, checkpointFile,
                               checkSum, metadata, progressCallback, self, headers,
-                              extensionHeaders=extensionHeaders, encoding_type=encoding_type)
+                              extensionHeaders=extensionHeaders, encoding_type=encoding_type, isAttachCrc64=isAttachCrc64)
 
     @funcCache
     def _downloadFileWithNotifier(self, bucketName, objectKey, downloadFile=None, partSize=5 * 1024 * 1024, taskNum=1,

diff --git a/src/obs/const.py b/src/obs/const.py
@@ -96,7 +96,7 @@
 DEFAULT_TASK_NUM = 8
 DEFAULT_TASK_QUEUE_SIZE = 20000
 
-OBS_SDK_VERSION = '3.24.6'
+OBS_SDK_VERSION = '3.24.6.1'
 
 V2_META_HEADER_PREFIX = 'x-amz-meta-'
 V2_HEADER_PREFIX = 'x-amz-'

diff --git a/src/obs/transfer.py b/src/obs/transfer.py
@@ -34,11 +34,13 @@
 
 
 def _resume_upload(bucketName, objectKey, uploadFile, partSize, taskNum, enableCheckPoint, checkPointFile, checkSum,
-                   metadata, progressCallback, obsClient, headers, extensionHeaders=None, encoding_type=None):
+                   metadata, progressCallback, obsClient, headers, extensionHeaders=None, encoding_type=None,
+                   isAttachCrc64=False):
     upload_operation = uploadOperation(util.to_string(bucketName), util.to_string(objectKey),
                                        util.to_string(uploadFile), partSize, taskNum, enableCheckPoint,
                                        util.to_string(checkPointFile), checkSum, metadata, progressCallback, obsClient,
-                                       headers, extensionHeaders=extensionHeaders, encoding_type=encoding_type)
+                                       headers, extensionHeaders=extensionHeaders, encoding_type=encoding_type,
+                                       isAttachCrc64=isAttachCrc64)
     return upload_operation._upload()
 
 
@@ -126,7 +128,8 @@ def _write_record(self, record):
 
 class uploadOperation(Operation):
     def __init__(self, bucketName, objectKey, uploadFile, partSize, taskNum, enableCheckPoint, checkPointFile,
-                 checkSum, metadata, progressCallback, obsClient, headers, extensionHeaders=None, encoding_type=None):
+                 checkSum, metadata, progressCallback, obsClient, headers, extensionHeaders=None, encoding_type=None,
+                 isAttachCrc64=False):
         if enableCheckPoint and not checkPointFile:
             checkPointFile = uploadFile + '.upload_record'
         super(uploadOperation, self).__init__(bucketName, objectKey, uploadFile, partSize, taskNum, enableCheckPoint,
@@ -136,6 +139,7 @@ def __init__(self, bucketName, objectKey, uploadFile, partSize, taskNum, enableC
         self.headers = headers
         self.extensionHeaders = extensionHeaders
         self.encoding_type = encoding_type
+        self.isAttachCrc64 = isAttachCrc64
 
         try:
             self.size = os.path.getsize(self.fileName)
@@ -229,13 +233,18 @@ def _upload(self):
                         raise Exception('some parts are failed when upload. Please try again')
 
             part_Etags = []
-            for part in self._record['partEtags']:
-                part_Etags.append(CompletePart(partNum=part['partNum'], etag=part['etag']))
-                self.obsClient.log_client.log(INFO, 'Completing to upload multi_parts')
+            if self.isAttachCrc64:
+                for part in self._record['partEtags']:
+                    part_Etags.append(CompletePart(partNum=part['partNum'], etag=part['etag'], crc64=part['crc64'], size=part['size']))
+            else:
+                for part in self._record['partEtags']:
+                    part_Etags.append(CompletePart(partNum=part['partNum'], etag=part['etag']))
+            self.obsClient.log_client.log(INFO, 'Completing to upload multi_parts')
             resp = self.obsClient.completeMultipartUpload(self.bucketName, self.objectKey, self._record['uploadId'],
                                                           CompleteMultipartUploadRequest(part_Etags),
                                                           extensionHeaders=self.extensionHeaders,
-                                                          encoding_type=self.encoding_type)
+                                                          encoding_type=self.encoding_type,
+                                                          isAttachCrc64=self.isAttachCrc64)
             self._upload_handle_response(resp)
             return resp
         finally:
@@ -389,7 +398,7 @@ def _upload_part(self, part):
                 resp = self.get_upload_part_resp(part)
                 if resp.status < 300:
                     self._record['uploadParts'][part['partNumber'] - 1]['isCompleted'] = True
-                    self._record['partEtags'].append(CompletePart(util.to_int(part['partNumber']), resp.body.etag))
+                    self._record['partEtags'].append(CompletePart(util.to_int(part['partNumber']), resp.body.etag, resp.body.crc64, util.to_int(part['length'])))
                     if self.enableCheckPoint:
                         with self._lock:
                             self._write_record(self._record)
@@ -410,7 +419,7 @@ def get_upload_part_resp(self, part):
                                                       isFile=True, partSize=part['length'],
                                                       offset=part['offset'], notifier=self.notifier,
                                                       extensionHeaders=self.extensionHeaders,
-                                                      sseHeader=self.headers.sseHeader)
+                                                      sseHeader=self.headers.sseHeader, isAttachCrc64=self.isAttachCrc64)
 
 
 class downloadOperation(Operation):

diff --git a/src/setup.py b/src/setup.py
@@ -22,7 +22,7 @@
 
 setup(
     name='esdk-obs-python',
-    version='3.24.3',
+    version='3.24.6.1',
     packages=find_packages(exclude=['tests']),
     zip_safe=False,
     description='OBS Python SDK',

diff --git a/src/tests/test_obs_client.py b/src/tests/test_obs_client.py
@@ -585,7 +585,7 @@ def test_uploadPart_with_crc64(self):
         part1 = CompletePart(partNum=1, etag=put_result.body.etag, crc64=put_result.body.crc64, size=1024 * 1024)
         completeMultipartUploadRequest = CompleteMultipartUploadRequest(parts=[part1])
         complete_result = crc64Client.completeMultipartUpload(test_config["bucketName"], object_name, uploadId,
-                                                            completeMultipartUploadRequest, isAttachCrc64=True)
+                                                              completeMultipartUploadRequest, isAttachCrc64=True)
         assert int(complete_result.body.crc64) == crc64
         get_result = crc64Client.getObject(test_config["bucketName"], object_name)
         assert int(get_result.body.crc64) == crc64
@@ -645,6 +645,19 @@ def test_getRangeObject_crc64(self):
 
         obsClient.deleteObject(test_config["bucketName"], object_name)
 
+    def test_uploadFile_with_crc64(self):
+        client_type, crc64Client, obsClient = self.get_client()
+        object_name = "test_crc64_object"
+        conftest.gen_random_file(object_name, 15 * 1024)
+        crc64 = util.calculate_file_crc64(test_config["path_prefix"] + object_name)
+        put_result = obsClient.uploadFile(test_config["bucketName"], object_name,
+                                          test_config["path_prefix"] + object_name, 5 * 1024 * 1024, 2, True,
+                                          isAttachCrc64=True)
+        assert put_result.status == 200
+        get_result = crc64Client.getObject(test_config["bucketName"], object_name)
+        assert int(get_result.body.crc64) == crc64
+        obsClient.deleteObject(test_config["bucketName"], object_name)
+
 
 if __name__ == "__main__":
     pytest.main(["-v", 'test_obs_client.py::TestOBSClient::test_uploadFile_with_metadata'])