default part-size to 64MB and provide a way to override the default p…

…art size (#759)

docs/API.md

@@ -64,6 +64,7 @@ __Parameters__
 |`region`    | _string_  |Set this value to override region cache. (Optional)|
 |`transport`    | _string_  |Set this value to pass in a custom transport. (Optional)|
 |`sessionToken`    | _string_  |Set this value to provide x-amz-security-token (AWS S3 specific). (Optional)|
+|`partSize`    | _number_  |Set this value to override default part size of 64MB for multipart uploads. (Optional)|
 
 
 __Example__

docs/zh_CN/API.md

@@ -61,6 +61,9 @@ __参数__
 |`secretKey`  |  _string_   | secretKey是你账户的密码。|
 |`useSSL`    | _bool_    |如果是true，则用的是https而不是http,默认值是true。 |
 |`region`    | _string_  |设置该值以覆盖自动发现存储桶region。（可选）|
+|`transport`    | _string_  |Set this value to pass in a custom transport. (Optional) - To be translated |
+|`sessionToken`    | _string_  |Set this value to provide x-amz-security-token (AWS S3 specific). (Optional) - To be translated|
+|`partSize`    | _number_  |Set this value to override default part size of 64MB for multipart uploads. (Optional) - To be translated|
 
 
 __示例__

src/main/minio.js

@@ -129,7 +129,18 @@ export class Client {
       this.region = params.region
     }
 
-    this.minimumPartSize = 5*1024*1024
+    this.partSize = 64*1024*1024
+    if (params.partSize) {
+      this.partSize = params.partSize
+      this.overRidePartSize = true
+    }
+    if (this.partSize < 5*1024*1024) {
+      throw new errors.InvalidArgumentError(`Part size should be greater than 5MB`)
+    }
+    if (this.partSize > 5*1024*1024*1024) {
+      throw new errors.InvalidArgumentError(`Part size should be less than 5GB`)
+    }
+
     this.maximumPartSize = 5*1024*1024*1024
     this.maxObjectSize = 5*1024*1024*1024*1024
     // SHA256 is enabled only for authenticated http requests. If the request is authenticated
@@ -244,20 +255,26 @@ export class Client {
     this.userAgent = `${this.userAgent} ${appName}/${appVersion}`
   }
 
-  // partSize will be atleast minimumPartSize or a multiple of minimumPartSize
-  // for size <= 50GiB partSize is always 5MiB (10000*5MiB = 50GiB)
-  // for size > 50GiB partSize will be a multiple of 5MiB
-  // for size = 5TiB partSize will be 525MiB
+  // Calculate part size given the object size. Part size will be atleast this.partSize
   calculatePartSize(size) {
     if (!isNumber(size)) {
       throw new TypeError('size should be of type "number"')
     }
     if (size > this.maxObjectSize) {
       throw new TypeError(`size should not be more than ${this.maxObjectSize}`)
     }
-    var partSize = Math.ceil(size/10000)
-    partSize = Math.ceil(partSize/this.minimumPartSize) * this.minimumPartSize
-    return partSize
+    if (this.overRidePartSize) {
+      return this.partSize
+    }
+    var partSize = this.partSize
+    for (;;) { 			// while(true) {...} throws linting error.
+      // If partSize is big enough to accomodate the object size, then use it.
+      if ((partSize * 10000) > size) {
+        return partSize
+      }
+      // Try part sizes as 64MB, 80MB, 96MB etc.
+      partSize += 16*1024*1024
+    }
   }
 
   // log the request, response, error
@@ -892,7 +909,7 @@ export class Client {
         if (size > this.maxObjectSize) {
           return cb(new Error(`${filePath} size : ${stats.size}, max allowed size : 5TB`))
         }
-        if (size < this.minimumPartSize) {
+        if (size <= this.partSize) {
           // simple PUT request, no multipart
           var multipart = false
           var uploader = this.getUploader(bucketName, objectName, metaData, multipart)
@@ -1053,7 +1070,7 @@ export class Client {
 
     size = this.calculatePartSize(size)
 
-    // s3 requires that all non-end chunks be at least `this.minimumPartSize`,
+    // s3 requires that all non-end chunks be at least `this.partSize`,
     // so we chunk the stream until we hit either that size or the end before
     // we flush it to s3.
     let chunker = BlockStream2({size, zeroPadding: false})

src/test/functional/functional-tests.js

@@ -66,6 +66,7 @@ describe('functional tests', function() {
   var dataDir = process.env['MINT_DATA_DIR']
 
   var client = new minio.Client(playConfig)
+  var clientMultipartTest = new minio.Client(Object.assign({partSize: 5*1024*1024}, playConfig))
   var usEastConfig = playConfig
   usEastConfig.region = 'us-east-1'
   var clientUsEastRegion = new minio.Client(usEastConfig)
@@ -248,7 +249,6 @@ describe('functional tests', function() {
 
   })
   describe('tests for putObject copyObject getObject getPartialObject statObject removeObject', function() {
-
     var tmpFileUpload = `${tmpDir}/${_100kbObjectName}`
     step(`fPutObject(bucketName, objectName, filePath, metaData, callback)_bucketName:${bucketName}, objectName:${_100kbObjectName}, filePath: ${tmpFileUpload}_`, done => {
       fs.writeFileSync(tmpFileUpload, _100kb)
@@ -411,6 +411,36 @@ describe('functional tests', function() {
       })
     })
 
+    step(`statObject(bucketName, objectName, cb)_bucketName:${bucketName}, objectName:${_6mbObjectName}_ - ensure non-multipart upload`, done => {
+      client.statObject(bucketName, _6mbObjectName, (e, stat) => {
+        if (e) return done(e)
+        var etagstrs = stat.etag.split("-")
+        if (etagstrs.length != 1) {
+          return done(new Error('etag should indicate that multipart upload was not used'))
+        }
+        done()
+      })
+    })
+
+    step(`putObject(bucketName, objectName, stream, cb)_bucketName:${bucketName}, objectName:${_6mbObjectName}_ - do multipart upload`, done => {
+      var stream = readableStream(_6mb)
+      clientMultipartTest.putObject(bucketName, _6mbObjectName, stream, done)
+    })
+
+    step(`statObject(bucketName, objectName, cb)_bucketName:${bucketName}, objectName:${_6mbObjectName}_ - ensure multipart upload with 2 parts`, done => {
+      client.statObject(bucketName, _6mbObjectName, (e, stat) => {
+        if (e) return done(e)
+        var etagstrs = stat.etag.split("-")
+        if (etagstrs.length != 2) {
+          return done(new Error('etag should indicate that multipart upload was used'))
+        }
+        if (etagstrs[1] != "2") {
+          return done(new Error('etag should indicate that two parts were used for upload'))
+        }
+        done()
+      })
+    })
+
     step(`getObject(bucketName, objectName, cb)_bucketName:${bucketName} non-existent object`, done => {
       client.getObject(bucketName, 'an-object-that-does-not-exist', (e, stream) => {
         if (stream) return done(new Error("on errors the stream object should not exist"))