metadata support and change return type for get object (#14)

* metadata support and change return type for get object (now can use content type and length) * remove unused import
icerockdev · Jul 26, 2021 · ae35888 · ae35888
1 parent 90bcc24
commit ae35888
Show file tree

Hide file tree

Showing 5 changed files with 129 additions and 61 deletions.
diff --git a/README.md b/README.md
@@ -10,13 +10,13 @@ repositories {
 }
 
 // Append dependency
-implementation("com.icerockdev:storage-service:0.5.1")
+implementation("com.icerockdev:storage-service:0.5.2")
 ````
 
 ## Library usage
 Lib include tools for:
  - s3 interface
-    - put object to bucket
+    - put object (supports metadata) to bucket
     - delete the object from bucket
     - list by prefix in bucket
     - create bucket

diff --git a/storage-service/build.gradle.kts b/storage-service/build.gradle.kts
@@ -15,7 +15,7 @@ apply(plugin = "java")
 apply(plugin = "kotlin")
 
 group = "com.icerockdev"
-version = "0.5.1"
+version = "0.5.2"
 
 val sourcesJar by tasks.registering(Jar::class) {
     archiveClassifier.set("sources")

diff --git a/storage-service/src/main/kotlin/com/icerockdev/service/storage/s3/IS3Storage.kt b/storage-service/src/main/kotlin/com/icerockdev/service/storage/s3/IS3Storage.kt
@@ -4,17 +4,18 @@
 
 package com.icerockdev.service.storage.s3
 
+import software.amazon.awssdk.core.ResponseInputStream
 import software.amazon.awssdk.services.s3.S3Configuration
+import software.amazon.awssdk.services.s3.model.GetObjectResponse
 import software.amazon.awssdk.services.s3.model.S3Object
-import java.io.FilterInputStream
 import java.io.InputStream
 import java.net.URI
 import java.time.Duration
 import java.util.UUID
 
 // TODO: change return type for support file storage (if needed)
 interface IS3Storage {
-    fun get(bucket: String, key: String): FilterInputStream?
+    fun get(bucket: String, key: String): ResponseInputStream<GetObjectResponse>?
 
     fun getBytes(bucket: String, key: String): ByteArray?
 
@@ -34,9 +35,9 @@ interface IS3Storage {
 
     fun objectExists(bucket: String, key: String): Boolean
 
-    fun put(bucket: String, key: String, stream: InputStream): Boolean
+    fun put(bucket: String, key: String, stream: InputStream, metadata: Map<String, String>? = null): Boolean
 
-    fun put(bucket: String, key: String, byteArray: ByteArray): Boolean
+    fun put(bucket: String, key: String, byteArray: ByteArray, metadata: Map<String, String>? = null): Boolean
 
     fun copy(srcBucket: String, srcKey: String, dstBucket: String, dstKey: String): Boolean
 

diff --git a/storage-service/src/main/kotlin/com/icerockdev/service/storage/s3/S3StorageImpl.kt b/storage-service/src/main/kotlin/com/icerockdev/service/storage/s3/S3StorageImpl.kt
@@ -4,44 +4,28 @@
 
 package com.icerockdev.service.storage.s3
 
+import org.slf4j.LoggerFactory
+import software.amazon.awssdk.core.ResponseInputStream
+import software.amazon.awssdk.core.sync.RequestBody
+import software.amazon.awssdk.services.s3.S3Client
+import software.amazon.awssdk.services.s3.model.*
+import software.amazon.awssdk.services.s3.presigner.S3Presigner
+import software.amazon.awssdk.services.s3.presigner.model.GetObjectPresignRequest
 import java.io.BufferedInputStream
-import java.io.FilterInputStream
 import java.io.InputStream
 import java.net.MalformedURLException
 import java.net.URI
 import java.net.URLConnection
 import java.net.URLEncoder
 import java.nio.charset.StandardCharsets
 import java.time.Duration
-import org.slf4j.LoggerFactory
-import software.amazon.awssdk.core.sync.RequestBody
-import software.amazon.awssdk.services.s3.S3Client
-import software.amazon.awssdk.services.s3.model.BucketAlreadyExistsException
-import software.amazon.awssdk.services.s3.model.BucketAlreadyOwnedByYouException
-import software.amazon.awssdk.services.s3.model.CopyObjectRequest
-import software.amazon.awssdk.services.s3.model.CreateBucketRequest
-import software.amazon.awssdk.services.s3.model.DeleteBucketRequest
-import software.amazon.awssdk.services.s3.model.DeleteObjectRequest
-import software.amazon.awssdk.services.s3.model.GetObjectRequest
-import software.amazon.awssdk.services.s3.model.GetUrlRequest
-import software.amazon.awssdk.services.s3.model.HeadBucketRequest
-import software.amazon.awssdk.services.s3.model.HeadObjectRequest
-import software.amazon.awssdk.services.s3.model.ListObjectsV2Request
-import software.amazon.awssdk.services.s3.model.NoSuchBucketException
-import software.amazon.awssdk.services.s3.model.NoSuchKeyException
-import software.amazon.awssdk.services.s3.model.ObjectCannedACL
-import software.amazon.awssdk.services.s3.model.PutObjectRequest
-import software.amazon.awssdk.services.s3.model.S3Exception
-import software.amazon.awssdk.services.s3.model.S3Object
-import software.amazon.awssdk.services.s3.presigner.S3Presigner
-import software.amazon.awssdk.services.s3.presigner.model.GetObjectPresignRequest
 
 /**
  * TODO: implements S3AsyncClient and change to coroutine usage
  */
 class S3StorageImpl(private val client: S3Client, private val preSigner: S3Presigner) : IS3Storage {
 
-    override fun get(bucket: String, key: String): FilterInputStream? {
+    override fun get(bucket: String, key: String): ResponseInputStream<GetObjectResponse>? {
         return try {
             client.getObject(
                 GetObjectRequest.builder()
@@ -68,10 +52,11 @@ class S3StorageImpl(private val client: S3Client, private val preSigner: S3Presi
                 .bucket(bucket)
                 .key(key)
                 .build()
-            preSigner.presignGetObject(GetObjectPresignRequest.builder()
-                .signatureDuration(duration)
-                .getObjectRequest(getObjectRequest)
-                .build()
+            preSigner.presignGetObject(
+                GetObjectPresignRequest.builder()
+                    .signatureDuration(duration)
+                    .getObjectRequest(getObjectRequest)
+                    .build()
             ).url().toExternalForm()
         } catch (e: Throwable) {
             null
@@ -167,16 +152,16 @@ class S3StorageImpl(private val client: S3Client, private val preSigner: S3Presi
         }
     }
 
-    override fun put(bucket: String, key: String, stream: InputStream): Boolean {
-        return put(bucket, key, stream.buffered())
+    override fun put(bucket: String, key: String, stream: InputStream, metadata: Map<String, String>?): Boolean {
+        return put(bucket, key, stream.buffered(), metadata)
     }
 
-    override fun put(bucket: String, key: String, byteArray: ByteArray): Boolean {
+    override fun put(bucket: String, key: String, byteArray: ByteArray, metadata: Map<String, String>?): Boolean {
         val stream = byteArray.inputStream().buffered()
-        return put(bucket, key, stream)
+        return put(bucket, key, stream, metadata)
     }
 
-    private fun put(bucket: String, key: String, stream: BufferedInputStream): Boolean {
+    private fun put(bucket: String, key: String, stream: BufferedInputStream, metadata: Map<String, String>?): Boolean {
         val contentType = URLConnection.guessContentTypeFromStream(stream)
 
         val request = PutObjectRequest.builder()
@@ -185,6 +170,7 @@ class S3StorageImpl(private val client: S3Client, private val preSigner: S3Presi
             .acl(ObjectCannedACL.PUBLIC_READ)
             .contentEncoding("UTF-8")
             .contentType(contentType)
+            .metadata(metadata ?: emptyMap())
             .build()
 
         return try {

diff --git a/storage-service/src/test/kotlin/S3StorageTest.kt b/storage-service/src/test/kotlin/S3StorageTest.kt
@@ -103,9 +103,7 @@ class S3StorageTest {
             storage.createBucket(bucketName)
         }
 
-        val fileName = storage.generateFileKey()
-        val stream = classLoader.getResourceAsStream(dotenv["JPG_TEST_OBJECT"])
-            ?: throw Exception("JPG File not found")
+        val (fileName, stream) = getFile(FileType.JPG)
 
         // check wrong cases
         assertFalse {
@@ -198,15 +196,9 @@ class S3StorageTest {
             storage.createBucket(bucketName)
         }
 
-        val jpgFileName = storage.generateFileKey()
-        val pngFileName = storage.generateFileKey()
-        val gifFileName = storage.generateFileKey()
-        val jpgStream = classLoader.getResourceAsStream(dotenv["JPG_TEST_OBJECT"])
-            ?: throw Exception("JPG File not found")
-        val pngStream = classLoader.getResourceAsStream(dotenv["PNG_TEST_OBJECT"])
-            ?: throw Exception("PNG File not found")
-        val gifStream = classLoader.getResourceAsStream(dotenv["GIF_TEST_OBJECT"])
-            ?: throw Exception("GIF File not found")
+        val (jpgFileName, jpgStream) = getFile(FileType.JPG)
+        val (pngFileName, pngStream) = getFile(FileType.PNG)
+        val (gifFileName, gifStream) = getFile(FileType.GIF)
 
         // Check wrong cases
         assertFalse(storage.objectExists(bucketName, jpgFileName))
@@ -231,9 +223,9 @@ class S3StorageTest {
         val pngObject = storage.get(bucketName, pngFileName)
         val gifObject = storage.get(bucketName, gifFileName)
 
-        assertEquals(getContentType(jpgObject?.buffered()!!), "image/jpeg")
-        assertEquals(getContentType(pngObject?.buffered()!!), "image/png")
-        assertEquals(getContentType(gifObject?.buffered()!!), "image/gif")
+        assertEquals("image/jpeg", jpgObject?.response()?.contentType())
+        assertEquals("image/png", pngObject?.response()?.contentType())
+        assertEquals("image/gif", gifObject?.response()?.contentType())
 
         assertTrue(storage.delete(bucketName, jpgFileName))
         assertTrue(storage.delete(bucketName, pngFileName))
@@ -244,6 +236,78 @@ class S3StorageTest {
         }
     }
 
+    @Test
+    fun testFileSizeAndType() {
+        // init storage
+        if (!storage.bucketExist(bucketName)) {
+            storage.createBucket(bucketName)
+        }
+
+        val (jpgFileName, jpgStream) = getFile(FileType.JPG)
+        val jpgFileByteArray = jpgStream.readAllBytes()
+
+        jpgStream.close()
+
+        // Check wrong cases
+        assertFalse(storage.objectExists(bucketName, jpgFileName))
+
+        // Put object to storage
+        assertTrue(storage.put(bucketName, jpgFileName, jpgFileByteArray.inputStream()))
+
+        // Check object exist
+        assertTrue(storage.objectExists(bucketName, jpgFileName))
+
+        val jpgObject = storage.get(bucketName, jpgFileName)
+
+        assertEquals("image/jpeg", jpgObject?.response()?.contentType())
+
+        assertEquals(jpgFileByteArray.size.toLong(), jpgObject?.response()?.contentLength())
+
+        assertTrue {
+            storage.deleteBucketWithObjects(bucketName)
+        }
+    }
+
+    @Test
+    fun testMetadata() {
+        // init storage
+        if (!storage.bucketExist(bucketName)) {
+            storage.createBucket(bucketName)
+        }
+
+        val metadata = mapOf("attribute1" to "testValue1", "attribute2" to "testValue2")
+
+        val (jpgFileName, jpgStream) = getFile(FileType.JPG)
+
+        // Check wrong cases
+        assertFalse(storage.objectExists(bucketName, jpgFileName))
+
+        // Put object to storage
+        assertTrue(storage.put(bucketName, jpgFileName, jpgStream, metadata))
+        jpgStream.close()
+
+        // Check object exist
+        assertTrue(storage.objectExists(bucketName, jpgFileName))
+
+        val jpgObject = storage.get(bucketName, jpgFileName)
+
+        assertEquals(metadata, jpgObject?.response()?.metadata())
+
+        // copy testing
+        val copyFileName = storage.generateFileKey()
+        // copy to current bucket
+        assertTrue {
+            storage.copy(bucketName, jpgFileName, bucketName, copyFileName)
+        }
+
+        val copyObject = storage.get(bucketName, copyFileName)
+
+        assertEquals(metadata, copyObject?.response()?.metadata())
+
+        assertTrue {
+            storage.deleteBucketWithObjects(bucketName)
+        }
+    }
 
     @Test
     fun testShareGetURL() {
@@ -267,8 +331,8 @@ class S3StorageTest {
         }
 
         assertEquals(
-            storage.getUrl(URI.create(dotenv["S3_ENDPOINT"]!!), bucketName, fileName),
-            "http://127.0.0.30:9000/${bucketName}/${fileName}"
+            "${dotenv["S3_ENDPOINT"]}/${bucketName}/${fileName}",
+            storage.getUrl(URI.create(dotenv["S3_ENDPOINT"]!!), bucketName, fileName)
         )
 
         runBlocking {
@@ -286,15 +350,15 @@ class S3StorageTest {
 
             val successResponse = httpClient.send(httpRequest, HttpResponse.BodyHandlers.ofString())
             val successHeaders = successResponse.headers()
-            assertEquals(successResponse.statusCode(), 200)
-            assertEquals(successHeaders.firstValue("content-type").get(), "image/jpeg")
+            assertEquals(200, successResponse.statusCode())
+            assertEquals("image/jpeg", successHeaders.firstValue("content-type").get())
 
             Thread.sleep(2000)
 
             val failResponse = httpClient.send(httpRequest, HttpResponse.BodyHandlers.ofString())
             val failHeaders = failResponse.headers()
-            assertEquals(failResponse.statusCode(), 403)
-            assertEquals(failHeaders.firstValue("content-type").get(), "application/xml")
+            assertEquals(403, failResponse.statusCode())
+            assertEquals("application/xml", failHeaders.firstValue("content-type").get())
         }
 
         assertTrue {
@@ -353,6 +417,23 @@ class S3StorageTest {
         s3.close()
     }
 
+    private fun getFile(fileType: FileType): Pair<String, InputStream> {
+        val key = storage.generateFileKey()
+        val fileName = when (fileType) {
+            FileType.JPG -> dotenv["JPG_TEST_OBJECT"] ?: throw Exception("JPG File not found")
+            FileType.GIF -> dotenv["GIF_TEST_OBJECT"] ?: throw Exception("GIF File not found")
+            FileType.PNG -> dotenv["PNG_TEST_OBJECT"] ?: throw Exception("PNG File not found")
+        }
+
+        return key to (classLoader.getResourceAsStream(fileName) ?: throw Exception("File not readable"))
+    }
+
+    private enum class FileType {
+        JPG,
+        GIF,
+        PNG;
+    }
+
     @Throws(IOException::class)
     private fun isEqual(i1: InputStream, i2: InputStream): Boolean {
         val ch1: ReadableByteChannel = Channels.newChannel(i1)