diff --git a/README.md b/README.md index c2ad8ac..7b61969 100644 --- a/README.md +++ b/README.md @@ -10,13 +10,13 @@ repositories { } // Append dependency -implementation("com.icerockdev:storage-service:0.5.1") +implementation("com.icerockdev:storage-service:0.5.2") ```` ## Library usage Lib include tools for: - s3 interface - - put object to bucket + - put object (supports metadata) to bucket - delete the object from bucket - list by prefix in bucket - create bucket diff --git a/storage-service/build.gradle.kts b/storage-service/build.gradle.kts index 7491678..62454aa 100644 --- a/storage-service/build.gradle.kts +++ b/storage-service/build.gradle.kts @@ -15,7 +15,7 @@ apply(plugin = "java") apply(plugin = "kotlin") group = "com.icerockdev" -version = "0.5.1" +version = "0.5.2" val sourcesJar by tasks.registering(Jar::class) { archiveClassifier.set("sources") diff --git a/storage-service/src/main/kotlin/com/icerockdev/service/storage/s3/IS3Storage.kt b/storage-service/src/main/kotlin/com/icerockdev/service/storage/s3/IS3Storage.kt index 62a0188..deb3f6e 100644 --- a/storage-service/src/main/kotlin/com/icerockdev/service/storage/s3/IS3Storage.kt +++ b/storage-service/src/main/kotlin/com/icerockdev/service/storage/s3/IS3Storage.kt @@ -4,9 +4,10 @@ package com.icerockdev.service.storage.s3 +import software.amazon.awssdk.core.ResponseInputStream import software.amazon.awssdk.services.s3.S3Configuration +import software.amazon.awssdk.services.s3.model.GetObjectResponse import software.amazon.awssdk.services.s3.model.S3Object -import java.io.FilterInputStream import java.io.InputStream import java.net.URI import java.time.Duration @@ -14,7 +15,7 @@ import java.util.UUID // TODO: change return type for support file storage (if needed) interface IS3Storage { - fun get(bucket: String, key: String): FilterInputStream? + fun get(bucket: String, key: String): ResponseInputStream? fun getBytes(bucket: String, key: String): ByteArray? @@ -34,9 +35,9 @@ interface IS3Storage { fun objectExists(bucket: String, key: String): Boolean - fun put(bucket: String, key: String, stream: InputStream): Boolean + fun put(bucket: String, key: String, stream: InputStream, metadata: Map? = null): Boolean - fun put(bucket: String, key: String, byteArray: ByteArray): Boolean + fun put(bucket: String, key: String, byteArray: ByteArray, metadata: Map? = null): Boolean fun copy(srcBucket: String, srcKey: String, dstBucket: String, dstKey: String): Boolean diff --git a/storage-service/src/main/kotlin/com/icerockdev/service/storage/s3/S3StorageImpl.kt b/storage-service/src/main/kotlin/com/icerockdev/service/storage/s3/S3StorageImpl.kt index 91438d6..f4d8e6d 100644 --- a/storage-service/src/main/kotlin/com/icerockdev/service/storage/s3/S3StorageImpl.kt +++ b/storage-service/src/main/kotlin/com/icerockdev/service/storage/s3/S3StorageImpl.kt @@ -4,8 +4,14 @@ package com.icerockdev.service.storage.s3 +import org.slf4j.LoggerFactory +import software.amazon.awssdk.core.ResponseInputStream +import software.amazon.awssdk.core.sync.RequestBody +import software.amazon.awssdk.services.s3.S3Client +import software.amazon.awssdk.services.s3.model.* +import software.amazon.awssdk.services.s3.presigner.S3Presigner +import software.amazon.awssdk.services.s3.presigner.model.GetObjectPresignRequest import java.io.BufferedInputStream -import java.io.FilterInputStream import java.io.InputStream import java.net.MalformedURLException import java.net.URI @@ -13,35 +19,13 @@ import java.net.URLConnection import java.net.URLEncoder import java.nio.charset.StandardCharsets import java.time.Duration -import org.slf4j.LoggerFactory -import software.amazon.awssdk.core.sync.RequestBody -import software.amazon.awssdk.services.s3.S3Client -import software.amazon.awssdk.services.s3.model.BucketAlreadyExistsException -import software.amazon.awssdk.services.s3.model.BucketAlreadyOwnedByYouException -import software.amazon.awssdk.services.s3.model.CopyObjectRequest -import software.amazon.awssdk.services.s3.model.CreateBucketRequest -import software.amazon.awssdk.services.s3.model.DeleteBucketRequest -import software.amazon.awssdk.services.s3.model.DeleteObjectRequest -import software.amazon.awssdk.services.s3.model.GetObjectRequest -import software.amazon.awssdk.services.s3.model.GetUrlRequest -import software.amazon.awssdk.services.s3.model.HeadBucketRequest -import software.amazon.awssdk.services.s3.model.HeadObjectRequest -import software.amazon.awssdk.services.s3.model.ListObjectsV2Request -import software.amazon.awssdk.services.s3.model.NoSuchBucketException -import software.amazon.awssdk.services.s3.model.NoSuchKeyException -import software.amazon.awssdk.services.s3.model.ObjectCannedACL -import software.amazon.awssdk.services.s3.model.PutObjectRequest -import software.amazon.awssdk.services.s3.model.S3Exception -import software.amazon.awssdk.services.s3.model.S3Object -import software.amazon.awssdk.services.s3.presigner.S3Presigner -import software.amazon.awssdk.services.s3.presigner.model.GetObjectPresignRequest /** * TODO: implements S3AsyncClient and change to coroutine usage */ class S3StorageImpl(private val client: S3Client, private val preSigner: S3Presigner) : IS3Storage { - override fun get(bucket: String, key: String): FilterInputStream? { + override fun get(bucket: String, key: String): ResponseInputStream? { return try { client.getObject( GetObjectRequest.builder() @@ -68,10 +52,11 @@ class S3StorageImpl(private val client: S3Client, private val preSigner: S3Presi .bucket(bucket) .key(key) .build() - preSigner.presignGetObject(GetObjectPresignRequest.builder() - .signatureDuration(duration) - .getObjectRequest(getObjectRequest) - .build() + preSigner.presignGetObject( + GetObjectPresignRequest.builder() + .signatureDuration(duration) + .getObjectRequest(getObjectRequest) + .build() ).url().toExternalForm() } catch (e: Throwable) { null @@ -167,16 +152,16 @@ class S3StorageImpl(private val client: S3Client, private val preSigner: S3Presi } } - override fun put(bucket: String, key: String, stream: InputStream): Boolean { - return put(bucket, key, stream.buffered()) + override fun put(bucket: String, key: String, stream: InputStream, metadata: Map?): Boolean { + return put(bucket, key, stream.buffered(), metadata) } - override fun put(bucket: String, key: String, byteArray: ByteArray): Boolean { + override fun put(bucket: String, key: String, byteArray: ByteArray, metadata: Map?): Boolean { val stream = byteArray.inputStream().buffered() - return put(bucket, key, stream) + return put(bucket, key, stream, metadata) } - private fun put(bucket: String, key: String, stream: BufferedInputStream): Boolean { + private fun put(bucket: String, key: String, stream: BufferedInputStream, metadata: Map?): Boolean { val contentType = URLConnection.guessContentTypeFromStream(stream) val request = PutObjectRequest.builder() @@ -185,6 +170,7 @@ class S3StorageImpl(private val client: S3Client, private val preSigner: S3Presi .acl(ObjectCannedACL.PUBLIC_READ) .contentEncoding("UTF-8") .contentType(contentType) + .metadata(metadata ?: emptyMap()) .build() return try { diff --git a/storage-service/src/test/kotlin/S3StorageTest.kt b/storage-service/src/test/kotlin/S3StorageTest.kt index dacffcd..d3c4f5e 100644 --- a/storage-service/src/test/kotlin/S3StorageTest.kt +++ b/storage-service/src/test/kotlin/S3StorageTest.kt @@ -103,9 +103,7 @@ class S3StorageTest { storage.createBucket(bucketName) } - val fileName = storage.generateFileKey() - val stream = classLoader.getResourceAsStream(dotenv["JPG_TEST_OBJECT"]) - ?: throw Exception("JPG File not found") + val (fileName, stream) = getFile(FileType.JPG) // check wrong cases assertFalse { @@ -198,15 +196,9 @@ class S3StorageTest { storage.createBucket(bucketName) } - val jpgFileName = storage.generateFileKey() - val pngFileName = storage.generateFileKey() - val gifFileName = storage.generateFileKey() - val jpgStream = classLoader.getResourceAsStream(dotenv["JPG_TEST_OBJECT"]) - ?: throw Exception("JPG File not found") - val pngStream = classLoader.getResourceAsStream(dotenv["PNG_TEST_OBJECT"]) - ?: throw Exception("PNG File not found") - val gifStream = classLoader.getResourceAsStream(dotenv["GIF_TEST_OBJECT"]) - ?: throw Exception("GIF File not found") + val (jpgFileName, jpgStream) = getFile(FileType.JPG) + val (pngFileName, pngStream) = getFile(FileType.PNG) + val (gifFileName, gifStream) = getFile(FileType.GIF) // Check wrong cases assertFalse(storage.objectExists(bucketName, jpgFileName)) @@ -231,9 +223,9 @@ class S3StorageTest { val pngObject = storage.get(bucketName, pngFileName) val gifObject = storage.get(bucketName, gifFileName) - assertEquals(getContentType(jpgObject?.buffered()!!), "image/jpeg") - assertEquals(getContentType(pngObject?.buffered()!!), "image/png") - assertEquals(getContentType(gifObject?.buffered()!!), "image/gif") + assertEquals("image/jpeg", jpgObject?.response()?.contentType()) + assertEquals("image/png", pngObject?.response()?.contentType()) + assertEquals("image/gif", gifObject?.response()?.contentType()) assertTrue(storage.delete(bucketName, jpgFileName)) assertTrue(storage.delete(bucketName, pngFileName)) @@ -244,6 +236,78 @@ class S3StorageTest { } } + @Test + fun testFileSizeAndType() { + // init storage + if (!storage.bucketExist(bucketName)) { + storage.createBucket(bucketName) + } + + val (jpgFileName, jpgStream) = getFile(FileType.JPG) + val jpgFileByteArray = jpgStream.readAllBytes() + + jpgStream.close() + + // Check wrong cases + assertFalse(storage.objectExists(bucketName, jpgFileName)) + + // Put object to storage + assertTrue(storage.put(bucketName, jpgFileName, jpgFileByteArray.inputStream())) + + // Check object exist + assertTrue(storage.objectExists(bucketName, jpgFileName)) + + val jpgObject = storage.get(bucketName, jpgFileName) + + assertEquals("image/jpeg", jpgObject?.response()?.contentType()) + + assertEquals(jpgFileByteArray.size.toLong(), jpgObject?.response()?.contentLength()) + + assertTrue { + storage.deleteBucketWithObjects(bucketName) + } + } + + @Test + fun testMetadata() { + // init storage + if (!storage.bucketExist(bucketName)) { + storage.createBucket(bucketName) + } + + val metadata = mapOf("attribute1" to "testValue1", "attribute2" to "testValue2") + + val (jpgFileName, jpgStream) = getFile(FileType.JPG) + + // Check wrong cases + assertFalse(storage.objectExists(bucketName, jpgFileName)) + + // Put object to storage + assertTrue(storage.put(bucketName, jpgFileName, jpgStream, metadata)) + jpgStream.close() + + // Check object exist + assertTrue(storage.objectExists(bucketName, jpgFileName)) + + val jpgObject = storage.get(bucketName, jpgFileName) + + assertEquals(metadata, jpgObject?.response()?.metadata()) + + // copy testing + val copyFileName = storage.generateFileKey() + // copy to current bucket + assertTrue { + storage.copy(bucketName, jpgFileName, bucketName, copyFileName) + } + + val copyObject = storage.get(bucketName, copyFileName) + + assertEquals(metadata, copyObject?.response()?.metadata()) + + assertTrue { + storage.deleteBucketWithObjects(bucketName) + } + } @Test fun testShareGetURL() { @@ -267,8 +331,8 @@ class S3StorageTest { } assertEquals( - storage.getUrl(URI.create(dotenv["S3_ENDPOINT"]!!), bucketName, fileName), - "http://127.0.0.30:9000/${bucketName}/${fileName}" + "${dotenv["S3_ENDPOINT"]}/${bucketName}/${fileName}", + storage.getUrl(URI.create(dotenv["S3_ENDPOINT"]!!), bucketName, fileName) ) runBlocking { @@ -286,15 +350,15 @@ class S3StorageTest { val successResponse = httpClient.send(httpRequest, HttpResponse.BodyHandlers.ofString()) val successHeaders = successResponse.headers() - assertEquals(successResponse.statusCode(), 200) - assertEquals(successHeaders.firstValue("content-type").get(), "image/jpeg") + assertEquals(200, successResponse.statusCode()) + assertEquals("image/jpeg", successHeaders.firstValue("content-type").get()) Thread.sleep(2000) val failResponse = httpClient.send(httpRequest, HttpResponse.BodyHandlers.ofString()) val failHeaders = failResponse.headers() - assertEquals(failResponse.statusCode(), 403) - assertEquals(failHeaders.firstValue("content-type").get(), "application/xml") + assertEquals(403, failResponse.statusCode()) + assertEquals("application/xml", failHeaders.firstValue("content-type").get()) } assertTrue { @@ -353,6 +417,23 @@ class S3StorageTest { s3.close() } + private fun getFile(fileType: FileType): Pair { + val key = storage.generateFileKey() + val fileName = when (fileType) { + FileType.JPG -> dotenv["JPG_TEST_OBJECT"] ?: throw Exception("JPG File not found") + FileType.GIF -> dotenv["GIF_TEST_OBJECT"] ?: throw Exception("GIF File not found") + FileType.PNG -> dotenv["PNG_TEST_OBJECT"] ?: throw Exception("PNG File not found") + } + + return key to (classLoader.getResourceAsStream(fileName) ?: throw Exception("File not readable")) + } + + private enum class FileType { + JPG, + GIF, + PNG; + } + @Throws(IOException::class) private fun isEqual(i1: InputStream, i2: InputStream): Boolean { val ch1: ReadableByteChannel = Channels.newChannel(i1)