Merge pull request #4481 from gchq/gh-4479_aws_assume_role

#4479 Add ability to assume role for S3
gchq · Sep 26, 2024 · 3b7b77f · 3b7b77f
2 parents 31c85b6 + 3e0c86b
commit 3b7b77f
Show file tree

Hide file tree

Showing 16 changed files with 1,140 additions and 13 deletions.
diff --git a/build.gradle b/build.gradle
@@ -166,6 +166,7 @@ ext.libs = [
         aws_crt                             : "software.amazon.awssdk.crt:aws-crt:0.27.3",
         aws_s3_transfer_manager             : "software.amazon.awssdk:s3-transfer-manager", // version controlled by AWS BOM
         aws_sqs                             : "software.amazon.awssdk:sqs", // version controlled by AWS BOM
+        aws_sts                             : "software.amazon.awssdk:sts", // version controlled by AWS BOM
         bcrypt                              : 'de.svenkubiak:jBCrypt:0.4.1',
         bouncy_castle                       : 'org.bouncycastle:bcprov-jdk18on:1.78.1',
         caffeine                            : "com.github.ben-manes.caffeine:caffeine", // version controlled by dropwizard-dependencies

diff --git a/stroom-app/src/main/resources/ui/noauth/swagger/stroom.json b/stroom-app/src/main/resources/ui/noauth/swagger/stroom.json
@@ -10782,10 +10782,7 @@
             "content" : {
               "application/json" : {
                 "schema" : {
-                  "type" : "array",
-                  "items" : {
-                    "type" : "string"
-                  }
+                  "$ref" : "#/components/schemas/WordList"
                 }
               }
             },
@@ -11576,6 +11573,85 @@
           "$ref" : "#/components/schemas/AwsCredentials"
         } ]
       },
+      "AwsAssumeRole" : {
+        "type" : "object",
+        "properties" : {
+          "clientConfig" : {
+            "$ref" : "#/components/schemas/AwsAssumeRoleClientConfig"
+          },
+          "request" : {
+            "$ref" : "#/components/schemas/AwsAssumeRoleRequest"
+          }
+        }
+      },
+      "AwsAssumeRoleClientConfig" : {
+        "type" : "object",
+        "properties" : {
+          "credentials" : {
+            "$ref" : "#/components/schemas/AwsCredentials"
+          },
+          "endpointOverride" : {
+            "type" : "string"
+          },
+          "region" : {
+            "type" : "string"
+          }
+        }
+      },
+      "AwsAssumeRoleRequest" : {
+        "type" : "object",
+        "properties" : {
+          "durationSeconds" : {
+            "type" : "integer",
+            "format" : "int32"
+          },
+          "externalId" : {
+            "type" : "string"
+          },
+          "policy" : {
+            "type" : "string"
+          },
+          "policyArns" : {
+            "type" : "array",
+            "items" : {
+              "$ref" : "#/components/schemas/AwsPolicyDescriptorType"
+            }
+          },
+          "providedContexts" : {
+            "type" : "array",
+            "items" : {
+              "$ref" : "#/components/schemas/AwsProvidedContext"
+            }
+          },
+          "roleArn" : {
+            "type" : "string"
+          },
+          "roleSessionName" : {
+            "type" : "string"
+          },
+          "serialNumber" : {
+            "type" : "string"
+          },
+          "sourceIdentity" : {
+            "type" : "string"
+          },
+          "tags" : {
+            "type" : "array",
+            "items" : {
+              "$ref" : "#/components/schemas/AwsTag"
+            }
+          },
+          "tokenCode" : {
+            "type" : "string"
+          },
+          "transitiveTagKeys" : {
+            "type" : "array",
+            "items" : {
+              "type" : "string"
+            }
+          }
+        }
+      },
       "AwsBasicCredentials" : {
         "type" : "object",
         "allOf" : [ {
@@ -11630,6 +11706,14 @@
           }
         }
       },
+      "AwsPolicyDescriptorType" : {
+        "type" : "object",
+        "properties" : {
+          "arn" : {
+            "type" : "string"
+          }
+        }
+      },
       "AwsProfileCredentials" : {
         "type" : "object",
         "allOf" : [ {
@@ -11646,6 +11730,17 @@
           }
         } ]
       },
+      "AwsProvidedContext" : {
+        "type" : "object",
+        "properties" : {
+          "contextAssertion" : {
+            "type" : "string"
+          },
+          "providerArn" : {
+            "type" : "string"
+          }
+        }
+      },
       "AwsProxyConfig" : {
         "type" : "object",
         "properties" : {
@@ -11695,6 +11790,17 @@
           "$ref" : "#/components/schemas/AwsCredentials"
         } ]
       },
+      "AwsTag" : {
+        "type" : "object",
+        "properties" : {
+          "key" : {
+            "type" : "string"
+          },
+          "value" : {
+            "type" : "string"
+          }
+        }
+      },
       "AwsWebCredentials" : {
         "type" : "object",
         "allOf" : [ {
@@ -18658,6 +18764,9 @@
           "accelerate" : {
             "type" : "boolean"
           },
+          "assumeRole" : {
+            "$ref" : "#/components/schemas/AwsAssumeRole"
+          },
           "async" : {
             "type" : "boolean"
           },
@@ -21412,6 +21521,40 @@
         },
         "required" : [ "type" ]
       },
+      "Word" : {
+        "type" : "object",
+        "properties" : {
+          "additionalSourceUuids" : {
+            "type" : "array",
+            "items" : {
+              "type" : "string"
+            }
+          },
+          "sourceUuid" : {
+            "type" : "string"
+          },
+          "word" : {
+            "type" : "string"
+          }
+        }
+      },
+      "WordList" : {
+        "type" : "object",
+        "properties" : {
+          "sourceUuidToDocRefMap" : {
+            "type" : "object",
+            "additionalProperties" : {
+              "$ref" : "#/components/schemas/DocRef"
+            }
+          },
+          "wordList" : {
+            "type" : "array",
+            "items" : {
+              "$ref" : "#/components/schemas/Word"
+            }
+          }
+        }
+      },
       "XPathFilter" : {
         "type" : "object",
         "properties" : {

diff --git a/stroom-app/src/main/resources/ui/noauth/swagger/stroom.yaml b/stroom-app/src/main/resources/ui/noauth/swagger/stroom.yaml
@@ -7404,9 +7404,7 @@ paths:
           content:
             application/json:
               schema:
-                type: array
-                items:
-                  type: string
+                $ref: '#/components/schemas/WordList'
           description: default response
       summary: Fetch a list of words from a dictionary by its UUID
       tags:
@@ -7960,6 +7958,58 @@ components:
       type: object
       allOf:
       - $ref: '#/components/schemas/AwsCredentials'
+    AwsAssumeRole:
+      type: object
+      properties:
+        clientConfig:
+          $ref: '#/components/schemas/AwsAssumeRoleClientConfig'
+        request:
+          $ref: '#/components/schemas/AwsAssumeRoleRequest'
+    AwsAssumeRoleClientConfig:
+      type: object
+      properties:
+        credentials:
+          $ref: '#/components/schemas/AwsCredentials'
+        endpointOverride:
+          type: string
+        region:
+          type: string
+    AwsAssumeRoleRequest:
+      type: object
+      properties:
+        durationSeconds:
+          type: integer
+          format: int32
+        externalId:
+          type: string
+        policy:
+          type: string
+        policyArns:
+          type: array
+          items:
+            $ref: '#/components/schemas/AwsPolicyDescriptorType'
+        providedContexts:
+          type: array
+          items:
+            $ref: '#/components/schemas/AwsProvidedContext'
+        roleArn:
+          type: string
+        roleSessionName:
+          type: string
+        serialNumber:
+          type: string
+        sourceIdentity:
+          type: string
+        tags:
+          type: array
+          items:
+            $ref: '#/components/schemas/AwsTag'
+        tokenCode:
+          type: string
+        transitiveTagKeys:
+          type: array
+          items:
+            type: string
     AwsBasicCredentials:
       type: object
       allOf:
@@ -7996,6 +8046,11 @@ components:
           $ref: '#/components/schemas/AwsProxyConfig'
         trustAllCertificatesEnabled:
           type: boolean
+    AwsPolicyDescriptorType:
+      type: object
+      properties:
+        arn:
+          type: string
     AwsProfileCredentials:
       type: object
       allOf:
@@ -8006,6 +8061,13 @@ components:
             type: string
           profileName:
             type: string
+    AwsProvidedContext:
+      type: object
+      properties:
+        contextAssertion:
+          type: string
+        providerArn:
+          type: string
     AwsProxyConfig:
       type: object
       properties:
@@ -8038,6 +8100,13 @@ components:
       type: object
       allOf:
       - $ref: '#/components/schemas/AwsCredentials'
+    AwsTag:
+      type: object
+      properties:
+        key:
+          type: string
+        value:
+          type: string
     AwsWebCredentials:
       type: object
       allOf:
@@ -14372,6 +14441,8 @@ components:
       properties:
         accelerate:
           type: boolean
+        assumeRole:
+          $ref: '#/components/schemas/AwsAssumeRole'
         async:
           type: boolean
         bucketName:
@@ -16462,6 +16533,28 @@ components:
           type: string
       required:
       - type
+    Word:
+      type: object
+      properties:
+        additionalSourceUuids:
+          type: array
+          items:
+            type: string
+        sourceUuid:
+          type: string
+        word:
+          type: string
+    WordList:
+      type: object
+      properties:
+        sourceUuidToDocRefMap:
+          type: object
+          additionalProperties:
+            $ref: '#/components/schemas/DocRef'
+        wordList:
+          type: array
+          items:
+            $ref: '#/components/schemas/Word'
     XPathFilter:
       type: object
       properties:

diff --git a/stroom-aws/stroom-aws-s3-impl/build.gradle b/stroom-aws/stroom-aws-s3-impl/build.gradle
@@ -18,6 +18,7 @@ dependencies {
 
     implementation libs.aws_crt
     implementation libs.aws_s3_transfer_manager
+    implementation libs.aws_sts
     implementation libs.commons_compress
     implementation libs.guava
     implementation libs.guice
@@ -35,6 +36,7 @@ dependencies {
     testImplementation libs.junit_jupiter_api
     testImplementation libs.mockito_core
     testImplementation libs.mockito_junit_jupiter
+    testImplementation libs.mbknor_jackson_jsonSchema
 
     // The following logging libs are needed when running junits outside dropwizard
     testRuntimeOnly libs.jakarta_activation

diff --git a/stroom-aws/stroom-aws-s3-impl/src/main/java/stroom/aws/s3/impl/S3Appender.java b/stroom-aws/stroom-aws-s3-impl/src/main/java/stroom/aws/s3/impl/S3Appender.java
@@ -170,7 +170,7 @@ public void setBucketNamePattern(final String bucketNamePattern) {
 
     @PipelineProperty(
             description = "Set the key name pattern if you want to override the one provided by the S3 config.",
-            defaultValue = "${type}/${year}/${month}/${day}/${idPath}/${feed}/${idPadded}.zip",
+            defaultValue = "${type}/${year}/${month}/${day}/${idPath}/${feed}/${idPadded}.gz",
             displayPriority = 3)
     public void setKeyNamePattern(final String keyNamePattern) {
         this.keyNamePattern = keyNamePattern;
@@ -202,7 +202,7 @@ public void setSplitRecords(final boolean splitRecords) {
 
     @PipelineProperty(
             description = "Apply compression to output objects.",
-            defaultValue = "false",
+            defaultValue = "true",
             displayPriority = 7)
     public void setUseCompression(final boolean useCompression) {
         outputFactory.setUseCompression(useCompression);