Merge pull request #125 from aws-solutions/develop

Update to version v1.9.0
aws-solutions · Feb 21, 2024 · 492373a · 492373a
2 parents bbdea9e + 9fe4849
commit 492373a
Show file tree

Hide file tree

Showing 48 changed files with 2,105 additions and 11,507 deletions.
diff --git a/CHANGELOG.md b/CHANGELOG.md
@@ -5,6 +5,19 @@ All notable changes to this project will be documented in this file.
 The format is based on [Keep a Changelog](https://keepachangelog.com/en/1.0.0/),
 and this project adheres to [Semantic Versioning](https://semver.org/spec/v2.0.0.html).
 
+## [1.9.0] - 2024-02-21
+
+### Added
+- Encrypt Glue Data Catalog data at-rest
+- Add custom CloudWatch dashboard
+
+### Updated
+- Fix SASL authentication deployment
+- Upgrade MSK to 3.5.1
+- Upgrade Apache Flink to Flink 1.15
+- Upgrade Lambda Functions to Python 3.12 and NodeJS 20
+- Patch security vulnerabilities
+
 ## [1.8.1] - 2023-10-18
 
 ### Updated

diff --git a/README.md b/README.md
@@ -59,10 +59,10 @@ To customize the solution, follow the steps below:
 ### Prerequisites
 
 - [AWS Command Line Interface](https://aws.amazon.com/cli/)
-- Node.js 14.x (or later) and npm 7 (or later)
-- Python 3.8 or later
+- Node.js 20.x (or later) and npm 9 (or later)
+- Python 3.12 or later
 - Java 11 (only required if using Apache Flink)
-- Apache Maven 3.1 (only required if using Apache Flink)
+- Apache Maven 3.6 (only required if using Apache Flink)
 
 > **Note**: The commands listed below will build all patterns. To only include one, you can modify the CDK entrypoint file on `source/bin/streaming-data-solution.ts`
 

diff --git a/deployment/build-s3-dist.sh b/deployment/build-s3-dist.sh
@@ -95,7 +95,7 @@ echo "--------------------------------------------------------------------------
 echo "[Init] Generate jar files for demo Java applications"
 echo "------------------------------------------------------------------------------"
 cd $source_dir/kinesis
-flink_version=1.13.2
+flink_version=1.15.4
 
 for folder in */ ; do
     cd "$folder"

diff --git a/source/bin/streaming-data-solution.ts b/source/bin/streaming-data-solution.ts
@@ -40,151 +40,175 @@ function applyAspects(stacks: cdk.Stack[], solutionId: string) {
                 solutionID: solutionId
             })
         );
-        NagSuppressions.addStackSuppressions(stack, [
-            { id: 'AwsSolutions-IAM5', reason: 'IAM role requires more permissions' },
-            { id: 'AwsSolutions-SQS3', reason: 'SQS is already a deadletter queue' },
-            { id: 'AwsSolutions-APIG2', reason: 'Request Validation happens by default with construct and more validations can be added by customer' },
-            { id: 'AwsSolutions-APIG3', reason: 'REST API stage is not associated with AWS WAFv2 web ACL because the solution is data agnostic' },
-            { id: 'AwsSolutions-COG2', reason: 'Customer will need to setup MFA with their information that they want to provide' },
-            { id: 'AwsSolutions-COG4', reason: 'Customer will need to setup Cognito authorizer as we have the default settings enabled. See patterns/apigw-kds-lambda.ts' },
-            { id: 'AwsSolutions-VPC7', reason: 'MSKLabs do not need VPC flow logs' },
-            { id: 'AwsSolutions-EC29', reason: 'EC2 does not need ASG' },
-            { id: 'AwsSolutions-MSK2', reason: 'MSKLabs uses Plaintext communication' },
-            { id: 'AwsSolutions-MSK6', reason: 'MSKLabs uses broker logs' },
-            {id: 'AwsSolutions-S10', reason: 'Bucket policy has condition to block non HTTPS traffic' }
-          ]);
     }
 }
 
 function createSolutionKinesisStacks() {
     const stacks: cdk.Stack[] = [];
-
-    stacks.push(
-        new ApiGwKdsLambda(app, 'streaming-data-solution-for-kinesis-using-api-gateway-and-lambda', {
-            synthesizer: new cdk.DefaultStackSynthesizer({
-                generateBootstrapVersionRule: false,
-            }),
-            description: `(${solutionIdKds}) - Streaming Data Solution for Amazon Kinesis (APIGW -> KDS -> Lambda). Version %%VERSION%%`,
-            solutionId: solutionIdKds
-        })
-    );
-
-    stacks.push(
-        new KplKdsKda(app, 'streaming-data-solution-for-kinesis-using-kpl-and-kinesis-data-analytics', {
-            synthesizer: new cdk.DefaultStackSynthesizer({
-                generateBootstrapVersionRule: false,
-            }),
-            description: `(${solutionIdKds}) - Streaming Data Solution for Amazon Kinesis (KPL -> KDS -> KDA). Version %%VERSION%%`,
-            solutionId: solutionIdKds
-        })
-    );
-
-    stacks.push(
-        new KdsKdfS3(app, 'streaming-data-solution-for-kinesis-using-kinesis-data-firehose-and-amazon-s3', {
-            synthesizer: new cdk.DefaultStackSynthesizer({
-                generateBootstrapVersionRule: false,
-            }),
-            description: `(${solutionIdKds}) - Streaming Data Solution for Amazon Kinesis (KDS -> KDF -> S3). Version %%VERSION%%`,
-            solutionId: solutionIdKds
-        })
-    );
-
-    stacks.push(
-        new KdsKdaApiGw(app, 'streaming-data-solution-for-kinesis-using-kinesis-data-analytics-and-api-gateway', {
-            synthesizer: new cdk.DefaultStackSynthesizer({
-                generateBootstrapVersionRule: false,
-            }),
-            description: `(${solutionIdKds}) - Streaming Data Solution for Amazon Kinesis (KDS -> KDA -> APIGW). Version %%VERSION%%`,
-            solutionId: solutionIdKds
-        })
-    );
+
+    const apiGwKdsLambda = new ApiGwKdsLambda(app, 'streaming-data-solution-for-kinesis-using-api-gateway-and-lambda', {
+        synthesizer: new cdk.DefaultStackSynthesizer({
+            generateBootstrapVersionRule: false,
+        }),
+        description: `(${solutionIdKds}) - Streaming Data Solution for Amazon Kinesis (APIGW -> KDS -> Lambda). Version %%VERSION%%`,
+        solutionId: solutionIdKds
+    });
+    NagSuppressions.addStackSuppressions(apiGwKdsLambda, [
+        { id: 'AwsSolutions-IAM5', reason: 'IAM role requires more permissions' },
+        { id: 'AwsSolutions-SQS3', reason: 'SQS is already a deadletter queue' },
+        { id: 'AwsSolutions-APIG2', reason: 'Request Validation happens by default with construct and more validations can be added by customer' },
+        { id: 'AwsSolutions-APIG3', reason: 'REST API stage is not associated with AWS WAFv2 web ACL because the solution is data agnostic' },
+        { id: 'AwsSolutions-COG2', reason: 'Customer will need to setup MFA with their information that they want to provide' },
+    ]);
+    stacks.push(apiGwKdsLambda);
+
+    const kplKdsKda = new KplKdsKda(app, 'streaming-data-solution-for-kinesis-using-kpl-and-kinesis-data-analytics', {
+        synthesizer: new cdk.DefaultStackSynthesizer({
+            generateBootstrapVersionRule: false,
+        }),
+        description: `(${solutionIdKds}) - Streaming Data Solution for Amazon Kinesis (KPL -> KDS -> KDA). Version %%VERSION%%`,
+        solutionId: solutionIdKds
+    });
+    NagSuppressions.addStackSuppressions(kplKdsKda, [
+        { id: 'AwsSolutions-IAM5', reason: 'IAM role requires more permissions' },
+        { id: 'AwsSolutions-EC29', reason: 'EC2 does not need ASG' },
+      ]);
+    stacks.push(kplKdsKda);
+
+    const kdsKdfS3 = new KdsKdfS3(app, 'streaming-data-solution-for-kinesis-using-kinesis-data-firehose-and-amazon-s3', {
+        synthesizer: new cdk.DefaultStackSynthesizer({
+            generateBootstrapVersionRule: false,
+        }),
+        description: `(${solutionIdKds}) - Streaming Data Solution for Amazon Kinesis (KDS -> KDF -> S3). Version %%VERSION%%`,
+        solutionId: solutionIdKds
+    });
+    NagSuppressions.addStackSuppressions(kdsKdfS3, [
+        { id: 'AwsSolutions-IAM5', reason: 'IAM role requires more permissions' },
+      ]);
+    stacks.push(kdsKdfS3);
+
+    const kdsKdaApiGw = new KdsKdaApiGw(app, 'streaming-data-solution-for-kinesis-using-kinesis-data-analytics-and-api-gateway', {
+        synthesizer: new cdk.DefaultStackSynthesizer({
+            generateBootstrapVersionRule: false,
+        }),
+        description: `(${solutionIdKds}) - Streaming Data Solution for Amazon Kinesis (KDS -> KDA -> APIGW). Version %%VERSION%%`,
+        solutionId: solutionIdKds
+    });
+    NagSuppressions.addStackSuppressions(kdsKdaApiGw, [
+        { id: 'AwsSolutions-IAM5', reason: 'IAM role requires more permissions' },
+        { id: 'AwsSolutions-APIG2', reason: 'Request Validation happens by default with construct and more validations can be added by customer' },
+        { id: 'AwsSolutions-APIG3', reason: 'REST API stage is not associated with AWS WAFv2 web ACL because the solution is data agnostic' },
+        { id: 'AwsSolutions-COG4', reason: 'Customer will need to setup Cognito authorizer as we have the default settings enabled. See patterns/apigw-kds-lambda.ts' },
+        { id: 'AwsSolutions-EC29', reason: 'EC2 does not need ASG' },
+      ]);
+    stacks.push(kdsKdaApiGw);
 
     applyAspects(stacks, solutionIdKds);
 }
 
 function createSolutionMskStacks() {
     const stacks: cdk.Stack[] = [];
 
-    stacks.push(
-        new MskStandalone(app, 'streaming-data-solution-for-msk', {
-            synthesizer: new cdk.DefaultStackSynthesizer({
-                generateBootstrapVersionRule: false,
-            }),
-            description: `(${solutionIdMsk}) - Streaming Data Solution for Amazon MSK. Version %%VERSION%%`,
-            solutionId: solutionIdMsk
-        })
-    );
-
-    stacks.push(
-        new MskLambda(app, 'streaming-data-solution-for-msk-using-aws-lambda', {
-            synthesizer: new cdk.DefaultStackSynthesizer({
-                generateBootstrapVersionRule: false,
-            }),
-            description: `(${solutionIdMskLambda}) - Streaming Data Solution for Amazon MSK (MSK -> Lambda). Version %%VERSION%%`,
-            solutionId: solutionIdMskLambda
-        })
-    );
-
-    stacks.push(
-        new MskLambdaKdf(app, 'streaming-data-solution-for-msk-using-aws-lambda-and-kinesis-data-firehose', {
-            synthesizer: new cdk.DefaultStackSynthesizer({
-                generateBootstrapVersionRule: false,
-            }),
-            description: `(${solutionIdMskLambdaKdf}) - Streaming Data Solution for Amazon MSK (MSK -> Lambda -> KDF). Version %%VERSION%%`,
-            solutionId: solutionIdMskLambdaKdf
-        })
-    );
-
-    stacks.push(
-        new MskKdaS3(app, 'streaming-data-solution-for-msk-using-kinesis-data-analytics-and-amazon-s3', {
-            synthesizer: new cdk.DefaultStackSynthesizer({
-                generateBootstrapVersionRule: false,
-            }),
-            description: `(${solutionIdMskKdaS3}) - Streaming Data Solution for Amazon MSK (MSK -> KDA -> S3). Version %%VERSION%%`,
-            solutionId: solutionIdMskKdaS3
-        })
-    );
+    const mskStandalone = new MskStandalone(app, 'streaming-data-solution-for-msk', {
+        synthesizer: new cdk.DefaultStackSynthesizer({
+            generateBootstrapVersionRule: false,
+        }),
+        description: `(${solutionIdMsk}) - Streaming Data Solution for Amazon MSK. Version %%VERSION%%`,
+        solutionId: solutionIdMsk
+    })
+    NagSuppressions.addStackSuppressions(mskStandalone, [
+        { id: 'AwsSolutions-IAM5', reason: 'IAM role requires more permissions' },
+        { id: 'AwsSolutions-EC29', reason: 'EC2 does not need ASG' },
+      ]);
+    stacks.push(mskStandalone);
+
+    const mskLambda = new MskLambda(app, 'streaming-data-solution-for-msk-using-aws-lambda', {
+        synthesizer: new cdk.DefaultStackSynthesizer({
+            generateBootstrapVersionRule: false,
+        }),
+        description: `(${solutionIdMskLambda}) - Streaming Data Solution for Amazon MSK (MSK -> Lambda). Version %%VERSION%%`,
+        solutionId: solutionIdMskLambda
+    });
+    NagSuppressions.addStackSuppressions(mskLambda, [
+        { id: 'AwsSolutions-IAM5', reason: 'IAM role requires more permissions' },
+      ]);
+    stacks.push(mskLambda);
+
+    const mskLambdaKdf = new MskLambdaKdf(app, 'streaming-data-solution-for-msk-using-aws-lambda-and-kinesis-data-firehose', {
+        synthesizer: new cdk.DefaultStackSynthesizer({
+            generateBootstrapVersionRule: false,
+        }),
+        description: `(${solutionIdMskLambdaKdf}) - Streaming Data Solution for Amazon MSK (MSK -> Lambda -> KDF). Version %%VERSION%%`,
+        solutionId: solutionIdMskLambdaKdf
+    });
+    NagSuppressions.addStackSuppressions(mskLambdaKdf, [
+        { id: 'AwsSolutions-IAM5', reason: 'IAM role requires more permissions' },
+      ]);
+    stacks.push(mskLambdaKdf);
+
+    const mskKdaS3 = new MskKdaS3(app, 'streaming-data-solution-for-msk-using-kinesis-data-analytics-and-amazon-s3', {
+        synthesizer: new cdk.DefaultStackSynthesizer({
+            generateBootstrapVersionRule: false,
+        }),
+        description: `(${solutionIdMskKdaS3}) - Streaming Data Solution for Amazon MSK (MSK -> KDA -> S3). Version %%VERSION%%`,
+        solutionId: solutionIdMskKdaS3
+    });
+    NagSuppressions.addStackSuppressions(mskKdaS3, [
+        { id: 'AwsSolutions-IAM5', reason: 'IAM role requires more permissions' },
+      ]);
+    stacks.push(mskKdaS3);
 
     applyAspects(stacks, solutionIdMsk);
 }
 
 function createSolutionMskLabsStacks() {
     const stacks: cdk.Stack[] = [];
 
-    stacks.push(
-        new MskLambdaRoleStack(app, 'amazon-msk-labs-lambda-role', {
-            synthesizer: new cdk.DefaultStackSynthesizer({
-                generateBootstrapVersionRule: false,
-            }),
-            description: `(${solutionIdMskLabs}) - Amazon MSK Labs (IAM Role). Version %%VERSION%%`,
-            solutionId: solutionIdMskLabs
-        })
-    );
-
-    stacks.push(
-        new MskClientStack(app, 'amazon-msk-labs-ec2-client', {
-            synthesizer: new cdk.DefaultStackSynthesizer({
-                generateBootstrapVersionRule: false,
-            }),
-            description: `(${solutionIdMskLabs}) - Amazon MSK Labs (EC2 Client). Version %%VERSION%%`,
-            solutionId: solutionIdMskLabs
-        })
-    );
-
-    stacks.push(
-        new MskClusterStack(app, 'amazon-msk-labs-cluster', {
-            synthesizer: new cdk.DefaultStackSynthesizer({
-                generateBootstrapVersionRule: false,
-            }),
-            description: `(${solutionIdMskLabs}) - Amazon MSK Labs (MSK Cluster). Version %%VERSION%%`,
-            solutionId: solutionIdMskLabs
-        })
-    );
+    const mskLambdaRoleStack = new MskLambdaRoleStack(app, 'amazon-msk-labs-lambda-role', {
+        synthesizer: new cdk.DefaultStackSynthesizer({
+            generateBootstrapVersionRule: false,
+        }),
+        description: `(${solutionIdMskLabs}) - Amazon MSK Labs (IAM Role). Version %%VERSION%%`,
+        solutionId: solutionIdMskLabs
+    });
+    NagSuppressions.addStackSuppressions(mskLambdaRoleStack, [
+        { id: 'AwsSolutions-IAM5', reason: 'IAM role requires more permissions' },
+      ]);
+    stacks.push(mskLambdaRoleStack);
+
+    const mskClientStack = new MskClientStack(app, 'amazon-msk-labs-ec2-client', {
+        synthesizer: new cdk.DefaultStackSynthesizer({
+            generateBootstrapVersionRule: false,
+        }),
+        description: `(${solutionIdMskLabs}) - Amazon MSK Labs (EC2 Client). Version %%VERSION%%`,
+        solutionId: solutionIdMskLabs
+    });
+    NagSuppressions.addStackSuppressions(mskClientStack, [
+        { id: 'AwsSolutions-IAM5', reason: 'IAM role requires more permissions' },
+        { id: 'AwsSolutions-VPC7', reason: 'MSKLabs do not need VPC flow logs' },
+        { id: 'AwsSolutions-EC29', reason: 'EC2 does not need ASG' },
+        { id: 'CdkNagValidationFailure', reason: 'We are using an intrinsic function to determine the IP address' }
+      ]);
+    stacks.push(mskClientStack);
+
+    const mskClusterStack = new MskClusterStack(app, 'amazon-msk-labs-cluster', {
+        synthesizer: new cdk.DefaultStackSynthesizer({
+            generateBootstrapVersionRule: false,
+        }),
+        description: `(${solutionIdMskLabs}) - Amazon MSK Labs (MSK Cluster). Version %%VERSION%%`,
+        solutionId: solutionIdMskLabs
+    });
+    NagSuppressions.addStackSuppressions(mskClusterStack, [
+        { id: 'AwsSolutions-IAM5', reason: 'IAM role requires more permissions' },
+        { id: 'AwsSolutions-MSK2', reason: 'MSKLabs uses Plaintext communication' },
+        { id: 'AwsSolutions-MSK6', reason: 'MSKLabs uses broker logs' },
+      ]);
+    stacks.push(mskClusterStack);
 
     applyAspects(stacks, solutionIdMskLabs);
 }
 
-cdk.Aspects.of(app).add(new AwsSolutionsChecks());
+cdk.Aspects.of(app).add(new AwsSolutionsChecks( {verbose: true} ));
 createSolutionKinesisStacks();
 createSolutionMskStacks();
 createSolutionMskLabsStacks();
diff --git a/source/kinesis/kda-flink-ml/pom.xml b/source/kinesis/kda-flink-ml/pom.xml
@@ -13,10 +13,9 @@
         <project.build.sourceEncoding>UTF-8</project.build.sourceEncoding>
 
         <java.version>1.11</java.version>
-        <scala.binary.version>2.12</scala.binary.version>
         <kda.version>2.0.0</kda.version>
         <kda.runtime.version>1.2.0</kda.runtime.version>
-        <aws.sdk.version>2.15.38</aws.sdk.version>
+        <aws.sdk.version>2.23.4</aws.sdk.version>
 
         <compiler-plugin.version>3.8.1</compiler-plugin.version>
         <shade-plugin.version>3.2.4</shade-plugin.version>
@@ -34,7 +33,7 @@
 
         <dependency>
             <groupId>org.apache.flink</groupId>
-            <artifactId>flink-streaming-java_${scala.binary.version}</artifactId>
+            <artifactId>flink-streaming-java</artifactId>
             <version>${flink.version}</version>
             <scope>provided</scope>
         </dependency>
@@ -48,7 +47,7 @@
         <!-- Connector dependencies. They must be in the default scope (compile). -->
         <dependency>
             <groupId>org.apache.flink</groupId>
-            <artifactId>flink-connector-kinesis_${scala.binary.version}</artifactId>
+            <artifactId>flink-connector-kinesis</artifactId>
             <version>${flink.version}</version>
         </dependency>
 

diff --git a/source/labs/msk-client-setup.ts b/source/labs/msk-client-setup.ts
@@ -121,7 +121,7 @@ export class MskClientStack extends cdk.Stack {
         });
 
         const cloud9Setup = new lambda.Function(this, 'CustomResource', {
-            runtime: lambda.Runtime.PYTHON_3_10,
+            runtime: lambda.Runtime.PYTHON_3_12,
             handler: 'lambda_function.handler',
             description: 'This function creates prerequisite resources for Cloud9 (such as IAM roles)',
             code: lambda.Code.fromAsset('lambda/cloud9-setup'),

diff --git a/source/lambda/cloud9-setup/requirements-test.txt b/source/lambda/cloud9-setup/requirements-test.txt
@@ -1,3 +1,3 @@
 boto3
-botocore
+botocore==1.31.68
 coverage
diff --git a/source/lambda/cloud9-setup/requirements.txt b/source/lambda/cloud9-setup/requirements.txt
@@ -1 +1 @@
-crhelper==2.0.10
+crhelper==2.0.11
diff --git a/source/lambda/kda-vpc-config/requirements-test.txt b/source/lambda/kda-vpc-config/requirements-test.txt
@@ -1,3 +1,3 @@
 boto3
-botocore
+botocore==1.31.68
 coverage
diff --git a/source/lambda/kda-vpc-config/requirements.txt b/source/lambda/kda-vpc-config/requirements.txt
@@ -1 +1 @@
-crhelper==2.0.10
+crhelper==2.0.11