diff --git a/CHANGELOG.md b/CHANGELOG.md index fcdd62b..e597035 100644 --- a/CHANGELOG.md +++ b/CHANGELOG.md @@ -4,6 +4,15 @@ All notable changes to this project will be documented in this file. The format is based on [Keep a Changelog](https://keepachangelog.com/en/1.0.0/), and this project adheres to [Semantic Versioning](https://semver.org/spec/v2.0.0.html). +## [1.2.0] - 2022-01-31 +### Added +- The solution now supports batch segment jobs to get user segments with your solution version. Each user segment is +sorted in descending order based on the probability that each user will interact with items in your inventory. +- The solution now supports domain dataset groups. + +### Changed +- Upgraded to CDKv2. + ## [1.1.0] - 2021-11-22 ### Added - The solution now creates an Amazon EventBridge event bus, and puts messages to the bus when resources have been diff --git a/README.md b/README.md index 717dba2..8ed4ede 100644 --- a/README.md +++ b/README.md @@ -293,7 +293,7 @@ The following procedures assumes that all the OS-level configuration has been co * [AWS Command Line Interface](https://aws.amazon.com/cli/) * [Python](https://www.python.org/) 3.9 or newer * [Node.js](https://nodejs.org/en/) 16.x or newer -* [AWS CDK](https://aws.amazon.com/cdk/) 1.126.0 or newer +* [AWS CDK](https://aws.amazon.com/cdk/) 2.7.0 or newer * [Amazon Corretto OpenJDK](https://docs.aws.amazon.com/corretto/) 11 > **Please ensure you test the templates before updating any production deployments.** @@ -349,7 +349,7 @@ export SOLUTION_NAME=my-solution-name export VERSION=my-version export REGION_NAME=my-region -build-s3-cdk-dist \ +build-s3-cdk-dist deploy \ --source-bucket-name DIST_BUCKET_PREFIX \ --solution-name SOLUTION_NAME \ --version_code VERSION \ diff --git a/deployment/run-unit-tests.sh b/deployment/run-unit-tests.sh index d0de7aa..6238d7b 100644 --- a/deployment/run-unit-tests.sh +++ b/deployment/run-unit-tests.sh @@ -45,6 +45,7 @@ virtualenv .venv source .venv/bin/activate cd $source_dir +pip install --upgrade pip pip install -r $source_dir/requirements-dev.txt cd - diff --git a/source/aws_lambda/create_batch_inference_job/handler.py b/source/aws_lambda/create_batch_inference_job/handler.py index 0b6eb72..08cca79 100644 --- a/source/aws_lambda/create_batch_inference_job/handler.py +++ b/source/aws_lambda/create_batch_inference_job/handler.py @@ -18,6 +18,52 @@ from shared.sfn_middleware import PersonalizeResource +RESOURCE = "batchInferenceJob" +STATUS = "batchInferenceJob.status" +CONFIG = { + "jobName": { + "source": "event", + "path": "serviceConfig.jobName", + }, + "solutionVersionArn": { + "source": "event", + "path": "serviceConfig.solutionVersionArn", + }, + "filterArn": { + "source": "event", + "path": "serviceConfig.filterArn", + "default": "omit", + }, + "numResults": { + "source": "event", + "path": "serviceConfig.numResults", + "default": "omit", + }, + "jobInput": { + "source": "event", + "path": "serviceConfig.jobInput", + }, + "jobOutput": {"source": "event", "path": "serviceConfig.jobOutput"}, + "roleArn": {"source": "environment", "path": "ROLE_ARN"}, + "batchInferenceJobConfig": { + "source": "event", + "path": "serviceConfig.batchInferenceJobConfig", + "default": "omit", + }, + "maxAge": { + "source": "event", + "path": "workflowConfig.maxAge", + "default": "omit", + "as": "seconds", + }, + "timeStarted": { + "source": "event", + "path": "workflowConfig.timeStarted", + "default": "omit", + "as": "iso8601", + }, +} + logger = Logger() tracer = Tracer() metrics = Metrics() @@ -26,51 +72,9 @@ @metrics.log_metrics @tracer.capture_lambda_handler @PersonalizeResource( - resource="batchInferenceJob", - status="batchInferenceJob.status", - config={ - "jobName": { - "source": "event", - "path": "serviceConfig.jobName", - }, - "solutionVersionArn": { - "source": "event", - "path": "serviceConfig.solutionVersionArn", - }, - "filterArn": { - "source": "event", - "path": "serviceConfig.filterArn", - "default": "omit", - }, - "numResults": { - "source": "event", - "path": "serviceConfig.numResults", - "default": "omit", - }, - "jobInput": { - "source": "event", - "path": "serviceConfig.jobInput", - }, - "jobOutput": {"source": "event", "path": "serviceConfig.jobOutput"}, - "roleArn": {"source": "environment", "path": "ROLE_ARN"}, - "batchInferenceJobConfig": { - "source": "event", - "path": "serviceConfig.batchInferenceJobConfig", - "default": "omit", - }, - "maxAge": { - "source": "event", - "path": "workflowConfig.maxAge", - "default": "omit", - "as": "seconds", - }, - "timeStarted": { - "source": "event", - "path": "workflowConfig.timeStarted", - "default": "omit", - "as": "iso8601", - }, - }, + resource=RESOURCE, + status=STATUS, + config=CONFIG, ) def lambda_handler(event: Dict[str, Any], context: LambdaContext) -> Dict: """Create a batch inference job in Amazon Personalize based on the configuration in `event` diff --git a/source/aws_lambda/create_batch_segment_job/__init__.py b/source/aws_lambda/create_batch_segment_job/__init__.py new file mode 100644 index 0000000..ef2f9eb --- /dev/null +++ b/source/aws_lambda/create_batch_segment_job/__init__.py @@ -0,0 +1,12 @@ +# ###################################################################################################################### +# Copyright Amazon.com, Inc. or its affiliates. All Rights Reserved. # +# # +# Licensed under the Apache License, Version 2.0 (the "License"). You may not use this file except in compliance # +# with the License. You may obtain a copy of the License at # +# # +# http://www.apache.org/licenses/LICENSE-2.0 # +# # +# Unless required by applicable law or agreed to in writing, software distributed under the License is distributed # +# on an "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. See the License for # +# the specific language governing permissions and limitations under the License. # +# ###################################################################################################################### diff --git a/source/aws_lambda/create_batch_segment_job/handler.py b/source/aws_lambda/create_batch_segment_job/handler.py new file mode 100644 index 0000000..e055405 --- /dev/null +++ b/source/aws_lambda/create_batch_segment_job/handler.py @@ -0,0 +1,80 @@ +# ###################################################################################################################### +# Copyright Amazon.com, Inc. or its affiliates. All Rights Reserved. # +# # +# Licensed under the Apache License, Version 2.0 (the "License"). You may not use this file except in compliance # +# with the License. You may obtain a copy of the License at # +# # +# http://www.apache.org/licenses/LICENSE-2.0 # +# # +# Unless required by applicable law or agreed to in writing, software distributed under the License is distributed # +# on an "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. See the License for # +# the specific language governing permissions and limitations under the License. # +# ###################################################################################################################### + +from typing import Dict, Any + +from aws_lambda_powertools import Logger, Tracer, Metrics +from aws_lambda_powertools.utilities.typing import LambdaContext + +from shared.sfn_middleware import PersonalizeResource + +RESOURCE = "batchSegmentJob" +STATUS = "batchSegmentJob.status" +CONFIG = { + "filterArn": { + "source": "event", + "path": "serviceConfig.filterArn", + "default": "omit", + }, + "jobInput": { + "source": "event", + "path": "serviceConfig.jobInput", + }, + "jobName": { + "source": "event", + "path": "serviceConfig.jobName", + }, + "jobOutput": {"source": "event", "path": "serviceConfig.jobOutput"}, + "solutionVersionArn": { + "source": "event", + "path": "serviceConfig.solutionVersionArn", + }, + "numResults": { + "source": "event", + "path": "serviceConfig.numResults", + "default": "omit", + }, + "roleArn": {"source": "environment", "path": "ROLE_ARN"}, + "maxAge": { + "source": "event", + "path": "workflowConfig.maxAge", + "default": "omit", + "as": "seconds", + }, + "timeStarted": { + "source": "event", + "path": "workflowConfig.timeStarted", + "default": "omit", + "as": "iso8601", + }, +} + +logger = Logger() +tracer = Tracer() +metrics = Metrics() + + +@metrics.log_metrics +@tracer.capture_lambda_handler +@PersonalizeResource( + resource=RESOURCE, + status=STATUS, + config=CONFIG, +) +def lambda_handler(event: Dict[str, Any], context: LambdaContext) -> Dict: + """Create a batch segment job in Amazon Personalize based on the configuration in `event` + :param event: AWS Lambda Event + :param context: AWS Lambda Context + :return: the configured batch inference job + """ + return event.get("resource") # return the batch inference job diff --git a/source/aws_lambda/create_campaign/handler.py b/source/aws_lambda/create_campaign/handler.py index 4ff4bf7..06109ab 100644 --- a/source/aws_lambda/create_campaign/handler.py +++ b/source/aws_lambda/create_campaign/handler.py @@ -18,6 +18,41 @@ from shared.sfn_middleware import PersonalizeResource +RESOURCE = "campaign" +STATUS = "campaign.latestCampaignUpdate.status || campaign.status" +CONFIG = { + "name": { + "source": "event", + "path": "serviceConfig.name", + }, + "solutionVersionArn": { + "source": "event", + "path": "serviceConfig.solutionVersionArn", + }, + "minProvisionedTPS": { + "source": "event", + "path": "serviceConfig.minProvisionedTPS", + "as": "int", + }, + "campaignConfig": { + "source": "event", + "path": "serviceConfig.campaignConfig", + "default": "omit", + }, + "maxAge": { + "source": "event", + "path": "workflowConfig.maxAge", + "default": "omit", + "as": "seconds", + }, + "timeStarted": { + "source": "event", + "path": "workflowConfig.timeStarted", + "default": "omit", + "as": "iso8601", + }, +} + logger = Logger() tracer = Tracer() metrics = Metrics() @@ -26,40 +61,9 @@ @metrics.log_metrics @tracer.capture_lambda_handler @PersonalizeResource( - resource="campaign", - config={ - "name": { - "source": "event", - "path": "serviceConfig.name", - }, - "solutionVersionArn": { - "source": "event", - "path": "serviceConfig.solutionVersionArn", - }, - "minProvisionedTPS": { - "source": "event", - "path": "serviceConfig.minProvisionedTPS", - "as": "int", - }, - "campaignConfig": { - "source": "event", - "path": "serviceConfig.campaignConfig", - "default": "omit", - }, - "maxAge": { - "source": "event", - "path": "workflowConfig.maxAge", - "default": "omit", - "as": "seconds", - }, - "timeStarted": { - "source": "event", - "path": "workflowConfig.timeStarted", - "default": "omit", - "as": "iso8601", - }, - }, - status="campaign.latestCampaignUpdate.status || campaign.status", + resource=RESOURCE, + config=CONFIG, + status=STATUS, ) def lambda_handler(event: Dict[str, Any], context: LambdaContext) -> Dict: """Create a campaign in Amazon Personalize based on the configuration in `event` diff --git a/source/aws_lambda/create_dataset/handler.py b/source/aws_lambda/create_dataset/handler.py index 62aaedd..7c74d04 100644 --- a/source/aws_lambda/create_dataset/handler.py +++ b/source/aws_lambda/create_dataset/handler.py @@ -18,6 +18,29 @@ from shared.sfn_middleware import PersonalizeResource +RESOURCE = "dataset" +CONFIG = { + "name": { + "source": "event", + "path": "serviceConfig.name", + }, + "datasetType": { + "source": "event", + "path": "serviceConfig.datasetType", + }, + "datasetGroupArn": { + "source": "event", + "path": "serviceConfig.datasetGroupArn", + }, + "schemaArn": {"source": "event", "path": "serviceConfig.schemaArn"}, + "timeStarted": { + "source": "event", + "path": "workflowConfig.timeStarted", + "default": "omit", + "as": "iso8601", + }, +} + logger = Logger() tracer = Tracer() metrics = Metrics() @@ -26,28 +49,8 @@ @metrics.log_metrics @tracer.capture_lambda_handler @PersonalizeResource( - resource="dataset", - config={ - "name": { - "source": "event", - "path": "serviceConfig.name", - }, - "datasetType": { - "source": "event", - "path": "serviceConfig.datasetType", - }, - "datasetGroupArn": { - "source": "event", - "path": "serviceConfig.datasetGroupArn", - }, - "schemaArn": {"source": "event", "path": "serviceConfig.schemaArn"}, - "timeStarted": { - "source": "event", - "path": "workflowConfig.timeStarted", - "default": "omit", - "as": "iso8601", - }, - }, + resource=RESOURCE, + config=CONFIG, ) def lambda_handler(event: Dict[str, Any], context: LambdaContext) -> Dict: """Create a dataset in Amazon Personalize based on the configuration in `event` diff --git a/source/aws_lambda/create_dataset_group/handler.py b/source/aws_lambda/create_dataset_group/handler.py index 4c72647..20fcc21 100644 --- a/source/aws_lambda/create_dataset_group/handler.py +++ b/source/aws_lambda/create_dataset_group/handler.py @@ -18,6 +18,36 @@ from shared.sfn_middleware import PersonalizeResource +RESOURCE = "datasetGroup" +STATUS = "datasetGroup.status" +CONFIG = { + "name": { + "source": "event", + "path": "serviceConfig.name", + }, + "domain": { + "source": "event", + "path": "serviceConfig.domain", + "default": "omit", + }, + "roleArn": { + "source": "environment", + "path": "KMS_ROLE_ARN", + "default": "omit", + }, + "kmsKeyArn": { + "source": "environment", + "path": "KMS_KEY_ARN", + "default": "omit", + }, + "timeStarted": { + "source": "event", + "path": "workflowConfig.timeStarted", + "default": "omit", + "as": "iso8601", + }, +} + tracer = Tracer() logger = Logger() metrics = Metrics() @@ -26,30 +56,9 @@ @metrics.log_metrics @tracer.capture_lambda_handler @PersonalizeResource( - resource="datasetGroup", - status="datasetGroup.status", - config={ - "name": { - "source": "event", - "path": "serviceConfig.name", - }, - "roleArn": { - "source": "environment", - "path": "KMS_ROLE_ARN", - "default": "omit", - }, - "kmsKeyArn": { - "source": "environment", - "path": "KMS_KEY_ARN", - "default": "omit", - }, - "timeStarted": { - "source": "event", - "path": "workflowConfig.timeStarted", - "default": "omit", - "as": "iso8601", - }, - }, + resource=RESOURCE, + status=STATUS, + config=CONFIG, ) def lambda_handler(event: Dict[str, Any], context: LambdaContext) -> Dict: """Create a dataset group in Amazon Personalize based on the configuration in `event` diff --git a/source/aws_lambda/create_dataset_import_job/handler.py b/source/aws_lambda/create_dataset_import_job/handler.py index 7c8899b..fc08b24 100644 --- a/source/aws_lambda/create_dataset_import_job/handler.py +++ b/source/aws_lambda/create_dataset_import_job/handler.py @@ -18,6 +18,36 @@ from shared.sfn_middleware import PersonalizeResource +RESOURCE = "datasetImportJob" +STATUS = "datasetImportJob.status" +CONFIG = { + "jobName": { + "source": "event", + "path": "serviceConfig.jobName", + }, + "datasetArn": { + "source": "event", + "path": "serviceConfig.datasetArn", + }, + "dataSource": { + "source": "event", + "path": "serviceConfig.dataSource", + }, + "roleArn": {"source": "environment", "path": "ROLE_ARN"}, + "maxAge": { + "source": "event", + "path": "workflowConfig.maxAge", + "default": "omit", + "as": "seconds", + }, + "timeStarted": { + "source": "event", + "path": "workflowConfig.timeStarted", + "default": "omit", + "as": "iso8601", + }, +} + logger = Logger() tracer = Tracer() metrics = Metrics() @@ -26,35 +56,9 @@ @metrics.log_metrics @tracer.capture_lambda_handler @PersonalizeResource( - resource="datasetImportJob", - status="datasetImportJob.status", - config={ - "jobName": { - "source": "event", - "path": "serviceConfig.jobName", - }, - "datasetArn": { - "source": "event", - "path": "serviceConfig.datasetArn", - }, - "dataSource": { - "source": "event", - "path": "serviceConfig.dataSource", - }, - "roleArn": {"source": "environment", "path": "ROLE_ARN"}, - "maxAge": { - "source": "event", - "path": "workflowConfig.maxAge", - "default": "omit", - "as": "seconds", - }, - "timeStarted": { - "source": "event", - "path": "workflowConfig.timeStarted", - "default": "omit", - "as": "iso8601", - }, - }, + resource=RESOURCE, + status=STATUS, + config=CONFIG, ) def lambda_handler(event: Dict[str, Any], context: LambdaContext) -> Dict: """Create a dataset import job in Amazon Personalize based on the configuration in `event` diff --git a/source/aws_lambda/create_event_tracker/handler.py b/source/aws_lambda/create_event_tracker/handler.py index e46ea7e..9a87406 100644 --- a/source/aws_lambda/create_event_tracker/handler.py +++ b/source/aws_lambda/create_event_tracker/handler.py @@ -18,6 +18,25 @@ from shared.sfn_middleware import PersonalizeResource +RESOURCE = "eventTracker" +STATUS = "eventTracker.status" +CONFIG = { + "name": { + "source": "event", + "path": "serviceConfig.name", + }, + "datasetGroupArn": { + "source": "event", + "path": "serviceConfig.datasetGroupArn", + }, + "timeStarted": { + "source": "event", + "path": "workflowConfig.timeStarted", + "default": "omit", + "as": "iso8601", + }, +} + logger = Logger() tracer = Tracer() metrics = Metrics() @@ -26,24 +45,9 @@ @metrics.log_metrics @tracer.capture_lambda_handler @PersonalizeResource( - resource="eventTracker", - status="eventTracker.status", - config={ - "name": { - "source": "event", - "path": "serviceConfig.name", - }, - "datasetGroupArn": { - "source": "event", - "path": "serviceConfig.datasetGroupArn", - }, - "timeStarted": { - "source": "event", - "path": "workflowConfig.timeStarted", - "default": "omit", - "as": "iso8601", - }, - }, + resource=RESOURCE, + status=STATUS, + config=CONFIG, ) def lambda_handler(event: Dict[str, Any], context: LambdaContext) -> Dict: """Create an event tracker in Amazon Personalize based on the configuration in `event` diff --git a/source/aws_lambda/create_filter/handler.py b/source/aws_lambda/create_filter/handler.py index 654d20b..473d098 100644 --- a/source/aws_lambda/create_filter/handler.py +++ b/source/aws_lambda/create_filter/handler.py @@ -18,6 +18,29 @@ from shared.sfn_middleware import PersonalizeResource +RESOURCE = "filter" +STATUS = "filter.status" +CONFIG = { + "name": { + "source": "event", + "path": "serviceConfig.name", + }, + "datasetGroupArn": { + "source": "event", + "path": "serviceConfig.datasetGroupArn", + }, + "filterExpression": { + "source": "event", + "path": "serviceConfig.filterExpression", + }, + "timeStarted": { + "source": "event", + "path": "workflowConfig.timeStarted", + "default": "omit", + "as": "iso8601", + }, +} + logger = Logger() tracer = Tracer() metrics = Metrics() @@ -26,28 +49,9 @@ @metrics.log_metrics @tracer.capture_lambda_handler @PersonalizeResource( - resource="filter", - status="filter.status", - config={ - "name": { - "source": "event", - "path": "serviceConfig.name", - }, - "datasetGroupArn": { - "source": "event", - "path": "serviceConfig.datasetGroupArn", - }, - "filterExpression": { - "source": "event", - "path": "serviceConfig.filterExpression", - }, - "timeStarted": { - "source": "event", - "path": "workflowConfig.timeStarted", - "default": "omit", - "as": "iso8601", - }, - }, + resource=RESOURCE, + status=STATUS, + config=CONFIG, ) def lambda_handler(event: Dict[str, Any], context: LambdaContext) -> Dict: """Create a filter in Amazon Personalize based on the configuration in `event` diff --git a/source/aws_lambda/create_recommender/__init__.py b/source/aws_lambda/create_recommender/__init__.py new file mode 100644 index 0000000..ef2f9eb --- /dev/null +++ b/source/aws_lambda/create_recommender/__init__.py @@ -0,0 +1,12 @@ +# ###################################################################################################################### +# Copyright Amazon.com, Inc. or its affiliates. All Rights Reserved. # +# # +# Licensed under the Apache License, Version 2.0 (the "License"). You may not use this file except in compliance # +# with the License. You may obtain a copy of the License at # +# # +# http://www.apache.org/licenses/LICENSE-2.0 # +# # +# Unless required by applicable law or agreed to in writing, software distributed under the License is distributed # +# on an "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. See the License for # +# the specific language governing permissions and limitations under the License. # +# ###################################################################################################################### diff --git a/source/aws_lambda/create_recommender/handler.py b/source/aws_lambda/create_recommender/handler.py new file mode 100644 index 0000000..3e02880 --- /dev/null +++ b/source/aws_lambda/create_recommender/handler.py @@ -0,0 +1,64 @@ +# ###################################################################################################################### +# Copyright Amazon.com, Inc. or its affiliates. All Rights Reserved. # +# # +# Licensed under the Apache License, Version 2.0 (the "License"). You may not use this file except in compliance # +# with the License. You may obtain a copy of the License at # +# # +# http://www.apache.org/licenses/LICENSE-2.0 # +# # +# Unless required by applicable law or agreed to in writing, software distributed under the License is distributed # +# on an "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. See the License for # +# the specific language governing permissions and limitations under the License. # +# ###################################################################################################################### + +from typing import Dict, Any + +from aws_lambda_powertools import Logger, Tracer, Metrics +from aws_lambda_powertools.utilities.typing import LambdaContext + +from shared.sfn_middleware import PersonalizeResource + +RESOURCE = "recommender" +STATUS = "recommender.status" +CONFIG = { + "name": { + "source": "event", + "path": "serviceConfig.name", + }, + "datasetGroupArn": { + "source": "event", + "path": "serviceConfig.datasetGroupArn", + }, + "recipeArn": {"source": "event", "path": "serviceConfig.recipeArn"}, + "recommenderConfig": { + "source": "event", + "path": "serviceConfig.recommenderConfig", + "default": "omit", + }, + "timeStarted": { + "source": "event", + "path": "workflowConfig.timeStarted", + "default": "omit", + "as": "iso8601", + }, +} + +logger = Logger() +tracer = Tracer() +metrics = Metrics() + + +@metrics.log_metrics +@tracer.capture_lambda_handler +@PersonalizeResource( + resource=RESOURCE, + status=STATUS, + config=CONFIG, +) +def lambda_handler(event: Dict[str, Any], context: LambdaContext) -> Dict: + """Create a recommender in Amazon Personalize based on the configuration in `event` + :param event: AWS Lambda Event + :param context: AWS Lambda Context + :return: the configured dataset + """ + return event.get("resource") # return the dataset diff --git a/source/aws_lambda/create_schema/handler.py b/source/aws_lambda/create_schema/handler.py index 12fbd1e..b92d3ec 100644 --- a/source/aws_lambda/create_schema/handler.py +++ b/source/aws_lambda/create_schema/handler.py @@ -18,6 +18,19 @@ from shared.sfn_middleware import PersonalizeResource +RESOURCE = "schema" +CONFIG = { + "name": { + "source": "event", + "path": "serviceConfig.name", + }, + "domain": { + "source": "event", + "path": "serviceConfig.domain", + "default": "omit", + }, + "schema": {"source": "event", "path": "serviceConfig.schema", "as": "string"}, +} logger = Logger() tracer = Tracer() metrics = Metrics() @@ -26,14 +39,8 @@ @metrics.log_metrics @tracer.capture_lambda_handler @PersonalizeResource( - resource="schema", - config={ - "name": { - "source": "event", - "path": "serviceConfig.name", - }, - "schema": {"source": "event", "path": "serviceConfig.schema", "as": "string"}, - }, + resource=RESOURCE, + config=CONFIG, ) def lambda_handler(event: Dict[str, Any], context: LambdaContext) -> Dict: """Create a schema in Amazon Personalize based on the configuration in `event` diff --git a/source/aws_lambda/create_solution/handler.py b/source/aws_lambda/create_solution/handler.py index d0a0ecb..a458227 100644 --- a/source/aws_lambda/create_solution/handler.py +++ b/source/aws_lambda/create_solution/handler.py @@ -18,6 +18,49 @@ from shared.sfn_middleware import PersonalizeResource +RESOURCE = "solution" +STATUS = "solution.status" +CONFIG = { + "name": { + "source": "event", + "path": "serviceConfig.name", + }, + "performHPO": { + "source": "event", + "path": "serviceConfig.performHPO", + "default": "omit", + }, + "performAutoML": { + "source": "event", + "path": "serviceConfig.performAutoML", + "default": "omit", + }, + "recipeArn": { + "source": "event", + "path": "serviceConfig.recipeArn", + "default": "omit", + }, + "datasetGroupArn": { + "source": "event", + "path": "serviceConfig.datasetGroupArn", + }, + "eventType": { + "source": "event", + "path": "serviceConfig.eventType", + "default": "omit", + }, + "solutionConfig": { + "source": "event", + "path": "serviceConfig.solutionConfig", + "default": "omit", + }, + "timeStarted": { + "source": "event", + "path": "workflowConfig.timeStarted", + "default": "omit", + "as": "iso8601", + }, +} logger = Logger() tracer = Tracer() metrics = Metrics() @@ -26,49 +69,9 @@ @metrics.log_metrics @tracer.capture_lambda_handler @PersonalizeResource( - resource="solution", - status="solution.status", - config={ - "name": { - "source": "event", - "path": "serviceConfig.name", - }, - "performHPO": { - "source": "event", - "path": "serviceConfig.performHPO", - "default": "omit", - }, - "performAutoML": { - "source": "event", - "path": "serviceConfig.performAutoML", - "default": "omit", - }, - "recipeArn": { - "source": "event", - "path": "serviceConfig.recipeArn", - "default": "omit", - }, - "datasetGroupArn": { - "source": "event", - "path": "serviceConfig.datasetGroupArn", - }, - "eventType": { - "source": "event", - "path": "serviceConfig.eventType", - "default": "omit", - }, - "solutionConfig": { - "source": "event", - "path": "serviceConfig.solutionConfig", - "default": "omit", - }, - "timeStarted": { - "source": "event", - "path": "workflowConfig.timeStarted", - "default": "omit", - "as": "iso8601", - }, - }, + resource=RESOURCE, + status=STATUS, + config=CONFIG, ) def lambda_handler(event: Dict[str, Any], context: LambdaContext) -> Dict: """Create a solution in Amazon Personalize based on the configuration in `event` diff --git a/source/aws_lambda/create_solution_version/handler.py b/source/aws_lambda/create_solution_version/handler.py index 6f38648..594c5c8 100644 --- a/source/aws_lambda/create_solution_version/handler.py +++ b/source/aws_lambda/create_solution_version/handler.py @@ -18,6 +18,36 @@ from shared.sfn_middleware import PersonalizeResource +RESOURCE = "solutionVersion" +STATUS = "solutionVersion.status" +CONFIG = { + "solutionArn": { + "source": "event", + "path": "serviceConfig.solutionArn", + }, + "trainingMode": { + "source": "event", + "path": "serviceConfig.trainingMode", + "default": "omit", + }, + "maxAge": { + "source": "event", + "path": "workflowConfig.maxAge", + "default": "omit", + "as": "seconds", + }, + "solutionVersionArn": { + "source": "event", + "path": "workflowConfig.solutionVersionArn", + "default": "omit", + }, + "timeStarted": { + "source": "event", + "path": "workflowConfig.timeStarted", + "default": "omit", + "as": "iso8601", + }, +} logger = Logger() tracer = Tracer() metrics = Metrics() @@ -26,36 +56,9 @@ @metrics.log_metrics @tracer.capture_lambda_handler @PersonalizeResource( - resource="solutionVersion", - status="solutionVersion.status", - config={ - "solutionArn": { - "source": "event", - "path": "serviceConfig.solutionArn", - }, - "trainingMode": { - "source": "event", - "path": "serviceConfig.trainingMode", - "default": "omit", - }, - "maxAge": { - "source": "event", - "path": "workflowConfig.maxAge", - "default": "omit", - "as": "seconds", - }, - "solutionVersionArn": { - "source": "event", - "path": "workflowConfig.solutionVersionArn", - "default": "omit", - }, - "timeStarted": { - "source": "event", - "path": "workflowConfig.timeStarted", - "default": "omit", - "as": "iso8601", - }, - }, + resource=RESOURCE, + status=STATUS, + config=CONFIG, ) def lambda_handler(event: Dict[str, Any], context: LambdaContext) -> Dict: """Create a solution version in Amazon Personalize based on the configuration in `event` diff --git a/source/aws_lambda/shared/personalize_service.py b/source/aws_lambda/shared/personalize_service.py index 511e1fb..3d185f3 100644 --- a/source/aws_lambda/shared/personalize_service.py +++ b/source/aws_lambda/shared/personalize_service.py @@ -47,6 +47,7 @@ Solution, SolutionVersion, BatchInferenceJob, + BatchSegmentJob, Schema, Filter, Campaign, @@ -116,6 +117,8 @@ def describe(self, resource: Resource, **kwargs): return self.describe_event_tracker(**kwargs) elif resource.name.camel == "batchInferenceJob": return self.describe_batch_inference_job(**kwargs) + elif resource.name.camel == "batchSegmentJob": + return self.describe_batch_segment_job(**kwargs) elif resource.name.camel == "campaign": return self.describe_with_update(resource, **kwargs) else: @@ -388,6 +391,17 @@ def is_active_batch_inference_job(job: Dict): **kwargs, ) + def describe_batch_segment_job(self, **kwargs): + def is_active_batch_segment_job(job: Dict): + return self.is_current(new_job=kwargs, old_job=job, name_key="jobName") + + return self._describe_from_parent( + resource=BatchSegmentJob(), + parent=SolutionVersion(), + condition=is_active_batch_segment_job, + **kwargs, + ) + @Notifies("UPDATING") def update(self, resource: Resource, **kwargs): update_fn_name = f"update_{resource.name.snake}" @@ -546,6 +560,13 @@ class Configuration: ] ] }, + { + "recommenders": [ + [ + "serviceConfig", + ] + ] + }, { "solutions": [ [ @@ -560,6 +581,14 @@ class Configuration: ] ] }, + { + "batchSegmentJobs": [ + [ + "serviceConfig", + {"workflowConfig": ["schedule", "maxAge"]}, + ] + ] + }, ] ] }, @@ -684,6 +713,18 @@ def _validate_solutions(self, path="solutions[]"): batch_inference_jobs=batch_inference_jobs, ) + batch_segment_jobs = _solution.get("batchSegmentJobs", []) + if batch_segment_jobs and self._validate_type( + batch_segment_jobs, + list, + f"solutions[{idx}].batchSegmentJobs must be a list", + ): + self._validate_batch_segment_jobs( + path=f"solutions[{idx}].batchSegmentJobs", + solution_name=_solution.get("serviceConfig", {}).get("name", ""), + batch_segment_jobs=batch_segment_jobs, + ) + _solution = _solution.get("serviceConfig") if not self._validate_type( _solution, dict, f"solutions[{idx}].serviceConfig must be an object" @@ -767,6 +808,39 @@ def _validate_batch_inference_jobs( ) self._validate_resource(BatchInferenceJob(), batch_job) + def _validate_batch_segment_jobs( + self, path, solution_name, batch_segment_jobs: List[Dict] + ): + for idx, batch_job_config in enumerate(batch_segment_jobs): + current_path = f"{path}.batchSegmentJobs[{idx}]" + + batch_job = batch_job_config.get("serviceConfig") + if not self._validate_type( + batch_job, dict, f"{current_path}.batchSegmentJob must be an object" + ): + continue + else: + # service does not validate the batch job length client-side + job_name = f"batch_{solution_name}_{datetime.now().strftime('%Y_%m_%d_%H_%M_%S')}" + if len(job_name) > 63: + self._configuration_errors.append( + f"The generated batch segment job name {job_name} is longer than 63 characters. Use a shorter solution name." + ) + + # some values are provided by the solution - we introduce placeholders + batch_job.update( + { + "solutionVersionArn": SolutionVersion().arn("validation"), + "jobName": job_name, + "roleArn": "roleArn", + "jobInput": {"s3DataSource": {"path": "s3://data-source"}}, + "jobOutput": { + "s3DataDestination": {"path": "s3://data-destination"} + }, + } + ) + self._validate_resource(BatchSegmentJob(), batch_job) + def _validate_rate(self, expression): rate_re = re.compile( r"rate\((?P\d+) (?P(minutes?|hours?|day?s)\))" diff --git a/source/aws_lambda/shared/resource/__init__.py b/source/aws_lambda/shared/resource/__init__.py index e407fdf..662ed40 100644 --- a/source/aws_lambda/shared/resource/__init__.py +++ b/source/aws_lambda/shared/resource/__init__.py @@ -13,12 +13,14 @@ from shared.resource.base import Resource from shared.resource.batch_inference_job import BatchInferenceJob +from shared.resource.batch_segment_job import BatchSegmentJob from shared.resource.campaign import Campaign from shared.resource.dataset import Dataset from shared.resource.dataset_group import DatasetGroup from shared.resource.dataset_import_job import DatasetImportJob from shared.resource.event_tracker import EventTracker from shared.resource.filter import Filter +from shared.resource.recommender import Recommender from shared.resource.schema import Schema from shared.resource.solution import Solution from shared.resource.solution_version import SolutionVersion @@ -36,6 +38,8 @@ def get_resource(resource_type: str) -> Resource: "eventTracker": EventTracker(), "filter": Filter(), "batchInferenceJob": BatchInferenceJob(), + "batchSegmentJob": BatchSegmentJob(), + "recommender": Recommender(), }[resource_type] @@ -50,4 +54,6 @@ def get_resource(resource_type: str) -> Resource: EventTracker(), Filter(), BatchInferenceJob(), + BatchSegmentJob(), + Recommender(), ] diff --git a/source/aws_lambda/shared/resource/batch_segment_job.py b/source/aws_lambda/shared/resource/batch_segment_job.py new file mode 100644 index 0000000..2b32390 --- /dev/null +++ b/source/aws_lambda/shared/resource/batch_segment_job.py @@ -0,0 +1,17 @@ +# ###################################################################################################################### +# Copyright Amazon.com, Inc. or its affiliates. All Rights Reserved. # +# # +# Licensed under the Apache License, Version 2.0 (the "License"). You may not use this file except in compliance # +# with the License. You may obtain a copy of the License at # +# # +# http://www.apache.org/licenses/LICENSE-2.0 # +# # +# Unless required by applicable law or agreed to in writing, software distributed under the License is distributed # +# on an "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. See the License for # +# the specific language governing permissions and limitations under the License. # +# ###################################################################################################################### +from shared.resource.base import Resource + + +class BatchSegmentJob(Resource): + pass diff --git a/source/aws_lambda/shared/resource/dataset_group.py b/source/aws_lambda/shared/resource/dataset_group.py index 4cc9312..a3c32bf 100644 --- a/source/aws_lambda/shared/resource/dataset_group.py +++ b/source/aws_lambda/shared/resource/dataset_group.py @@ -14,8 +14,9 @@ from shared.resource.dataset import Dataset from shared.resource.event_tracker import EventTracker from shared.resource.filter import Filter +from shared.resource.recommender import Recommender from shared.resource.solution import Solution class DatasetGroup(Resource): - children = [Dataset(), Filter(), Solution(), EventTracker()] + children = [Dataset(), Filter(), Solution(), Recommender(), EventTracker()] diff --git a/source/aws_lambda/shared/resource/recommender.py b/source/aws_lambda/shared/resource/recommender.py new file mode 100644 index 0000000..12eab11 --- /dev/null +++ b/source/aws_lambda/shared/resource/recommender.py @@ -0,0 +1,19 @@ +# ###################################################################################################################### +# Copyright Amazon.com, Inc. or its affiliates. All Rights Reserved. # +# # +# Licensed under the Apache License, Version 2.0 (the "License"). You may not use this file except in compliance # +# with the License. You may obtain a copy of the License at # +# # +# http://www.apache.org/licenses/LICENSE-2.0 # +# # +# Unless required by applicable law or agreed to in writing, software distributed under the License is distributed # +# on an "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. See the License for # +# the specific language governing permissions and limitations under the License. # +# ###################################################################################################################### +from shared.resource.base import Resource +from shared.resource.batch_inference_job import BatchInferenceJob +from shared.resource.batch_segment_job import BatchSegmentJob + + +class Recommender(Resource): + children = [BatchInferenceJob(), BatchSegmentJob()] diff --git a/source/aws_lambda/shared/resource/solution_version.py b/source/aws_lambda/shared/resource/solution_version.py index f4f359a..230a9da 100644 --- a/source/aws_lambda/shared/resource/solution_version.py +++ b/source/aws_lambda/shared/resource/solution_version.py @@ -12,8 +12,9 @@ # ###################################################################################################################### from shared.resource.base import Resource from shared.resource.batch_inference_job import BatchInferenceJob +from shared.resource.batch_segment_job import BatchSegmentJob class SolutionVersion(Resource): - children = [BatchInferenceJob()] + children = [BatchInferenceJob(), BatchSegmentJob()] has_soft_limit = True diff --git a/source/aws_lambda/shared/sfn_middleware.py b/source/aws_lambda/shared/sfn_middleware.py index 89d5428..0f17ebf 100644 --- a/source/aws_lambda/shared/sfn_middleware.py +++ b/source/aws_lambda/shared/sfn_middleware.py @@ -82,8 +82,10 @@ def set_workflow_config(config: Dict) -> Dict: resources = { "datasetGroup": Arity.ONE, "solutions": Arity.MANY, + "recommenders": Arity.MANY, "campaigns": Arity.MANY, "batchInferenceJobs": Arity.MANY, + "batchSegmentJobs": Arity.MANY, "filters": Arity.MANY, "solutionVersion": Arity.ONE, } @@ -126,9 +128,18 @@ def set_defaults(config: Dict) -> Dict: for s_idx, solution in enumerate(solutions): # by default, don't include a solution version config["solutions"][s_idx].setdefault("solutionVersions", []) - # by default, don't include a campaign or batch inference job + # by default, don't include a campaign or batch inference or segment job config["solutions"][s_idx].setdefault("campaigns", []) config["solutions"][s_idx].setdefault("batchInferenceJobs", []) + config["solutions"][s_idx].setdefault("batchSegmentJobs", []) + + # by default, don't include a recommender + recommenders = config.setdefault("recommenders", []) + for r_idx, recommender in enumerate(recommenders): + # by default, don't include a campaign or batch inference or segment job + config["recommenders"][r_idx].setdefault("batchInferenceJobs", []) + config["recommenders"][r_idx].setdefault("batchSegmentJobs", []) + return config @@ -270,10 +281,11 @@ def check_status( # NOSONAR - allow higher complexity "roleArn", }: continue - if self.resource == "batchInferenceJob" and expected_key in { + if self.resource.startswith("batch") and expected_key in { "jobName", "jobInput", "jobOutput", + "roleArn", }: continue if self.resource == "solutionVersion" and expected_key == "trainingMode": diff --git a/source/cdk_solution_helper_py/CHANGELOG.md b/source/cdk_solution_helper_py/CHANGELOG.md index b5b0277..6f2d2c2 100644 --- a/source/cdk_solution_helper_py/CHANGELOG.md +++ b/source/cdk_solution_helper_py/CHANGELOG.md @@ -4,6 +4,10 @@ All notable changes to this project will be documented in this file. The format is based on [Keep a Changelog](https://keepachangelog.com/en/1.0.0/), and this project adheres to [Semantic Versioning](https://semver.org/spec/v2.0.0.html). +## [2.0.0] - 2022-01-31 +### Changed +- support for CDK 2.x added, support for CDK 1.x removed + ## [1.0.0] - 2021-09-23 ### Added - initial release diff --git a/source/cdk_solution_helper_py/README.md b/source/cdk_solution_helper_py/README.md index 4998e97..4c0762a 100644 --- a/source/cdk_solution_helper_py/README.md +++ b/source/cdk_solution_helper_py/README.md @@ -11,7 +11,7 @@ This README summarizes using the tool. Install this package. It requires at least - Python 3.7 -- AWS CDK version 1.95.2 or higher +- AWS CDK version 2.7.0 or higher To install the packages: @@ -62,13 +62,14 @@ This might be a file called `app.py` in your CDK application directory import logging from pathlib import Path -from aws_cdk import core -from aws_cdk.core import CfnParameter, Construct +from aws_cdk import CfnParameter, App +from constructs import Construct from aws_solutions.cdk import CDKSolution from aws_solutions.cdk.stack import SolutionStack from aws_solutions.cdk.aws_lambda.python.function import SolutionsPythonFunction + # The solution helper build script expects this logger to be used logger = logging.getLogger("cdk-helper") @@ -119,7 +120,7 @@ def build_app(context): The @solution.context decorators indicate that those are required CDK context variables The solution.synthesizer is required as a synthesizer for each solution stack """ - app = core.App(context=context) + app = App(context=context) # add constructs to your CDK app that are compatible with AWS Solutions MyStack( @@ -182,7 +183,7 @@ export DIST_OUTPUT_BUCKET=my-bucket-name export SOLUTION_NAME=my-solution-name export VERSION=my-version -build-s3-cdk-dist --source-bucket-name $DIST_OUTPUT_BUCKET --solution-name $SOLUTION_NAME --version-code $VERSION --cdk-app-path ../source/infrastructure/app.py --cdk-app-entrypoint app:build_app --sync +build-s3-cdk-dist deploy --source-bucket-name $DIST_OUTPUT_BUCKET --solution-name $SOLUTION_NAME --version-code $VERSION --cdk-app-path ../source/infrastructure/app.py --cdk-app-entrypoint app:build_app --sync ``` > **Note**: `build-s3-cdk-dist` will use your current configured `AWS_REGION` and `AWS_PROFILE`. To set your defaults diff --git a/source/cdk_solution_helper_py/helpers_cdk/aws_solutions/cdk/aspects.py b/source/cdk_solution_helper_py/helpers_cdk/aws_solutions/cdk/aspects.py index aa55560..29f0fec 100644 --- a/source/cdk_solution_helper_py/helpers_cdk/aws_solutions/cdk/aspects.py +++ b/source/cdk_solution_helper_py/helpers_cdk/aws_solutions/cdk/aspects.py @@ -11,7 +11,8 @@ # the specific language governing permissions and limitations under the License. # # ##################################################################################################################### import jsii -from aws_cdk.core import CfnCondition, IAspect, IConstruct +from aws_cdk import CfnCondition, IAspect +from constructs import IConstruct @jsii.implements(IAspect) diff --git a/source/cdk_solution_helper_py/helpers_cdk/aws_solutions/cdk/aws_lambda/cfn_custom_resources/resource_hash/hash.py b/source/cdk_solution_helper_py/helpers_cdk/aws_solutions/cdk/aws_lambda/cfn_custom_resources/resource_hash/hash.py index d912e44..e6b74e2 100644 --- a/source/cdk_solution_helper_py/helpers_cdk/aws_solutions/cdk/aws_lambda/cfn_custom_resources/resource_hash/hash.py +++ b/source/cdk_solution_helper_py/helpers_cdk/aws_solutions/cdk/aws_lambda/cfn_custom_resources/resource_hash/hash.py @@ -12,11 +12,11 @@ # ##################################################################################################################### from pathlib import Path -from aws_cdk.core import ( - Construct, +from aws_cdk import ( CfnResource, Stack, ) +from constructs import Construct from aws_solutions.cdk.aws_lambda.python.function import SolutionsPythonFunction from aws_solutions.cdk.cfn_nag import add_cfn_nag_suppressions, CfnNagSuppression diff --git a/source/cdk_solution_helper_py/helpers_cdk/aws_solutions/cdk/aws_lambda/cfn_custom_resources/resource_name/name.py b/source/cdk_solution_helper_py/helpers_cdk/aws_solutions/cdk/aws_lambda/cfn_custom_resources/resource_name/name.py index 48d6cc4..f6f4a65 100644 --- a/source/cdk_solution_helper_py/helpers_cdk/aws_solutions/cdk/aws_lambda/cfn_custom_resources/resource_name/name.py +++ b/source/cdk_solution_helper_py/helpers_cdk/aws_solutions/cdk/aws_lambda/cfn_custom_resources/resource_name/name.py @@ -13,12 +13,12 @@ from pathlib import Path from typing import Optional -from aws_cdk.core import ( - Construct, +from aws_cdk import ( CfnResource, Aws, Stack, ) +from constructs import Construct from aws_solutions.cdk.aws_lambda.python.function import SolutionsPythonFunction from aws_solutions.cdk.cfn_nag import add_cfn_nag_suppressions, CfnNagSuppression diff --git a/source/cdk_solution_helper_py/helpers_cdk/aws_solutions/cdk/aws_lambda/cfn_custom_resources/solutions_metrics/metrics.py b/source/cdk_solution_helper_py/helpers_cdk/aws_solutions/cdk/aws_lambda/cfn_custom_resources/solutions_metrics/metrics.py index 50d4ee0..88d0348 100644 --- a/source/cdk_solution_helper_py/helpers_cdk/aws_solutions/cdk/aws_lambda/cfn_custom_resources/solutions_metrics/metrics.py +++ b/source/cdk_solution_helper_py/helpers_cdk/aws_solutions/cdk/aws_lambda/cfn_custom_resources/solutions_metrics/metrics.py @@ -13,13 +13,13 @@ from pathlib import Path from typing import Dict -from aws_cdk.core import ( - Construct, +from aws_cdk import ( CfnResource, Fn, CfnCondition, Aws, ) +from constructs import Construct from aws_solutions.cdk.aws_lambda.python.function import SolutionsPythonFunction from aws_solutions.cdk.cfn_nag import add_cfn_nag_suppressions, CfnNagSuppression diff --git a/source/cdk_solution_helper_py/helpers_cdk/aws_solutions/cdk/aws_lambda/environment.py b/source/cdk_solution_helper_py/helpers_cdk/aws_solutions/cdk/aws_lambda/environment.py index 4388231..66f88a1 100644 --- a/source/cdk_solution_helper_py/helpers_cdk/aws_solutions/cdk/aws_lambda/environment.py +++ b/source/cdk_solution_helper_py/helpers_cdk/aws_solutions/cdk/aws_lambda/environment.py @@ -13,8 +13,8 @@ from dataclasses import dataclass, field +from aws_cdk import Aws from aws_cdk.aws_lambda import IFunction -from aws_cdk.core import Aws from aws_solutions.cdk.aws_lambda.environment_variable import EnvironmentVariable diff --git a/source/cdk_solution_helper_py/helpers_cdk/aws_solutions/cdk/aws_lambda/java/bundling.py b/source/cdk_solution_helper_py/helpers_cdk/aws_solutions/cdk/aws_lambda/java/bundling.py index e7465cc..b501947 100644 --- a/source/cdk_solution_helper_py/helpers_cdk/aws_solutions/cdk/aws_lambda/java/bundling.py +++ b/source/cdk_solution_helper_py/helpers_cdk/aws_solutions/cdk/aws_lambda/java/bundling.py @@ -17,7 +17,7 @@ from typing import Union, Dict, Optional import jsii -from aws_cdk.core import ILocalBundling, BundlingOptions +from aws_cdk import ILocalBundling, BundlingOptions from aws_solutions.cdk.helpers import copytree diff --git a/source/cdk_solution_helper_py/helpers_cdk/aws_solutions/cdk/aws_lambda/java/function.py b/source/cdk_solution_helper_py/helpers_cdk/aws_solutions/cdk/aws_lambda/java/function.py index f7a24af..7e79d3c 100644 --- a/source/cdk_solution_helper_py/helpers_cdk/aws_solutions/cdk/aws_lambda/java/function.py +++ b/source/cdk_solution_helper_py/helpers_cdk/aws_solutions/cdk/aws_lambda/java/function.py @@ -14,14 +14,14 @@ from typing import Optional import aws_cdk.aws_iam as iam -from aws_cdk.aws_lambda import Function, Runtime, RuntimeFamily, Code -from aws_cdk.core import ( - Construct, +from aws_cdk import ( BundlingOptions, - BundlingDockerImage, BundlingOutput, Aws, + DockerImage, ) +from aws_cdk.aws_lambda import Function, Runtime, RuntimeFamily, Code +from constructs import Construct from aws_solutions.cdk.aws_lambda.java.bundling import SolutionsJavaBundling @@ -79,7 +79,7 @@ def __init__( kwargs["code"] = Code.from_asset( path=str(project_path), bundling=BundlingOptions( - image=BundlingDockerImage.from_registry("scratch"), # NOT USED + image=DockerImage.from_registry("scratch"), # NOT USED command=["NOT-USED"], entrypoint=["NOT-USED"], local=bundling, diff --git a/source/cdk_solution_helper_py/helpers_cdk/aws_solutions/cdk/aws_lambda/layers/aws_lambda_powertools/layer.py b/source/cdk_solution_helper_py/helpers_cdk/aws_solutions/cdk/aws_lambda/layers/aws_lambda_powertools/layer.py index c46f933..f5ad9d0 100644 --- a/source/cdk_solution_helper_py/helpers_cdk/aws_solutions/cdk/aws_lambda/layers/aws_lambda_powertools/layer.py +++ b/source/cdk_solution_helper_py/helpers_cdk/aws_solutions/cdk/aws_lambda/layers/aws_lambda_powertools/layer.py @@ -13,7 +13,8 @@ from pathlib import Path -from aws_cdk.core import Construct, Stack +from aws_cdk import Stack +from constructs import Construct from aws_solutions.cdk.aws_lambda.python.layer import SolutionsPythonLayerVersion diff --git a/source/cdk_solution_helper_py/helpers_cdk/aws_solutions/cdk/aws_lambda/layers/aws_lambda_powertools/requirements/requirements.txt b/source/cdk_solution_helper_py/helpers_cdk/aws_solutions/cdk/aws_lambda/layers/aws_lambda_powertools/requirements/requirements.txt index b28f931..b36c5ee 100644 --- a/source/cdk_solution_helper_py/helpers_cdk/aws_solutions/cdk/aws_lambda/layers/aws_lambda_powertools/requirements/requirements.txt +++ b/source/cdk_solution_helper_py/helpers_cdk/aws_solutions/cdk/aws_lambda/layers/aws_lambda_powertools/requirements/requirements.txt @@ -1 +1 @@ -aws-lambda-powertools==1.15.0 \ No newline at end of file +aws-lambda-powertools>=1.24.0 \ No newline at end of file diff --git a/source/cdk_solution_helper_py/helpers_cdk/aws_solutions/cdk/aws_lambda/python/bundling.py b/source/cdk_solution_helper_py/helpers_cdk/aws_solutions/cdk/aws_lambda/python/bundling.py index d78785b..60a8447 100644 --- a/source/cdk_solution_helper_py/helpers_cdk/aws_solutions/cdk/aws_lambda/python/bundling.py +++ b/source/cdk_solution_helper_py/helpers_cdk/aws_solutions/cdk/aws_lambda/python/bundling.py @@ -20,8 +20,8 @@ from typing import Dict, Union import jsii +from aws_cdk import ILocalBundling, BundlingOptions from aws_cdk.aws_lambda import Runtime -from aws_cdk.core import ILocalBundling, BundlingOptions from aws_solutions.cdk.helpers import copytree @@ -165,7 +165,6 @@ def _local_bundle_with_pip(self, output_dir): str(requirements_build_path), "-r", str(Path(output_dir) / REQUIREMENTS_TXT_FILE), - "--use-feature=in-tree-build", ] self._invoke_local_command("pip", command, cwd=self.to_bundle) diff --git a/source/cdk_solution_helper_py/helpers_cdk/aws_solutions/cdk/aws_lambda/python/function.py b/source/cdk_solution_helper_py/helpers_cdk/aws_solutions/cdk/aws_lambda/python/function.py index 39ccc53..bedfdc6 100644 --- a/source/cdk_solution_helper_py/helpers_cdk/aws_solutions/cdk/aws_lambda/python/function.py +++ b/source/cdk_solution_helper_py/helpers_cdk/aws_solutions/cdk/aws_lambda/python/function.py @@ -17,14 +17,14 @@ from typing import List, Union import aws_cdk.aws_iam as iam -from aws_cdk.aws_lambda import Function, Runtime, RuntimeFamily, Code -from aws_cdk.core import ( - Construct, +from aws_cdk import ( AssetHashType, BundlingOptions, - BundlingDockerImage, + DockerImage, Aws, ) +from aws_cdk.aws_lambda import Function, Runtime, RuntimeFamily, Code +from constructs import Construct from aws_solutions.cdk.aws_lambda.python.bundling import SolutionsPythonBundling @@ -148,7 +148,7 @@ def _get_code(self, bundling: SolutionsPythonBundling, runtime: Runtime) -> Code # to enable docker only bundling, use image=self._get_bundling_docker_image(bundling, runtime=runtime) code = Code.from_asset( bundling=BundlingOptions( - image=BundlingDockerImage.from_registry( + image=DockerImage.from_registry( "scratch" ), # NOT USED - FOR NOW ALL BUNDLING IS LOCAL command=["NOT-USED"], diff --git a/source/cdk_solution_helper_py/helpers_cdk/aws_solutions/cdk/aws_lambda/python/layer.py b/source/cdk_solution_helper_py/helpers_cdk/aws_solutions/cdk/aws_lambda/python/layer.py index 5051c83..e7f8add 100644 --- a/source/cdk_solution_helper_py/helpers_cdk/aws_solutions/cdk/aws_lambda/python/layer.py +++ b/source/cdk_solution_helper_py/helpers_cdk/aws_solutions/cdk/aws_lambda/python/layer.py @@ -13,12 +13,16 @@ from pathlib import Path from typing import Union, List +from uuid import uuid4 +from aws_cdk import BundlingOptions, DockerImage, AssetHashType from aws_cdk.aws_lambda import LayerVersion, Code -from aws_cdk.core import Construct, BundlingOptions, BundlingDockerImage, AssetHashType +from constructs import Construct from aws_solutions.cdk.aws_lambda.python.function import SolutionsPythonBundling +DEPENDENCY_EXCLUDES = ["*.pyc"] + class SolutionsPythonLayerVersion(LayerVersion): """Handle local packaging of layer versions""" @@ -61,12 +65,14 @@ def _get_code(self, bundling: SolutionsPythonBundling) -> Code: # create the layer version locally code_parameters = { "path": str(self.requirements_path), - "asset_hash_type": AssetHashType.SOURCE, + "asset_hash_type": AssetHashType.CUSTOM, + "asset_hash": uuid4().hex, + "exclude": DEPENDENCY_EXCLUDES, } code = Code.from_asset( bundling=BundlingOptions( - image=BundlingDockerImage.from_registry( + image=DockerImage.from_registry( "scratch" ), # NEVER USED - FOR NOW ALL BUNDLING IS LOCAL command=["not_used"], diff --git a/source/cdk_solution_helper_py/helpers_cdk/aws_solutions/cdk/cfn_nag.py b/source/cdk_solution_helper_py/helpers_cdk/aws_solutions/cdk/cfn_nag.py index 84c7c28..8440e31 100644 --- a/source/cdk_solution_helper_py/helpers_cdk/aws_solutions/cdk/cfn_nag.py +++ b/source/cdk_solution_helper_py/helpers_cdk/aws_solutions/cdk/cfn_nag.py @@ -15,7 +15,8 @@ from typing import List import jsii -from aws_cdk.core import CfnResource, IAspect, IConstruct +from aws_cdk import CfnResource, IAspect +from constructs import IConstruct @dataclass diff --git a/source/cdk_solution_helper_py/helpers_cdk/aws_solutions/cdk/interfaces.py b/source/cdk_solution_helper_py/helpers_cdk/aws_solutions/cdk/interfaces.py index 5321524..c27737a 100644 --- a/source/cdk_solution_helper_py/helpers_cdk/aws_solutions/cdk/interfaces.py +++ b/source/cdk_solution_helper_py/helpers_cdk/aws_solutions/cdk/interfaces.py @@ -16,7 +16,7 @@ from typing import Union, List import jsii -from aws_cdk.core import ( +from aws_cdk import ( ITemplateOptions, Stack, NestedStack, @@ -102,6 +102,10 @@ def _get_metadata(self) -> dict: }, }, "aws:solutions:templatename": self.filename, + "aws:solutions:solution_id": self.stack.node.try_get_context("SOLUTION_ID"), + "aws:solutions:solution_version": self.stack.node.try_get_context( + "SOLUTION_VERSION" + ), } self.stack.template_options.metadata = metadata return metadata diff --git a/source/cdk_solution_helper_py/helpers_cdk/aws_solutions/cdk/mappings.py b/source/cdk_solution_helper_py/helpers_cdk/aws_solutions/cdk/mappings.py index fee788a..9f1e716 100644 --- a/source/cdk_solution_helper_py/helpers_cdk/aws_solutions/cdk/mappings.py +++ b/source/cdk_solution_helper_py/helpers_cdk/aws_solutions/cdk/mappings.py @@ -11,7 +11,8 @@ # the specific language governing permissions and limitations under the License. # # ##################################################################################################################### -from aws_cdk.core import Construct, CfnMapping +from aws_cdk import CfnMapping +from constructs import Construct class Mappings: diff --git a/source/cdk_solution_helper_py/helpers_cdk/aws_solutions/cdk/stack.py b/source/cdk_solution_helper_py/helpers_cdk/aws_solutions/cdk/stack.py index bf75bf0..4af2b71 100644 --- a/source/cdk_solution_helper_py/helpers_cdk/aws_solutions/cdk/stack.py +++ b/source/cdk_solution_helper_py/helpers_cdk/aws_solutions/cdk/stack.py @@ -11,9 +11,13 @@ # the specific language governing permissions and limitations under the License. # # ##################################################################################################################### +from __future__ import annotations + import re -from aws_cdk.core import Stack, Construct +import jsii +from aws_cdk import Stack, Aspects, IAspect +from constructs import Construct, IConstruct from aws_solutions.cdk.aws_lambda.cfn_custom_resources.solutions_metrics import Metrics from aws_solutions.cdk.interfaces import TemplateOptions @@ -37,6 +41,17 @@ def validate_template_filename(template_filename: str) -> str: return validate_re("template_filename", template_filename, RE_TEMPLATE_FILENAME) +@jsii.implements(IAspect) +class MetricsAspect: + def __init__(self, stack: SolutionStack): + self.stack = stack + + def visit(self, node: IConstruct): + """Called before synthesis, this allows us to set metrics at the end of synthesis""" + if node == self.stack: + self.stack.metrics = Metrics(self.stack, "Metrics", self.stack.metrics) + + class SolutionStack(Stack): def __init__( self, @@ -60,7 +75,4 @@ def __init__( description=f"({self.solution_id}) - {self.description}. Version {self.solution_version}", filename=template_filename, ) - - def _prepare(self) -> None: - """Called before synthesis, this allows us to set metrics at the end of synthesis""" - self.metrics = Metrics(self, "Metrics", self.metrics) + Aspects.of(self).add(MetricsAspect(self)) diff --git a/source/cdk_solution_helper_py/helpers_cdk/aws_solutions/cdk/stepfunctions/solution_fragment.py b/source/cdk_solution_helper_py/helpers_cdk/aws_solutions/cdk/stepfunctions/solution_fragment.py index 48e0c0e..9e98d86 100644 --- a/source/cdk_solution_helper_py/helpers_cdk/aws_solutions/cdk/stepfunctions/solution_fragment.py +++ b/source/cdk_solution_helper_py/helpers_cdk/aws_solutions/cdk/stepfunctions/solution_fragment.py @@ -13,10 +13,11 @@ from typing import List, Dict from typing import Optional +from aws_cdk import Duration from aws_cdk.aws_lambda import CfnFunction from aws_cdk.aws_stepfunctions import State, INextable, TaskInput, StateMachineFragment from aws_cdk.aws_stepfunctions_tasks import LambdaInvoke -from aws_cdk.core import Construct, Duration +from constructs import Construct class SolutionFragment(StateMachineFragment): diff --git a/source/cdk_solution_helper_py/helpers_cdk/aws_solutions/cdk/stepfunctions/solutionstep.py b/source/cdk_solution_helper_py/helpers_cdk/aws_solutions/cdk/stepfunctions/solutionstep.py index 279e61b..088e680 100644 --- a/source/cdk_solution_helper_py/helpers_cdk/aws_solutions/cdk/stepfunctions/solutionstep.py +++ b/source/cdk_solution_helper_py/helpers_cdk/aws_solutions/cdk/stepfunctions/solutionstep.py @@ -14,10 +14,11 @@ from pathlib import Path from typing import Optional, List +from aws_cdk import Duration from aws_cdk.aws_events import EventBus from aws_cdk.aws_lambda import Tracing, Runtime, RuntimeFamily from aws_cdk.aws_stepfunctions import IChainable, TaskInput, State -from aws_cdk.core import Construct, Duration +from constructs import Construct from aws_solutions.cdk.aws_lambda.environment import Environment from aws_solutions.cdk.aws_lambda.python.function import SolutionsPythonFunction diff --git a/source/cdk_solution_helper_py/helpers_cdk/aws_solutions/cdk/synthesizers.py b/source/cdk_solution_helper_py/helpers_cdk/aws_solutions/cdk/synthesizers.py index e7d7711..764cfa8 100644 --- a/source/cdk_solution_helper_py/helpers_cdk/aws_solutions/cdk/synthesizers.py +++ b/source/cdk_solution_helper_py/helpers_cdk/aws_solutions/cdk/synthesizers.py @@ -22,7 +22,7 @@ from typing import List, Dict import jsii -from aws_cdk.core import IStackSynthesizer, DefaultStackSynthesizer, ISynthesisSession +from aws_cdk import IStackSynthesizer, DefaultStackSynthesizer, ISynthesisSession logger = logging.getLogger("cdk-helper") diff --git a/source/cdk_solution_helper_py/helpers_cdk/setup.py b/source/cdk_solution_helper_py/helpers_cdk/setup.py index d830dfb..ae77483 100644 --- a/source/cdk_solution_helper_py/helpers_cdk/setup.py +++ b/source/cdk_solution_helper_py/helpers_cdk/setup.py @@ -49,8 +49,8 @@ def get_version(): ] }, install_requires=[ - "aws-cdk.core>=1.126.0", - "aws-cdk.aws_lambda>=1.126.0", + "pip>=21.3", + "aws_cdk_lib>=2.7.0", "Click>=7.1.2", "boto3>=1.17.52", "requests>=2.24.0", diff --git a/source/cdk_solution_helper_py/helpers_common/setup.py b/source/cdk_solution_helper_py/helpers_common/setup.py index d0a972c..c044268 100644 --- a/source/cdk_solution_helper_py/helpers_common/setup.py +++ b/source/cdk_solution_helper_py/helpers_common/setup.py @@ -43,6 +43,7 @@ def get_version(): packages=setuptools.find_namespace_packages(), install_requires=[ "boto3>=1.17.52", + "pip>=21.3", ], python_requires=">=3.7", classifiers=[ diff --git a/source/cdk_solution_helper_py/requirements-dev.txt b/source/cdk_solution_helper_py/requirements-dev.txt index 31540ea..4187019 100644 --- a/source/cdk_solution_helper_py/requirements-dev.txt +++ b/source/cdk_solution_helper_py/requirements-dev.txt @@ -1,5 +1,4 @@ -aws-cdk.core>=1.126.0 -aws-cdk.aws_lambda>=1.126.0 +aws_cdk_lib>=2.7.0 black boto3>=1.17.49 requests>=2.24.0 diff --git a/source/infrastructure/cdk.json b/source/infrastructure/cdk.json index 07ce503..8ef6ec4 100644 --- a/source/infrastructure/cdk.json +++ b/source/infrastructure/cdk.json @@ -3,14 +3,6 @@ "context": { "SOLUTION_NAME": "Maintaining Personalized Experiences with Machine Learning", "SOLUTION_ID": "SO0170", - "SOLUTION_VERSION": "v1.1.0", - "@aws-cdk/core:newStyleStackSynthesis": "true", - "@aws-cdk/core:enableStackNameDuplicates": "true", - "aws-cdk:enableDiffNoFail": "true", - "@aws-cdk/core:stackRelativeExports": "true", - "@aws-cdk/aws-ecr-assets:dockerIgnoreSupport": true, - "@aws-cdk/aws-secretsmanager:parseOwnedSecretName": true, - "@aws-cdk/aws-kms:defaultKeyPolicies": true, - "@aws-cdk/aws-s3:grantWriteWithoutAcl": true + "SOLUTION_VERSION": "v1.2.0" } } \ No newline at end of file diff --git a/source/infrastructure/deploy.py b/source/infrastructure/deploy.py index f27264d..dd37117 100644 --- a/source/infrastructure/deploy.py +++ b/source/infrastructure/deploy.py @@ -16,7 +16,7 @@ import logging from pathlib import Path -from aws_cdk import core as cdk +from aws_cdk import App from aws_solutions.cdk import CDKSolution from personalize.stack import PersonalizeStack @@ -30,7 +30,7 @@ @solution.context.requires("SOLUTION_VERSION") @solution.context.requires("BUCKET_NAME") def build_app(context): - app = cdk.App(context=context) + app = App(context=context) PersonalizeStack( app, "PersonalizeStack", diff --git a/source/infrastructure/personalize/aws_lambda/functions/__init__.py b/source/infrastructure/personalize/aws_lambda/functions/__init__.py index 1c533ee..6db50b3 100644 --- a/source/infrastructure/personalize/aws_lambda/functions/__init__.py +++ b/source/infrastructure/personalize/aws_lambda/functions/__init__.py @@ -14,6 +14,9 @@ from personalize.aws_lambda.functions.create_batch_inference_job import ( CreateBatchInferenceJob, ) +from personalize.aws_lambda.functions.create_batch_segment_job import ( + CreateBatchSegmentJob, +) from personalize.aws_lambda.functions.create_campaign import CreateCampaign from personalize.aws_lambda.functions.create_config import CreateConfig from personalize.aws_lambda.functions.create_dataset import CreateDataset @@ -23,6 +26,7 @@ ) from personalize.aws_lambda.functions.create_event_tracker import CreateEventTracker from personalize.aws_lambda.functions.create_filter import CreateFilter +from personalize.aws_lambda.functions.create_recommender import CreateRecommender from personalize.aws_lambda.functions.create_scheduled_task import CreateScheduledTask from personalize.aws_lambda.functions.create_schema import CreateSchema from personalize.aws_lambda.functions.create_solution import CreateSolution diff --git a/source/infrastructure/personalize/aws_lambda/functions/create_batch_inference_job.py b/source/infrastructure/personalize/aws_lambda/functions/create_batch_inference_job.py index a38de74..a4d9e04 100644 --- a/source/infrastructure/personalize/aws_lambda/functions/create_batch_inference_job.py +++ b/source/infrastructure/personalize/aws_lambda/functions/create_batch_inference_job.py @@ -13,8 +13,9 @@ from pathlib import Path import aws_cdk.aws_iam as iam +from aws_cdk import Aws from aws_cdk.aws_s3 import IBucket -from aws_cdk.core import Construct, Aws +from constructs import Construct from aws_solutions.cdk.stepfunctions.solutionstep import SolutionStep @@ -30,7 +31,7 @@ def __init__( self.personalize_bucket = personalize_bucket self.personalize_batch_inference_rw_role = iam.Role( scope, - "PersonalizeS3ReadWriteRole", + "PersonalizeInferenceS3ReadWriteRole", description="Grants Amazon Personalize access to read/write to S3 for batch inference jobs", assumed_by=iam.ServicePrincipal("personalize.amazonaws.com"), inline_policies={ diff --git a/source/infrastructure/personalize/aws_lambda/functions/create_batch_segment_job.py b/source/infrastructure/personalize/aws_lambda/functions/create_batch_segment_job.py new file mode 100644 index 0000000..35903e7 --- /dev/null +++ b/source/infrastructure/personalize/aws_lambda/functions/create_batch_segment_job.py @@ -0,0 +1,111 @@ +# ###################################################################################################################### +# Copyright Amazon.com, Inc. or its affiliates. All Rights Reserved. # +# # +# Licensed under the Apache License, Version 2.0 (the "License"). You may not use this file except in compliance # +# with the License. You may obtain a copy of the License at # +# # +# http://www.apache.org/licenses/LICENSE-2.0 # +# # +# Unless required by applicable law or agreed to in writing, software distributed under the License is distributed # +# on an "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. See the License for # +# the specific language governing permissions and limitations under the License. # +# ###################################################################################################################### +from pathlib import Path + +import aws_cdk.aws_iam as iam +from aws_cdk import Aws +from aws_cdk.aws_s3 import IBucket +from constructs import Construct + +from aws_solutions.cdk.stepfunctions.solutionstep import SolutionStep + + +class CreateBatchSegmentJob(SolutionStep): + def __init__( + self, + scope: Construct, + id: str, + personalize_bucket: IBucket, + layers=None, + ): + self.personalize_bucket = personalize_bucket + self.personalize_batch_inference_rw_role = iam.Role( + scope, + "PersonalizeSegmentS3ReadWriteRole", + description="Grants Amazon Personalize access to read/write to S3 for batch segment jobs", + assumed_by=iam.ServicePrincipal("personalize.amazonaws.com"), + inline_policies={ + "PersonalizeS3ReadPolicy": iam.PolicyDocument( + statements=[ + iam.PolicyStatement( + effect=iam.Effect.ALLOW, + actions=["s3:GetObject", "s3:ListBucket", "s3:PutObject"], + resources=[ + personalize_bucket.arn_for_objects("batch/*"), + personalize_bucket.bucket_arn, + ], + ) + ] + ) + }, + ) + personalize_bucket.add_to_resource_policy( + iam.PolicyStatement( + effect=iam.Effect.ALLOW, + actions=["s3:GetObject", "s3:ListBucket", "s3:PutObject"], + resources=[ + personalize_bucket.arn_for_objects("*"), + personalize_bucket.bucket_arn, + ], + principals=[iam.ServicePrincipal("personalize.amazonaws.com")], + ) + ) + + super().__init__( + scope, + id, + layers=layers, + entrypoint=( + Path(__file__).absolute().parents[4] + / "aws_lambda" + / "create_batch_segment_job" + / "handler.py" + ), + libraries=[Path(__file__).absolute().parents[4] / "aws_lambda" / "shared"], + ) + + def _set_permissions(self): + # personalize resource permissions + self.function.add_to_role_policy( + statement=iam.PolicyStatement( + actions=[ + "personalize:DescribeDatasetGroup", + "personalize:ListBatchSegmentJobs", + "personalize:ListSolutionVersions", + "personalize:ListSolutions", + "personalize:CreateBatchSegmentJob", + "personalize:DescribeBatchSegmentJob", + "personalize:DescribeSolution", + "personalize:DescribeSolutionVersion", + ], + effect=iam.Effect.ALLOW, + resources=[ + f"arn:{Aws.PARTITION}:personalize:{Aws.REGION}:{Aws.ACCOUNT_ID}:dataset-group/*", + f"arn:{Aws.PARTITION}:personalize:{Aws.REGION}:{Aws.ACCOUNT_ID}:batch-segment-job/*", + f"arn:{Aws.PARTITION}:personalize:{Aws.REGION}:{Aws.ACCOUNT_ID}:solution/*", + ], + ) + ) + self.personalize_bucket.grant_read_write(self.function, "batch/*") + + # passrole permissions + self.function.add_to_role_policy( + statement=iam.PolicyStatement( + effect=iam.Effect.ALLOW, + actions=["iam:PassRole"], + resources=[self.personalize_batch_inference_rw_role.role_arn], + ) + ) + self.function.add_environment( + "ROLE_ARN", self.personalize_batch_inference_rw_role.role_arn + ) diff --git a/source/infrastructure/personalize/aws_lambda/functions/create_campaign.py b/source/infrastructure/personalize/aws_lambda/functions/create_campaign.py index edf9f68..4e73c3f 100644 --- a/source/infrastructure/personalize/aws_lambda/functions/create_campaign.py +++ b/source/infrastructure/personalize/aws_lambda/functions/create_campaign.py @@ -13,7 +13,8 @@ from pathlib import Path import aws_cdk.aws_iam as iam -from aws_cdk.core import Construct, Aws +from aws_cdk import Aws +from constructs import Construct from aws_solutions.cdk.stepfunctions.solutionstep import SolutionStep diff --git a/source/infrastructure/personalize/aws_lambda/functions/create_config.py b/source/infrastructure/personalize/aws_lambda/functions/create_config.py index 9fc4432..5632d7d 100644 --- a/source/infrastructure/personalize/aws_lambda/functions/create_config.py +++ b/source/infrastructure/personalize/aws_lambda/functions/create_config.py @@ -13,9 +13,10 @@ from pathlib import Path -from aws_cdk.aws_lambda import Tracing, Runtime, RuntimeFamily -from aws_cdk.core import Construct, Duration, Aws import aws_cdk.aws_iam as iam +from aws_cdk import Duration, Aws +from aws_cdk.aws_lambda import Tracing, Runtime, RuntimeFamily +from constructs import Construct from aws_solutions.cdk.aws_lambda.environment import Environment from aws_solutions.cdk.aws_lambda.python.function import SolutionsPythonFunction diff --git a/source/infrastructure/personalize/aws_lambda/functions/create_dataset.py b/source/infrastructure/personalize/aws_lambda/functions/create_dataset.py index 06eace2..7bff7bc 100644 --- a/source/infrastructure/personalize/aws_lambda/functions/create_dataset.py +++ b/source/infrastructure/personalize/aws_lambda/functions/create_dataset.py @@ -14,8 +14,9 @@ from typing import Optional import aws_cdk.aws_iam as iam +from aws_cdk import Aws from aws_cdk.aws_stepfunctions import IChainable -from aws_cdk.core import Construct, Aws +from constructs import Construct from aws_solutions.cdk.stepfunctions.solutionstep import SolutionStep diff --git a/source/infrastructure/personalize/aws_lambda/functions/create_dataset_group.py b/source/infrastructure/personalize/aws_lambda/functions/create_dataset_group.py index 0dcd9ac..73b7f3c 100644 --- a/source/infrastructure/personalize/aws_lambda/functions/create_dataset_group.py +++ b/source/infrastructure/personalize/aws_lambda/functions/create_dataset_group.py @@ -14,8 +14,9 @@ from typing import Optional import aws_cdk.aws_iam as iam +from aws_cdk import Aws, CfnCondition, CfnParameter, Fn from aws_cdk.aws_stepfunctions import IChainable -from aws_cdk.core import Construct, Aws, CfnCondition, CfnParameter, Fn +from constructs import Construct from aws_solutions.cdk.aws_lambda.cfn_custom_resources.resource_hash import ResourceHash from aws_solutions.cdk.cfn_nag import add_cfn_nag_suppressions, CfnNagSuppression diff --git a/source/infrastructure/personalize/aws_lambda/functions/create_dataset_import_job.py b/source/infrastructure/personalize/aws_lambda/functions/create_dataset_import_job.py index 7959c5a..875f7f5 100644 --- a/source/infrastructure/personalize/aws_lambda/functions/create_dataset_import_job.py +++ b/source/infrastructure/personalize/aws_lambda/functions/create_dataset_import_job.py @@ -14,9 +14,10 @@ from typing import Optional import aws_cdk.aws_iam as iam +from aws_cdk import Aws from aws_cdk.aws_s3 import IBucket from aws_cdk.aws_stepfunctions import IChainable -from aws_cdk.core import Construct, Aws +from constructs import Construct from aws_solutions.cdk.stepfunctions.solutionstep import SolutionStep diff --git a/source/infrastructure/personalize/aws_lambda/functions/create_event_tracker.py b/source/infrastructure/personalize/aws_lambda/functions/create_event_tracker.py index 042d817..71b234a 100644 --- a/source/infrastructure/personalize/aws_lambda/functions/create_event_tracker.py +++ b/source/infrastructure/personalize/aws_lambda/functions/create_event_tracker.py @@ -14,8 +14,9 @@ from typing import Optional import aws_cdk.aws_iam as iam +from aws_cdk import Aws from aws_cdk.aws_stepfunctions import IChainable, TaskInput -from aws_cdk.core import Construct, Aws +from constructs import Construct from aws_solutions.cdk.stepfunctions.solutionstep import SolutionStep diff --git a/source/infrastructure/personalize/aws_lambda/functions/create_filter.py b/source/infrastructure/personalize/aws_lambda/functions/create_filter.py index 51e6f09..9c62e14 100644 --- a/source/infrastructure/personalize/aws_lambda/functions/create_filter.py +++ b/source/infrastructure/personalize/aws_lambda/functions/create_filter.py @@ -14,8 +14,9 @@ from typing import Optional import aws_cdk.aws_iam as iam +from aws_cdk import Aws from aws_cdk.aws_stepfunctions import IChainable -from aws_cdk.core import Construct, Aws +from constructs import Construct from aws_solutions.cdk.stepfunctions.solutionstep import SolutionStep diff --git a/source/infrastructure/personalize/aws_lambda/functions/create_recommender.py b/source/infrastructure/personalize/aws_lambda/functions/create_recommender.py new file mode 100644 index 0000000..279dafe --- /dev/null +++ b/source/infrastructure/personalize/aws_lambda/functions/create_recommender.py @@ -0,0 +1,57 @@ +# ###################################################################################################################### +# Copyright Amazon.com, Inc. or its affiliates. All Rights Reserved. # +# # +# Licensed under the Apache License, Version 2.0 (the "License"). You may not use this file except in compliance # +# with the License. You may obtain a copy of the License at # +# # +# http://www.apache.org/licenses/LICENSE-2.0 # +# # +# Unless required by applicable law or agreed to in writing, software distributed under the License is distributed # +# on an "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. See the License for # +# the specific language governing permissions and limitations under the License. # +# ###################################################################################################################### +from pathlib import Path + +import aws_cdk.aws_iam as iam +from aws_cdk import Aws +from constructs import Construct + +from aws_solutions.cdk.stepfunctions.solutionstep import SolutionStep + + +class CreateRecommender(SolutionStep): + def __init__( + self, + scope: Construct, + id: str, + layers=None, + ): + super().__init__( + scope, + id, + layers=layers, + entrypoint=( + Path(__file__).absolute().parents[4] + / "aws_lambda" + / "create_recommender" + / "handler.py" + ), + libraries=[Path(__file__).absolute().parents[4] / "aws_lambda" / "shared"], + ) + + def _set_permissions(self): + self.function.add_to_role_policy( + statement=iam.PolicyStatement( + actions=[ + "personalize:DescribeRecommender", + "personalize:CreateRecommender", + "personalize:ListRecommenders", + "personalize:DescribeDatasetGroup", + ], + effect=iam.Effect.ALLOW, + resources=[ + f"arn:{Aws.PARTITION}:personalize:{Aws.REGION}:{Aws.ACCOUNT_ID}:recommender/*", + f"arn:{Aws.PARTITION}:personalize:{Aws.REGION}:{Aws.ACCOUNT_ID}:dataset-group/*", + ], + ) + ) diff --git a/source/infrastructure/personalize/aws_lambda/functions/create_scheduled_task.py b/source/infrastructure/personalize/aws_lambda/functions/create_scheduled_task.py index 4f7a027..e067eb3 100644 --- a/source/infrastructure/personalize/aws_lambda/functions/create_scheduled_task.py +++ b/source/infrastructure/personalize/aws_lambda/functions/create_scheduled_task.py @@ -12,7 +12,7 @@ # ###################################################################################################################### from pathlib import Path -from aws_cdk.core import Construct +from constructs import Construct from aws_solutions.cdk.stepfunctions.solutionstep import SolutionStep diff --git a/source/infrastructure/personalize/aws_lambda/functions/create_schema.py b/source/infrastructure/personalize/aws_lambda/functions/create_schema.py index 4d11537..6412717 100644 --- a/source/infrastructure/personalize/aws_lambda/functions/create_schema.py +++ b/source/infrastructure/personalize/aws_lambda/functions/create_schema.py @@ -14,8 +14,9 @@ from typing import Optional import aws_cdk.aws_iam as iam +from aws_cdk import Aws from aws_cdk.aws_stepfunctions import IChainable -from aws_cdk.core import Construct, Aws +from constructs import Construct from aws_solutions.cdk.stepfunctions.solutionstep import SolutionStep diff --git a/source/infrastructure/personalize/aws_lambda/functions/create_solution.py b/source/infrastructure/personalize/aws_lambda/functions/create_solution.py index 3c98ab0..03052e3 100644 --- a/source/infrastructure/personalize/aws_lambda/functions/create_solution.py +++ b/source/infrastructure/personalize/aws_lambda/functions/create_solution.py @@ -13,7 +13,8 @@ from pathlib import Path import aws_cdk.aws_iam as iam -from aws_cdk.core import Construct, Aws +from aws_cdk import Aws +from constructs import Construct from aws_solutions.cdk.stepfunctions.solutionstep import SolutionStep diff --git a/source/infrastructure/personalize/aws_lambda/functions/create_solution_version.py b/source/infrastructure/personalize/aws_lambda/functions/create_solution_version.py index cbe3027..164bbe6 100644 --- a/source/infrastructure/personalize/aws_lambda/functions/create_solution_version.py +++ b/source/infrastructure/personalize/aws_lambda/functions/create_solution_version.py @@ -13,7 +13,8 @@ from pathlib import Path import aws_cdk.aws_iam as iam -from aws_cdk.core import Construct, Aws +from aws_cdk import Aws +from constructs import Construct from aws_solutions.cdk.stepfunctions.solutionstep import SolutionStep diff --git a/source/infrastructure/personalize/aws_lambda/functions/create_timestamp.py b/source/infrastructure/personalize/aws_lambda/functions/create_timestamp.py index f83fb91..c9f36a8 100644 --- a/source/infrastructure/personalize/aws_lambda/functions/create_timestamp.py +++ b/source/infrastructure/personalize/aws_lambda/functions/create_timestamp.py @@ -12,7 +12,7 @@ # ###################################################################################################################### from pathlib import Path -from aws_cdk.core import Construct +from constructs import Construct from aws_solutions.cdk.stepfunctions.solutionstep import SolutionStep diff --git a/source/infrastructure/personalize/aws_lambda/functions/environment.py b/source/infrastructure/personalize/aws_lambda/functions/environment.py index 4388231..66f88a1 100644 --- a/source/infrastructure/personalize/aws_lambda/functions/environment.py +++ b/source/infrastructure/personalize/aws_lambda/functions/environment.py @@ -13,8 +13,8 @@ from dataclasses import dataclass, field +from aws_cdk import Aws from aws_cdk.aws_lambda import IFunction -from aws_cdk.core import Aws from aws_solutions.cdk.aws_lambda.environment_variable import EnvironmentVariable diff --git a/source/infrastructure/personalize/aws_lambda/functions/prepare_input.py b/source/infrastructure/personalize/aws_lambda/functions/prepare_input.py index 2194f68..868c7a6 100644 --- a/source/infrastructure/personalize/aws_lambda/functions/prepare_input.py +++ b/source/infrastructure/personalize/aws_lambda/functions/prepare_input.py @@ -13,7 +13,7 @@ from pathlib import Path -from aws_cdk.core import Construct +from constructs import Construct from aws_solutions.cdk.stepfunctions.solutionstep import SolutionStep diff --git a/source/infrastructure/personalize/aws_lambda/functions/s3_event.py b/source/infrastructure/personalize/aws_lambda/functions/s3_event.py index 610c5de..e77dee4 100644 --- a/source/infrastructure/personalize/aws_lambda/functions/s3_event.py +++ b/source/infrastructure/personalize/aws_lambda/functions/s3_event.py @@ -13,11 +13,12 @@ from pathlib import Path +from aws_cdk import Duration from aws_cdk.aws_lambda import Tracing, Runtime, RuntimeFamily from aws_cdk.aws_s3 import Bucket from aws_cdk.aws_sns import Topic from aws_cdk.aws_stepfunctions import StateMachine -from aws_cdk.core import Construct, Duration +from constructs import Construct from aws_solutions.cdk.aws_lambda.environment import Environment from aws_solutions.cdk.aws_lambda.python.function import SolutionsPythonFunction diff --git a/source/infrastructure/personalize/aws_lambda/layers/aws_solutions/layer.py b/source/infrastructure/personalize/aws_lambda/layers/aws_solutions/layer.py index 597c91c..2778d7d 100644 --- a/source/infrastructure/personalize/aws_lambda/layers/aws_solutions/layer.py +++ b/source/infrastructure/personalize/aws_lambda/layers/aws_solutions/layer.py @@ -13,7 +13,8 @@ from pathlib import Path -from aws_cdk.core import Construct, Stack +from aws_cdk import Stack +from constructs import Construct from aws_solutions.cdk.aws_lambda.python.layer import SolutionsPythonLayerVersion diff --git a/source/infrastructure/personalize/aws_lambda/layers/aws_solutions/requirements/requirements.txt b/source/infrastructure/personalize/aws_lambda/layers/aws_solutions/requirements/requirements.txt index 68789cf..43d0685 100644 --- a/source/infrastructure/personalize/aws_lambda/layers/aws_solutions/requirements/requirements.txt +++ b/source/infrastructure/personalize/aws_lambda/layers/aws_solutions/requirements/requirements.txt @@ -3,4 +3,5 @@ avro==1.10.2 cronex==0.1.3.1 jmespath==0.10.0 -parsedatetime==2.6 \ No newline at end of file +parsedatetime==2.6 +boto3>=1.20.28 \ No newline at end of file diff --git a/source/infrastructure/personalize/cloudwatch/dashboard.py b/source/infrastructure/personalize/cloudwatch/dashboard.py index f0c87d8..00c3c1b 100644 --- a/source/infrastructure/personalize/cloudwatch/dashboard.py +++ b/source/infrastructure/personalize/cloudwatch/dashboard.py @@ -13,7 +13,8 @@ from typing import Optional import aws_cdk.aws_cloudwatch as cw -from aws_cdk.core import Construct, Aws +from aws_cdk import Aws +from constructs import Construct GREEN = "#32cd32" RED = "#ff4500" @@ -108,6 +109,11 @@ def __init__( "BatchInferenceJobCreated", "Batch Inference Jobs Created", ), + self._metric( + "BatchSegmentJobCreated", + "Batch Segment Jobs Created", + ), + self._metric("RecommenderCreated", "Recommenders Created"), self._metric("FilterCreated", "Filters Created"), ], set_period_to_time_range=True, @@ -135,7 +141,7 @@ def _metric( return cw.Metric( namespace=f"personalize_solution_{Aws.STACK_NAME}", metric_name=name, - dimensions={"service": service}, + dimensions_map={"service": service}, label=label, statistic="Sum", color=color, diff --git a/source/infrastructure/personalize/s3/utils.py b/source/infrastructure/personalize/s3/utils.py index 49152c7..a15a01f 100644 --- a/source/infrastructure/personalize/s3/utils.py +++ b/source/infrastructure/personalize/s3/utils.py @@ -15,8 +15,9 @@ from typing import List import aws_cdk.aws_iam as iam +from aws_cdk import RemovalPolicy, CfnResource from aws_cdk.aws_s3 import Bucket, BucketEncryption, BlockPublicAccess -from aws_cdk.core import RemovalPolicy, Construct, CfnResource +from constructs import Construct from aws_solutions.cdk.cfn_nag import add_cfn_nag_suppressions, CfnNagSuppression diff --git a/source/infrastructure/personalize/sns/notifications.py b/source/infrastructure/personalize/sns/notifications.py index 86b7811..6ca7c7d 100644 --- a/source/infrastructure/personalize/sns/notifications.py +++ b/source/infrastructure/personalize/sns/notifications.py @@ -13,16 +13,16 @@ from pathlib import Path from typing import Optional -from aws_cdk.aws_sns import Subscription, SubscriptionProtocol -from aws_cdk.aws_sns import TopicProps -from aws_cdk.aws_stepfunctions import IChainable -from aws_cdk.core import ( - Construct, +from aws_cdk import ( CfnParameter, CfnCondition, Aspects, ) +from aws_cdk.aws_sns import Subscription, SubscriptionProtocol +from aws_cdk.aws_sns import TopicProps +from aws_cdk.aws_stepfunctions import IChainable from aws_solutions_constructs.aws_lambda_sns import LambdaToSns +from constructs import Construct from aws_solutions.cdk.aspects import ConditionalResources from aws_solutions.cdk.stepfunctions.solutionstep import SolutionStep diff --git a/source/infrastructure/personalize/stack.py b/source/infrastructure/personalize/stack.py index 3864f5b..48c848a 100644 --- a/source/infrastructure/personalize/stack.py +++ b/source/infrastructure/personalize/stack.py @@ -11,7 +11,16 @@ # the specific language governing permissions and limitations under the License. # # ###################################################################################################################### -from aws_cdk import core as cdk +from aws_cdk import ( + CfnCondition, + Fn, + Aws, + Duration, + CfnOutput, + CfnParameter, + Tags, + Aspects, +) from aws_cdk.aws_events import EventBus from aws_cdk.aws_s3 import EventType, NotificationKeyFilter from aws_cdk.aws_s3_notifications import LambdaDestination @@ -21,7 +30,7 @@ Parallel, TaskInput, ) -from aws_cdk.core import CfnCondition, Fn, Aws, Duration +from constructs import Construct from aws_solutions.cdk.aws_lambda.cfn_custom_resources.resource_name import ResourceName from aws_solutions.cdk.aws_lambda.layers.aws_lambda_powertools import PowertoolsLayer @@ -46,6 +55,8 @@ CreateBatchInferenceJob, CreateTimestamp, CreateConfig, + CreateBatchSegmentJob, + CreateRecommender, ) from personalize.aws_lambda.functions.prepare_input import PrepareInput from personalize.aws_lambda.layers import SolutionsLayer @@ -66,13 +77,11 @@ class PersonalizeStack(SolutionStack): - def __init__( - self, scope: cdk.Construct, construct_id: str, *args, **kwargs - ) -> None: + def __init__(self, scope: Construct, construct_id: str, *args, **kwargs) -> None: super().__init__(scope, construct_id, *args, **kwargs) # CloudFormation Parameters - self.email = cdk.CfnParameter( + self.email = CfnParameter( self, id="Email", type="String", @@ -91,7 +100,7 @@ def __init__( expression=Fn.condition_not(Fn.condition_equals(self.email, "")), ) - self.personalize_kms_key_arn = cdk.CfnParameter( + self.personalize_kms_key_arn = CfnParameter( self, id="PersonalizeKmsKeyArn", description="Provide Amazon Personalize with an alternate AWS Key Management (KMS) key to use to encrypt your datasets", @@ -103,7 +112,7 @@ def __init__( "(Optional) KMS key ARN used to encrypt Datasets managed by Amazon Personalize", "Security Configuration", ) - kms_enabled = cdk.CfnCondition( + kms_enabled = CfnCondition( self, "PersonalizeSseKmsEnabled", expression=Fn.condition_not( @@ -184,6 +193,11 @@ def __init__( "Create Solution", layers=common_layers, ) + create_recommender = CreateRecommender( + self, + "Create Recommender", + layers=common_layers, + ) create_solution_version = CreateSolutionVersion( self, "Create Solution Version", @@ -200,6 +214,12 @@ def __init__( layers=common_layers, personalize_bucket=data_bucket, ) + create_batch_segment_job = CreateBatchSegmentJob( + self, + "Create Batch Segment Job", + layers=common_layers, + personalize_bucket=data_bucket, + ) create_filter = CreateFilter(self, "Create Filter", layers=common_layers) create_timestamp = CreateTimestamp( self, "Create Timestamp", layers=[layer_powertools] @@ -225,10 +245,12 @@ def __init__( create_dataset.grant_put_events(event_bus) create_dataset_import_job.grant_put_events(event_bus) create_event_tracker.grant_put_events(event_bus) + create_recommender.grant_put_events(event_bus) create_solution.grant_put_events(event_bus) create_solution_version.grant_put_events(event_bus) create_campaign.grant_put_events(event_bus) create_batch_inference_job.grant_put_events(event_bus) + create_batch_segment_job.grant_put_events(event_bus) create_filter.grant_put_events(event_bus) dataset_management_functions = { @@ -262,8 +284,10 @@ def __init__( create_solution_version=create_solution_version, create_campaign=create_campaign, create_batch_inference_job=create_batch_inference_job, + create_batch_segment_job=create_batch_segment_job, create_timestamp=create_timestamp, notifications=notifications, + create_recommender=create_recommender, ).state_machine # scheduler and step function to schedule @@ -297,6 +321,8 @@ def __init__( create_solution_version=create_solution_version, create_campaign=create_campaign, create_batch_inference_job=create_batch_inference_job, + create_batch_segment_job=create_batch_segment_job, + create_recommender=create_recommender, scheduler=scheduler, to_schedule=solution_maintenance_schedule_sfn, ) @@ -400,15 +426,13 @@ def __init__( ], ) - cdk.Tags.of(self).add("SOLUTION_ID", self.node.try_get_context("SOLUTION_ID")) - cdk.Tags.of(self).add( - "SOLUTION_NAME", self.node.try_get_context("SOLUTION_NAME") - ) - cdk.Tags.of(self).add( + Tags.of(self).add("SOLUTION_ID", self.node.try_get_context("SOLUTION_ID")) + Tags.of(self).add("SOLUTION_NAME", self.node.try_get_context("SOLUTION_NAME")) + Tags.of(self).add( "SOLUTION_VERSION", self.node.try_get_context("SOLUTION_VERSION") ) - cdk.Aspects.of(self).add( + Aspects.of(self).add( CfnNagSuppressAll( suppress=[ CfnNagSuppression( @@ -437,43 +461,43 @@ def __init__( ) # outputs - cdk.CfnOutput( + CfnOutput( self, "PersonalizeBucketName", value=data_bucket.bucket_name, export_name=f"{Aws.STACK_NAME}-PersonalizeBucketName", ) - cdk.CfnOutput( + CfnOutput( self, "SchedulerTableName", value=scheduler.scheduler_table.table_name, export_name=f"{Aws.STACK_NAME}-SchedulerTableName", ) - cdk.CfnOutput( + CfnOutput( self, "SchedulerStepFunctionArn", value=scheduler.state_machine_arn, export_name=f"{Aws.STACK_NAME}-SchedulerStepFunctionArn", ) - cdk.CfnOutput( + CfnOutput( self, "Dashboard", value=self.dashboard.name, export_name=f"{Aws.STACK_NAME}-Dashboard", ) - cdk.CfnOutput( + CfnOutput( self, "SNSTopicArn", value=notifications.topic.topic_arn, export_name=f"{Aws.STACK_NAME}-SNSTopicArn", ) - cdk.CfnOutput( + CfnOutput( self, "EventBusArn", value=event_bus.event_bus_arn, export_name=f"{Aws.STACK_NAME}-EventBusArn", ) - cdk.CfnOutput( + CfnOutput( self, "CreateConfigFunctionArn", value=create_config.function_arn, diff --git a/source/infrastructure/personalize/step_functions/batch_inference_jobs_fragment.py b/source/infrastructure/personalize/step_functions/batch_inference_jobs_fragment.py index b83ab10..984a23c 100644 --- a/source/infrastructure/personalize/step_functions/batch_inference_jobs_fragment.py +++ b/source/infrastructure/personalize/step_functions/batch_inference_jobs_fragment.py @@ -12,6 +12,7 @@ # ###################################################################################################################### from typing import List, Optional +from aws_cdk import Duration from aws_cdk.aws_stepfunctions import ( StateMachineFragment, State, @@ -23,7 +24,7 @@ Chain, StateMachine, ) -from aws_cdk.core import Construct, Duration +from constructs import Construct from aws_solutions.scheduler.cdk.construct import Scheduler from aws_solutions.scheduler.cdk.scheduler_fragment import SchedulerFragment diff --git a/source/infrastructure/personalize/step_functions/batch_segment_jobs_fragment.py b/source/infrastructure/personalize/step_functions/batch_segment_jobs_fragment.py new file mode 100644 index 0000000..e939c14 --- /dev/null +++ b/source/infrastructure/personalize/step_functions/batch_segment_jobs_fragment.py @@ -0,0 +1,177 @@ +# ###################################################################################################################### +# Copyright Amazon.com, Inc. or its affiliates. All Rights Reserved. # +# # +# Licensed under the Apache License, Version 2.0 (the "License"). You may not use this file except in compliance # +# with the License. You may obtain a copy of the License at # +# # +# http://www.apache.org/licenses/LICENSE-2.0 # +# # +# Unless required by applicable law or agreed to in writing, software distributed under the License is distributed # +# on an "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. See the License for # +# the specific language governing permissions and limitations under the License. # +# ###################################################################################################################### +from typing import List, Optional + +from aws_cdk import Duration +from aws_cdk.aws_stepfunctions import ( + StateMachineFragment, + State, + INextable, + Choice, + Pass, + Map, + Condition, + Chain, + StateMachine, +) +from constructs import Construct + +from aws_solutions.scheduler.cdk.construct import Scheduler +from aws_solutions.scheduler.cdk.scheduler_fragment import SchedulerFragment +from personalize.aws_lambda.functions import CreateBatchSegmentJob + +TEMPORARY_PATH = "$._tmp" +BATCH_SEGMENT_JOB_PATH = "$.batchSegmentJob" +BUCKET_PATH = "$.bucket" +CURRENT_DATE_PATH = "$.currentDate" + + +class BatchSegmentJobsFragment(StateMachineFragment): + def __init__( + self, + scope: Construct, + id: str, + create_batch_segment_job: CreateBatchSegmentJob, + scheduler: Optional[Scheduler] = None, + to_schedule: Optional[StateMachine] = None, + ): + super().__init__(scope, id) + + # total allowed elapsed duration ~ 5h + retry_config = { + "backoff_rate": 1.02, + "interval": Duration.seconds(60), + "max_attempts": 100, + } + + self.batch_segment_jobs_not_available = Pass( + self, "Batch Segment Jobs Not Provided" + ) + batch_segment_jobs_available = Choice( + self, "Check for Batch Segment Jobs" + ).otherwise(self.batch_segment_jobs_not_available) + + _prepare_batch_segment_job_input_job_name = Pass( + self, + "Set Batch Segment Job Input Data - Job Name", + input_path="$.batchSegmentJobName", + result_path=f"{BATCH_SEGMENT_JOB_PATH}.serviceConfig.jobName", + ) + + _prepare_batch_segment_job_input_solution_version_arn = Pass( + self, + "Set Batch Segment Job Input Data - Solution Version ARN", + input_path="$.solutionVersionArn", # NOSONAR (python:S1192) - string for clarity + result_path=f"{BATCH_SEGMENT_JOB_PATH}.serviceConfig.solutionVersionArn", + ) + + _prepare_batch_segment_job_job_input = Pass( + self, + "Set Batch Segment Job Input Data - Job Input", + result_path=f"{BATCH_SEGMENT_JOB_PATH}.serviceConfig.jobInput", + parameters={ + "s3DataSource": { + "path.$": f"States.Format('s3://{{}}/batch/{{}}/{{}}/job_config.json', $.bucket.name, $.datasetGroupName, $.solution.serviceConfig.name)" # NOSONAR (python:S1192) - string for clarity + } + }, + ) + + _prepare_batch_segment_job_job_output = Pass( + self, + "Set Batch Segment Job Input Data - Job Output", + result_path=f"{BATCH_SEGMENT_JOB_PATH}.serviceConfig.jobOutput", + parameters={ + "s3DataDestination": { + "path.$": f"States.Format('s3://{{}}/batch/{{}}/{{}}/{{}}/', $.bucket.name, $.datasetGroupName, $.solution.serviceConfig.name, $.batchSegmentJobName)" # NOSONAR (python:S1192) - string for clarity + } + }, + ) + + _prepare_batch_segment_job_input = Chain.start( + _prepare_batch_segment_job_input_job_name.next( + _prepare_batch_segment_job_input_solution_version_arn + ) + .next(_prepare_batch_segment_job_job_input) + .next(_prepare_batch_segment_job_job_output) + ) + + _create_batch_segment_job = create_batch_segment_job.state( + self, + "Create Batch Segment Job", + result_path=f"{BATCH_SEGMENT_JOB_PATH}.serviceConfig", + input_path=f"{BATCH_SEGMENT_JOB_PATH}", + **retry_config, + ) + if scheduler and to_schedule: + _create_batch_segment_job.next( + SchedulerFragment( + self, + schedule_for="batch segment", + schedule_for_suffix="$.solution.serviceConfig.name", # NOSONAR (python:S1192) - string for clarity + scheduler=scheduler, + target=to_schedule, + schedule_path="$.batchSegmentJob.workflowConfig.schedule", + schedule_input={ + "bucket.$": "$.bucket", + "datasetGroup": { + "serviceConfig": { + "name.$": "$.datasetGroupName", + "datasetGroupArn.$": "$.datasetGroupArn", + } + }, + "solutions": [ + { + "serviceConfig.$": "$.solution.serviceConfig", + "batchSegmentJobs": [ + { + "serviceConfig.$": "$.batchSegmentJob.serviceConfig", + "workflowConfig": {"maxAge": "1 second"}, + } + ], + } + ], + }, + ) + ) + + self.create_batch_segment_jobs = batch_segment_jobs_available.when( + Condition.is_present("$.solution.batchSegmentJobs[0]"), + Map( + self, + "Create Batch Segment Jobs", + items_path="$.solution.batchSegmentJobs", + parameters={ + "solutionVersionArn.$": "$.solution.solutionVersion.serviceConfig.solutionVersionArn", + "batchSegmentJob.$": "$$.Map.Item.Value", + "batchSegmentJobName.$": f"States.Format('batch_{{}}_{{}}', $.solution.serviceConfig.name, {CURRENT_DATE_PATH})", + "bucket.$": BUCKET_PATH, # NOSONAR (python:S1192) - string for clarity + "currentDate.$": CURRENT_DATE_PATH, # NOSONAR (python:S1192) - string for clarity + "datasetGroupName.$": "$.datasetGroupName", + "datasetGroupArn.$": "$.datasetGroupArn", + "solution.$": "$.solution", + }, + ).iterator( + _prepare_batch_segment_job_input.next(_create_batch_segment_job) + ), + ) + + @property + def start_state(self) -> State: + return self.create_batch_segment_jobs.start_state + + @property + def end_states(self) -> List[INextable]: + return [ + self.create_batch_segment_jobs.start_state, + self.batch_segment_jobs_not_available, + ] diff --git a/source/infrastructure/personalize/step_functions/dataset_import_fragment.py b/source/infrastructure/personalize/step_functions/dataset_import_fragment.py index ca00627..3605e56 100644 --- a/source/infrastructure/personalize/step_functions/dataset_import_fragment.py +++ b/source/infrastructure/personalize/step_functions/dataset_import_fragment.py @@ -12,6 +12,7 @@ # ###################################################################################################################### from typing import List +from aws_cdk import Duration from aws_cdk.aws_stepfunctions import ( StateMachineFragment, State, @@ -22,7 +23,7 @@ JsonPath, Pass, ) -from aws_cdk.core import Construct, Duration +from constructs import Construct from personalize.aws_lambda.functions import ( CreateDataset, diff --git a/source/infrastructure/personalize/step_functions/dataset_imports_fragment.py b/source/infrastructure/personalize/step_functions/dataset_imports_fragment.py index 180aff2..5cc3aeb 100644 --- a/source/infrastructure/personalize/step_functions/dataset_imports_fragment.py +++ b/source/infrastructure/personalize/step_functions/dataset_imports_fragment.py @@ -8,7 +8,7 @@ State, INextable, ) -from aws_cdk.core import Construct +from constructs import Construct from personalize.aws_lambda.functions import ( CreateSchema, diff --git a/source/infrastructure/personalize/step_functions/event_tracker_fragment.py b/source/infrastructure/personalize/step_functions/event_tracker_fragment.py index d359778..0970b58 100644 --- a/source/infrastructure/personalize/step_functions/event_tracker_fragment.py +++ b/source/infrastructure/personalize/step_functions/event_tracker_fragment.py @@ -12,6 +12,7 @@ # ###################################################################################################################### from typing import List +from aws_cdk import Duration from aws_cdk.aws_stepfunctions import ( StateMachineFragment, State, @@ -20,7 +21,7 @@ Pass, Condition, ) -from aws_cdk.core import Construct, Duration +from constructs import Construct from personalize.aws_lambda.functions import ( CreateEventTracker, diff --git a/source/infrastructure/personalize/step_functions/failure_fragment.py b/source/infrastructure/personalize/step_functions/failure_fragment.py index 532a159..7719fbe 100644 --- a/source/infrastructure/personalize/step_functions/failure_fragment.py +++ b/source/infrastructure/personalize/step_functions/failure_fragment.py @@ -19,7 +19,7 @@ Fail, TaskInput, ) -from aws_cdk.core import Construct +from constructs import Construct from personalize.sns.notifications import Notifications diff --git a/source/infrastructure/personalize/step_functions/filter_fragment.py b/source/infrastructure/personalize/step_functions/filter_fragment.py index ea4b0d2..803c467 100644 --- a/source/infrastructure/personalize/step_functions/filter_fragment.py +++ b/source/infrastructure/personalize/step_functions/filter_fragment.py @@ -12,6 +12,7 @@ # ###################################################################################################################### from typing import List +from aws_cdk import Duration from aws_cdk.aws_stepfunctions import ( StateMachineFragment, State, @@ -22,7 +23,7 @@ Map, JsonPath, ) -from aws_cdk.core import Construct, Duration +from constructs import Construct from personalize.aws_lambda.functions import ( CreateFilter, diff --git a/source/infrastructure/personalize/step_functions/personalization_fragment.py b/source/infrastructure/personalize/step_functions/personalization_fragment.py index 4fecd16..4614432 100644 --- a/source/infrastructure/personalize/step_functions/personalization_fragment.py +++ b/source/infrastructure/personalize/step_functions/personalization_fragment.py @@ -13,10 +13,11 @@ from typing import List, Dict from typing import Optional +from aws_cdk import Duration from aws_cdk.aws_lambda import CfnFunction from aws_cdk.aws_stepfunctions import State, INextable, TaskInput, StateMachineFragment from aws_cdk.aws_stepfunctions_tasks import LambdaInvoke -from aws_cdk.core import Construct, Duration +from constructs import Construct class PersonalizationFragment(StateMachineFragment): diff --git a/source/infrastructure/personalize/step_functions/scheduled_dataset_import.py b/source/infrastructure/personalize/step_functions/scheduled_dataset_import.py index 4438aaf..dadd7d2 100644 --- a/source/infrastructure/personalize/step_functions/scheduled_dataset_import.py +++ b/source/infrastructure/personalize/step_functions/scheduled_dataset_import.py @@ -13,7 +13,7 @@ from typing import Dict from aws_cdk.aws_stepfunctions import StateMachine, Chain, Parallel, TaskInput -from aws_cdk.core import Construct +from constructs import Construct from aws_solutions.cdk.aws_lambda.cfn_custom_resources.resource_name import ResourceName from aws_solutions.cdk.cfn_nag import add_cfn_nag_suppressions, CfnNagSuppression diff --git a/source/infrastructure/personalize/step_functions/scheduled_solution_maintenance.py b/source/infrastructure/personalize/step_functions/scheduled_solution_maintenance.py index b97ee78..ea90f40 100644 --- a/source/infrastructure/personalize/step_functions/scheduled_solution_maintenance.py +++ b/source/infrastructure/personalize/step_functions/scheduled_solution_maintenance.py @@ -12,7 +12,7 @@ # ###################################################################################################################### from aws_cdk.aws_stepfunctions import StateMachine, Chain, Parallel, TaskInput -from aws_cdk.core import Construct +from constructs import Construct from aws_solutions.cdk.aws_lambda.cfn_custom_resources.resource_name import ResourceName from aws_solutions.cdk.cfn_nag import add_cfn_nag_suppressions, CfnNagSuppression @@ -22,6 +22,8 @@ CreateSolution, CreateSolutionVersion, CreateCampaign, + CreateBatchSegmentJob, + CreateRecommender, ) from personalize.aws_lambda.functions.prepare_input import PrepareInput from personalize.step_functions.failure_fragment import FailureFragment @@ -37,9 +39,11 @@ def __init__( create_solution_version: CreateSolutionVersion, create_campaign: CreateCampaign, create_batch_inference_job: CreateBatchInferenceJob, + create_batch_segment_job: CreateBatchSegmentJob, prepare_input: PrepareInput, create_timestamp: SolutionStep, notifications: SolutionStep, + create_recommender: CreateRecommender, ): super().__init__(scope, construct_id) @@ -68,6 +72,8 @@ def __init__( create_solution_version=create_solution_version, create_campaign=create_campaign, create_batch_inference_job=create_batch_inference_job, + create_batch_segment_job=create_batch_segment_job, + create_recommender=create_recommender, ) ) ) diff --git a/source/infrastructure/personalize/step_functions/scheduler_fragment.py b/source/infrastructure/personalize/step_functions/scheduler_fragment.py index 28d860f..9d2d75e 100644 --- a/source/infrastructure/personalize/step_functions/scheduler_fragment.py +++ b/source/infrastructure/personalize/step_functions/scheduler_fragment.py @@ -24,7 +24,7 @@ Condition, JsonPath, ) -from aws_cdk.core import Construct +from constructs import Construct from aws_solutions.scheduler.cdk.construct import Scheduler diff --git a/source/infrastructure/personalize/step_functions/solution_fragment.py b/source/infrastructure/personalize/step_functions/solution_fragment.py index 0396000..d130c4c 100644 --- a/source/infrastructure/personalize/step_functions/solution_fragment.py +++ b/source/infrastructure/personalize/step_functions/solution_fragment.py @@ -12,6 +12,7 @@ # ###################################################################################################################### from typing import List, Optional +from aws_cdk import Duration from aws_cdk.aws_stepfunctions import ( StateMachineFragment, State, @@ -24,7 +25,7 @@ Parallel, StateMachine, ) -from aws_cdk.core import Construct, Duration +from constructs import Construct from aws_solutions.scheduler.cdk.construct import Scheduler from personalize.aws_lambda.functions import ( @@ -32,14 +33,18 @@ CreateSolutionVersion, CreateCampaign, CreateBatchInferenceJob, + CreateBatchSegmentJob, + CreateRecommender, ) from personalize.step_functions.batch_inference_jobs_fragment import ( BatchInferenceJobsFragment, ) +from personalize.step_functions.batch_segment_jobs_fragment import ( + BatchSegmentJobsFragment, +) from personalize.step_functions.scheduler_fragment import SchedulerFragment TEMPORARY_PATH = "$._tmp" -BATCH_INFERENCE_JOB_PATH = "$.batchInferenceJob" BUCKET_PATH = "$.bucket" CURRENT_DATE_PATH = "$.currentDate" MINIMUM_TIME = "1 second" @@ -54,6 +59,8 @@ def __init__( create_solution_version: CreateSolutionVersion, create_campaign: CreateCampaign, create_batch_inference_job: CreateBatchInferenceJob, + create_batch_segment_job: CreateBatchSegmentJob, + create_recommender: CreateRecommender, scheduler: Optional[Scheduler] = None, to_schedule: Optional[StateMachine] = None, ): @@ -69,7 +76,9 @@ def __init__( } # fmt: off + self.recommenders_not_available = Pass(self, "Recommenders not Provided") self.solutions_not_available = Pass(self, "Solutions not Provided") + self.recommenders_available = Choice(self, "Check for Recommenders").otherwise(self.recommenders_not_available) self.solutions_available = Choice(self, "Check for Solutions").otherwise(self.solutions_not_available) campaigns_available = Choice(self, "Check for Campaigns").otherwise(Pass(self, "Campaigns not Provided")) @@ -79,6 +88,12 @@ def __init__( input_path="$.datasetGroupArn", # NOSONAR (python:S1192) - string for clarity result_path="$.solution.serviceConfig.datasetGroupArn", ) + _prepare_recommender_input = Pass( + self, + "Prepare Recommender Input Data", + input_path="$.datasetGroupArn", # NOSONAR (python:S1192) - string for clarity + result_path="$.recommender.serviceConfig.datasetGroupArn" + ) _prepare_solution_output = Pass( self, @@ -126,6 +141,16 @@ def __init__( "solutionArn.$": "$.solutionArn" } ) + _create_recommender = create_recommender.state( + self, + "Create Recommender", + result_path=TEMPORARY_PATH, + input_path="$.recommender", + result_selector={ + "recommenderArn.$": "$.recommenderArn" + }, + **retry_config + ) _create_solution_version = create_solution_version.state( self, @@ -183,6 +208,14 @@ def __init__( to_schedule=to_schedule, ).start_state + _create_batch_segment_jobs = BatchSegmentJobsFragment( + self, + "Create Batch Segment Jobs", + create_batch_segment_job=create_batch_segment_job, + scheduler=scheduler, + to_schedule=to_schedule, + ).start_state + self.create_campaigns = campaigns_available.when( Condition.is_present("$.solution.campaigns[0]"), Map( @@ -204,6 +237,7 @@ def __init__( ) campaigns_and_batch.branch(self.create_campaigns) campaigns_and_batch.branch(_create_batch_inference_jobs) + campaigns_and_batch.branch(_create_batch_segment_jobs) if scheduler and to_schedule: campaigns_and_batch.next( SchedulerFragment( @@ -285,6 +319,21 @@ def __init__( ) _check_solution_version.otherwise(_create_solution_version) + self._create_recommenders = Map( + self, + "Create Recommenders", + items_path="$.recommenders", + result_path=JsonPath.DISCARD, + parameters={ + "datasetGroupArn.$": "$.datasetGroup.serviceConfig.datasetGroupArn", + "datasetGroupName.$": "$.datasetGroup.serviceConfig.name", + "recommender.$": "$$.Map.Item.Value", + "bucket.$": BUCKET_PATH, + "currentDate.$": CURRENT_DATE_PATH, # NOSONAR (python:S1192) - string for clarity + } + ).iterator(_prepare_recommender_input + .next(_create_recommender) + ) self._create_solutions = Map( self, "Create Solutions", @@ -301,15 +350,22 @@ def __init__( .next(_create_solution) .next(_prepare_solution_output) .next(_check_solution_version) + ) + + self.recommenders_and_solutions = Parallel(self, "Recommenders and Solutions", result_path=JsonPath.DISCARD).branch( + self.solutions_available + ).branch( + self.recommenders_available ) self.solutions_available.when(Condition.is_present("$.solutions[0]"), self._create_solutions) + self.recommenders_available.when(Condition.is_present("$.recommenders[0]"), self._create_recommenders) # fmt: on @property def start_state(self) -> State: - return self.solutions_available + return self.recommenders_and_solutions @property def end_states(self) -> List[INextable]: - return [self._create_solutions, self.solutions_not_available] + return [self.recommenders_and_solutions] diff --git a/source/infrastructure/setup.py b/source/infrastructure/setup.py index fc3dea2..0643144 100644 --- a/source/infrastructure/setup.py +++ b/source/infrastructure/setup.py @@ -35,7 +35,8 @@ author="AWS Solutions Builders", packages=setuptools.find_packages(), install_requires=[ - "aws-cdk.core>=1.126.0", + "aws-cdk-lib>=2.7.0", + "pip>=21.3", ], python_requires=">=3.7", classifiers=[ diff --git a/source/requirements-dev.txt b/source/requirements-dev.txt index c81dec0..e2625cd 100644 --- a/source/requirements-dev.txt +++ b/source/requirements-dev.txt @@ -1,25 +1,24 @@ avro==1.10.2 black boto3 -aws_cdk.core==1.126.0 -aws_cdk.aws_stepfunctions_tasks==1.126.0 -aws_solutions_constructs.aws_lambda_sns==1.126.0 -requests==2.24.0 -crhelper==2.0.6 +aws_cdk_lib==2.7.0 +aws_solutions_constructs.aws_lambda_sns==2.0.0 +requests~=2.27.1 +crhelper~=2.0.10 cronex==0.1.3.1 -moto==2.0.8 +moto==2.3.0 parsedatetime==2.6 pytest pytest-cov>=2.11.1 pytest-env>=0.6.2 pytest-mock>=3.5.1 pyyaml==5.4.1 -responses==0.14.0 +responses~=0.17.0 tenacity>=8.0.1 -e cdk_solution_helper_py/helpers_cdk -e cdk_solution_helper_py/helpers_common -e scheduler/cdk -e scheduler/common -aws-lambda-powertools>=1.15.0 +aws-lambda-powertools==1.24.0 docker==5.0.0 -e infrastructure \ No newline at end of file diff --git a/source/scheduler/CHANGELOG.md b/source/scheduler/CHANGELOG.md index 0bc7be1..0458505 100644 --- a/source/scheduler/CHANGELOG.md +++ b/source/scheduler/CHANGELOG.md @@ -4,6 +4,10 @@ All notable changes to this project will be documented in this file. The format is based on [Keep a Changelog](https://keepachangelog.com/en/1.0.0/), and this project adheres to [Semantic Versioning](https://semver.org/spec/v2.0.0.html). +## [2.0.0] - 2022-01-31 +### Changed +- support for CDK 2.x added, support for CDK 1.x removed + ## [1.1.0] - 2021-11-11 ### Added - initial release diff --git a/source/scheduler/README.md b/source/scheduler/README.md index 6a0e43d..3c9f80c 100644 --- a/source/scheduler/README.md +++ b/source/scheduler/README.md @@ -10,7 +10,7 @@ This README summarizes using the scheduler. Install this package. It requires at least: - Python 3.7 -- AWS CDK version 1.126.0 or higher +- AWS CDK version 2.7.0 or higher To install the packages: @@ -24,7 +24,7 @@ pip install /scheduler/common # where is the path to the scheduler ```python from pathlib import Path -from aws_cdk.core import Construct +from constructs import Construct from aws_cdk.aws_stepfunctions import StateMachine from aws_solutions.cdk import CDKSolution diff --git a/source/scheduler/cdk/aws_solutions/scheduler/cdk/aws_lambda/create_scheduled_task.py b/source/scheduler/cdk/aws_solutions/scheduler/cdk/aws_lambda/create_scheduled_task.py index 90a8da5..3a7533c 100644 --- a/source/scheduler/cdk/aws_solutions/scheduler/cdk/aws_lambda/create_scheduled_task.py +++ b/source/scheduler/cdk/aws_solutions/scheduler/cdk/aws_lambda/create_scheduled_task.py @@ -17,7 +17,7 @@ import aws_cdk.aws_iam as iam from aws_cdk.aws_dynamodb import ITable from aws_cdk.aws_stepfunctions import IChainable -from aws_cdk.core import Construct +from constructs import Construct from aws_solutions.cdk.stepfunctions.solutionstep import SolutionStep diff --git a/source/scheduler/cdk/aws_solutions/scheduler/cdk/aws_lambda/delete_scheduled_task.py b/source/scheduler/cdk/aws_solutions/scheduler/cdk/aws_lambda/delete_scheduled_task.py index cd4ab6c..b211ade 100644 --- a/source/scheduler/cdk/aws_solutions/scheduler/cdk/aws_lambda/delete_scheduled_task.py +++ b/source/scheduler/cdk/aws_solutions/scheduler/cdk/aws_lambda/delete_scheduled_task.py @@ -17,7 +17,7 @@ import aws_cdk.aws_iam as iam from aws_cdk.aws_dynamodb import ITable from aws_cdk.aws_stepfunctions import IChainable -from aws_cdk.core import Construct +from constructs import Construct from aws_solutions.cdk.stepfunctions.solutionstep import SolutionStep diff --git a/source/scheduler/cdk/aws_solutions/scheduler/cdk/aws_lambda/read_scheduled_task.py b/source/scheduler/cdk/aws_solutions/scheduler/cdk/aws_lambda/read_scheduled_task.py index b4c0222..1e73deb 100644 --- a/source/scheduler/cdk/aws_solutions/scheduler/cdk/aws_lambda/read_scheduled_task.py +++ b/source/scheduler/cdk/aws_solutions/scheduler/cdk/aws_lambda/read_scheduled_task.py @@ -17,7 +17,7 @@ import aws_cdk.aws_iam as iam from aws_cdk.aws_dynamodb import ITable from aws_cdk.aws_stepfunctions import IChainable -from aws_cdk.core import Construct +from constructs import Construct from aws_solutions.cdk.stepfunctions.solutionstep import SolutionStep diff --git a/source/scheduler/cdk/aws_solutions/scheduler/cdk/aws_lambda/update_scheduled_task.py b/source/scheduler/cdk/aws_solutions/scheduler/cdk/aws_lambda/update_scheduled_task.py index 22ab0c3..f71851d 100644 --- a/source/scheduler/cdk/aws_solutions/scheduler/cdk/aws_lambda/update_scheduled_task.py +++ b/source/scheduler/cdk/aws_solutions/scheduler/cdk/aws_lambda/update_scheduled_task.py @@ -17,7 +17,7 @@ import aws_cdk.aws_iam as iam from aws_cdk.aws_dynamodb import ITable from aws_cdk.aws_stepfunctions import IChainable -from aws_cdk.core import Construct +from constructs import Construct from aws_solutions.cdk.stepfunctions.solutionstep import SolutionStep diff --git a/source/scheduler/cdk/aws_solutions/scheduler/cdk/construct.py b/source/scheduler/cdk/aws_solutions/scheduler/cdk/construct.py index ba0586e..0787934 100644 --- a/source/scheduler/cdk/aws_solutions/scheduler/cdk/construct.py +++ b/source/scheduler/cdk/aws_solutions/scheduler/cdk/construct.py @@ -10,11 +10,16 @@ # on an "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. See the License for # # the specific language governing permissions and limitations under the License. # # ###################################################################################################################### + +from __future__ import annotations + from pathlib import Path from typing import Union, Dict, List import aws_cdk.aws_dynamodb as ddb import aws_cdk.aws_iam as iam +import jsii +from aws_cdk import Aws, IAspect, Aspects from aws_cdk.aws_lambda import Tracing from aws_cdk.aws_stepfunctions import ( StateMachine, @@ -28,7 +33,7 @@ Condition, ) from aws_cdk.aws_stepfunctions_tasks import LambdaInvoke -from aws_cdk.core import Construct, Aws +from constructs import Construct, IConstruct from aws_solutions.cdk.aws_lambda.cfn_custom_resources.resource_name import ResourceName from aws_solutions.cdk.aws_lambda.environment import Environment @@ -47,6 +52,64 @@ SCHEDULE_PATH = "$.task.schedule" +@jsii.implements(IAspect) +class SchedulerPermissionsAspect: + def __init__(self, scheduler: Scheduler): + self.scheduler = scheduler + + def visit(self, node: IConstruct): + if node == self.scheduler: + # permision: allow the scheduler to call itself + self.scheduler.state_machine.add_to_role_policy( + iam.PolicyStatement( + effect=iam.Effect.ALLOW, + resources=[self.scheduler.state_machine_arn], + actions=["states:StartExecution"], + ) + ) + if self.scheduler.sync: + self.scheduler.state_machine.add_to_role_policy( + iam.PolicyStatement( + effect=iam.Effect.ALLOW, + resources=["*"], + actions=["states:DescribeExecution", "states:StopExecution"], + ) + ) + self.scheduler.state_machine.add_to_role_policy( + iam.PolicyStatement( + effect=iam.Effect.ALLOW, + resources=[ + f"arn:{Aws.PARTITION}:events:{Aws.REGION}:{Aws.ACCOUNT_ID}:rule/StepFunctionsGetEventsForStepFunctionsExecutionRule" + ], + actions=[ + "events:PutTargets", + "events:PutRule", + "events:DescribeRule", + ], + ) + ) + + add_cfn_nag_suppressions( + self.scheduler.state_machine.role.node.try_find_child( + "DefaultPolicy" + ).node.find_child("Resource"), + [ + CfnNagSuppression( + "W12", + "IAM policy for nested synchronous invocation of step functions requires * on Describe and Stop Execution", + ), + CfnNagSuppression( + "W76", + "Large step functions need larger IAM roles to access all managed AWS Lambda functions", + ), + ], + ) + + # permission: allow the scheduler to call its referenced children + for child in self.scheduler._scheduler_child_state_machines: + child.grant_start_execution(self.scheduler.state_machine) + + class Scheduler(Construct): """ A Scheduler that leverages AWS Step Functions to invoke other AWS Step Functions on a specified cron() schedule. @@ -182,6 +245,9 @@ def __init__(self, scope: Construct, construct_id: str, sync: bool = True): tracing_enabled=True, ) + # permissions are applied at synthesis time based on the configuration of the scheduler + Aspects.of(self).add(SchedulerPermissionsAspect(self)) + def grant_invoke(self, state_machine: IStateMachine) -> None: """ Allow the Scheduler to start executions of the provided state machine @@ -279,61 +345,6 @@ def state_machine_name(self) -> str: """ return self._state_machine_namer.resource_name.to_string() - def _prepare(self) -> None: - """ - Finalize/ prepare the state machine and associated permissions - :return: None - """ - # permision: allow the scheduler to call itself - self.state_machine.add_to_role_policy( - iam.PolicyStatement( - effect=iam.Effect.ALLOW, - resources=[self.state_machine_arn], - actions=["states:StartExecution"], - ) - ) - if self.sync: - self.state_machine.add_to_role_policy( - iam.PolicyStatement( - effect=iam.Effect.ALLOW, - resources=["*"], - actions=["states:DescribeExecution", "states:StopExecution"], - ) - ) - self.state_machine.add_to_role_policy( - iam.PolicyStatement( - effect=iam.Effect.ALLOW, - resources=[ - f"arn:{Aws.PARTITION}:events:{Aws.REGION}:{Aws.ACCOUNT_ID}:rule/StepFunctionsGetEventsForStepFunctionsExecutionRule" - ], - actions=[ - "events:PutTargets", - "events:PutRule", - "events:DescribeRule", - ], - ) - ) - - add_cfn_nag_suppressions( - self.state_machine.role.node.try_find_child( - "DefaultPolicy" - ).node.find_child("Resource"), - [ - CfnNagSuppression( - "W12", - "IAM policy for nested synchronous invocation of step functions requires * on Describe and Stop Execution", - ), - CfnNagSuppression( - "W76", - "Large step functions need larger IAM roles to access all managed AWS Lambda functions", - ), - ], - ) - - # permission: allow the scheduler to call its referenced children - for child in self._scheduler_child_state_machines: - child.grant_start_execution(self.state_machine) - def _scheduler_table(self, scope: Construct) -> ddb.Table: """ Creates the table for tracking scheduled tasks managed by this Scheduler diff --git a/source/scheduler/cdk/aws_solutions/scheduler/cdk/scheduler_fragment.py b/source/scheduler/cdk/aws_solutions/scheduler/cdk/scheduler_fragment.py index 28d860f..9d2d75e 100644 --- a/source/scheduler/cdk/aws_solutions/scheduler/cdk/scheduler_fragment.py +++ b/source/scheduler/cdk/aws_solutions/scheduler/cdk/scheduler_fragment.py @@ -24,7 +24,7 @@ Condition, JsonPath, ) -from aws_cdk.core import Construct +from constructs import Construct from aws_solutions.scheduler.cdk.construct import Scheduler diff --git a/source/scheduler/cdk/setup.py b/source/scheduler/cdk/setup.py index 36b6121..49c8a3f 100644 --- a/source/scheduler/cdk/setup.py +++ b/source/scheduler/cdk/setup.py @@ -42,9 +42,8 @@ def get_version(): license="Apache License 2.0", packages=setuptools.find_namespace_packages(), install_requires=[ - "aws-cdk.core>=1.126.0", - "aws-cdk.aws_lambda>=1.126.0", - "aws-cdk.aws_stepfunctions>=1.126.0", + "pip>=21.3", + "aws_cdk_lib>=2.7.0", "Click>=7.1.2", "boto3>=1.17.52", ], diff --git a/source/scheduler/common/aws_solutions/scheduler/common/scripts/scheduler_cli.py b/source/scheduler/common/aws_solutions/scheduler/common/scripts/scheduler_cli.py index 4c9d60b..3de463f 100644 --- a/source/scheduler/common/aws_solutions/scheduler/common/scripts/scheduler_cli.py +++ b/source/scheduler/common/aws_solutions/scheduler/common/scripts/scheduler_cli.py @@ -50,6 +50,21 @@ def get_stack_tag_value(stack, key: str) -> str: return results[0]["Value"] +def get_stack_metadata_value(stack, key: str) -> str: + """ + Get a stack template metadata value + :param stack: the boto3 stack resource + :param key: the metadata key + :return: str + """ + summary = stack.meta.client.get_template_summary(StackName=stack.name) + metadata = json.loads(summary.get("Metadata", "{}")) + try: + return metadata[key] + except KeyError: + raise ValueError(f"could not find metadata with key {key} in stack") + + def setup_cli_env(stack, region: str) -> None: """ Set the environment variables required by the scheduler @@ -58,8 +73,12 @@ def setup_cli_env(stack, region: str) -> None: :return: None """ os.environ["AWS_REGION"] = region - os.environ["SOLUTION_ID"] = get_stack_tag_value(stack, "SOLUTION_ID") - os.environ["SOLUTION_VERSION"] = f"{get_stack_tag_value(stack, 'SOLUTION_VERSION')}" + os.environ["SOLUTION_ID"] = get_stack_metadata_value( + stack, "aws:solutions:solution_id" + ) + os.environ["SOLUTION_VERSION"] = get_stack_metadata_value( + stack, "aws:solutions:solution_version" + ) @click.group() @@ -90,7 +109,10 @@ def cli( cloudformation = boto3.resource("cloudformation", region_name=region) stack = cloudformation.Stack(stack) - setup_cli_env(stack, region) + try: + setup_cli_env(stack, region) + except ValueError as exc: + raise click.ClickException(exc) ctx.obj["SCHEDULER"] = Scheduler( table_name=get_stack_output_value(stack, scheduler_table_name_output), diff --git a/source/scheduler/common/setup.py b/source/scheduler/common/setup.py index eccc3b3..b9860d6 100644 --- a/source/scheduler/common/setup.py +++ b/source/scheduler/common/setup.py @@ -42,9 +42,8 @@ def get_version(): license="Apache License 2.0", packages=setuptools.find_namespace_packages(), install_requires=[ - "aws-cdk.core>=1.126.0", - "aws-cdk.aws_lambda>=1.126.0", - "aws-lambda-powertools>=1.21.1", + "pip>=21.3", + "aws-lambda-powertools>=1.24.0", "aws-solutions-python>=1.0.0", "Click>=7.1.2", "cronex==0.1.3.1", diff --git a/source/tests/aws_lambda/create_batch_inference_job/test_batch_inference_job_handler.py b/source/tests/aws_lambda/create_batch_inference_job/test_batch_inference_job_handler.py index f587496..1e3fff4 100644 --- a/source/tests/aws_lambda/create_batch_inference_job/test_batch_inference_job_handler.py +++ b/source/tests/aws_lambda/create_batch_inference_job/test_batch_inference_job_handler.py @@ -13,9 +13,15 @@ import pytest -from aws_lambda.create_batch_inference_job.handler import lambda_handler +from aws_lambda.create_batch_inference_job.handler import ( + lambda_handler, + RESOURCE, + STATUS, + CONFIG, +) -def test_create_batch_inference_job_handler(): +def test_create_batch_inference_job_handler(validate_handler_config): + validate_handler_config(RESOURCE, CONFIG, STATUS) with pytest.raises(ValueError): lambda_handler({}, None) diff --git a/source/tests/aws_lambda/create_batch_segment_job/test_batch_segment_job_handler.py b/source/tests/aws_lambda/create_batch_segment_job/test_batch_segment_job_handler.py new file mode 100644 index 0000000..429ac0c --- /dev/null +++ b/source/tests/aws_lambda/create_batch_segment_job/test_batch_segment_job_handler.py @@ -0,0 +1,27 @@ +# ###################################################################################################################### +# Copyright Amazon.com, Inc. or its affiliates. All Rights Reserved. # +# # +# Licensed under the Apache License, Version 2.0 (the "License"). You may not use this file except in compliance # +# with the License. You may obtain a copy of the License at # +# # +# http://www.apache.org/licenses/LICENSE-2.0 # +# # +# Unless required by applicable law or agreed to in writing, software distributed under the License is distributed # +# on an "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. See the License for # +# the specific language governing permissions and limitations under the License. # +# ###################################################################################################################### + +import pytest + +from aws_lambda.create_batch_segment_job.handler import ( + lambda_handler, + RESOURCE, + STATUS, + CONFIG, +) + + +def test_create_batch_segment_job_handler(validate_handler_config): + validate_handler_config(RESOURCE, CONFIG, STATUS) + with pytest.raises(ValueError): + lambda_handler({}, None) diff --git a/source/tests/aws_lambda/create_campaign/test_create_campaign_handler.py b/source/tests/aws_lambda/create_campaign/test_create_campaign_handler.py index 3f8d181..2115823 100644 --- a/source/tests/aws_lambda/create_campaign/test_create_campaign_handler.py +++ b/source/tests/aws_lambda/create_campaign/test_create_campaign_handler.py @@ -18,12 +18,20 @@ from dateutil.tz import tzlocal from moto import mock_sts -from aws_lambda.create_campaign.handler import lambda_handler +from aws_lambda.create_campaign.handler import ( + lambda_handler, + RESOURCE, + STATUS, + CONFIG, +) from shared.exceptions import ResourcePending from shared.resource import Campaign, SolutionVersion -def test_create_campaign(): +def test_create_campaign(validate_handler_config): + for status in STATUS.split("||"): + status = status.strip() + validate_handler_config(RESOURCE, CONFIG, status) with pytest.raises(ValueError): lambda_handler({}, None) diff --git a/source/tests/aws_lambda/create_config/test_create_config_handler.py b/source/tests/aws_lambda/create_config/test_create_config_handler.py index cf6cce7..b1bc892 100644 --- a/source/tests/aws_lambda/create_config/test_create_config_handler.py +++ b/source/tests/aws_lambda/create_config/test_create_config_handler.py @@ -21,6 +21,7 @@ BatchInferenceJob, EventTracker, Schema, + BatchSegmentJob, ) @@ -84,6 +85,19 @@ def test_create_config(personalize_stubber): ] }, ) + personalize_stubber.add_response( + method="list_batch_segment_jobs", + service_response={ + "batchSegmentJobs": [ + {"batchSegmentJobArn": BatchSegmentJob().arn("dsgbatch")} + ] + }, + ) + personalize_stubber.add_response( + method="list_recommenders", + expected_params={"datasetGroupArn": dsg_arn}, + service_response={"recommenders": []}, + ) personalize_stubber.add_response( method="list_event_trackers", service_response={"eventTrackers": [{"eventTrackerArn": event_tracker_arn}]}, diff --git a/source/tests/aws_lambda/create_dataset/test_dataset_handler.py b/source/tests/aws_lambda/create_dataset/test_dataset_handler.py index 9b6b2aa..80ac0d6 100644 --- a/source/tests/aws_lambda/create_dataset/test_dataset_handler.py +++ b/source/tests/aws_lambda/create_dataset/test_dataset_handler.py @@ -13,9 +13,14 @@ import pytest -from aws_lambda.create_dataset.handler import lambda_handler +from aws_lambda.create_dataset.handler import ( + lambda_handler, + RESOURCE, + CONFIG, +) -def test_create_dataset_handler(): +def test_create_dataset_handler(validate_handler_config): + validate_handler_config(RESOURCE, CONFIG) with pytest.raises(ValueError): lambda_handler({}, None) diff --git a/source/tests/aws_lambda/create_dataset_group/test_dataset_group_handler.py b/source/tests/aws_lambda/create_dataset_group/test_dataset_group_handler.py index 21abaf3..caedbd7 100644 --- a/source/tests/aws_lambda/create_dataset_group/test_dataset_group_handler.py +++ b/source/tests/aws_lambda/create_dataset_group/test_dataset_group_handler.py @@ -13,9 +13,15 @@ import pytest -from aws_lambda.create_dataset_group.handler import lambda_handler +from aws_lambda.create_dataset_group.handler import ( + lambda_handler, + RESOURCE, + STATUS, + CONFIG, +) -def test_handler(): +def test_handler(validate_handler_config): + validate_handler_config(RESOURCE, CONFIG, STATUS) with pytest.raises(ValueError): lambda_handler({}, None) diff --git a/source/tests/aws_lambda/create_dataset_import_job/test_dataset_import_job_handler.py b/source/tests/aws_lambda/create_dataset_import_job/test_dataset_import_job_handler.py index 6e82eb4..30e483e 100644 --- a/source/tests/aws_lambda/create_dataset_import_job/test_dataset_import_job_handler.py +++ b/source/tests/aws_lambda/create_dataset_import_job/test_dataset_import_job_handler.py @@ -13,9 +13,15 @@ import pytest -from aws_lambda.create_dataset_import_job.handler import lambda_handler +from aws_lambda.create_dataset_import_job.handler import ( + lambda_handler, + RESOURCE, + STATUS, + CONFIG, +) -def test_create_dataset_import_job_handler(): +def test_create_dataset_import_job_handler(validate_handler_config): + validate_handler_config(RESOURCE, CONFIG, STATUS) with pytest.raises(ValueError): lambda_handler({}, None) diff --git a/source/tests/aws_lambda/create_event_tracker/test_create_event_tracker_handler.py b/source/tests/aws_lambda/create_event_tracker/test_create_event_tracker_handler.py index 4c8f7ac..449ceea 100644 --- a/source/tests/aws_lambda/create_event_tracker/test_create_event_tracker_handler.py +++ b/source/tests/aws_lambda/create_event_tracker/test_create_event_tracker_handler.py @@ -13,9 +13,15 @@ import pytest -from aws_lambda.create_event_tracker.handler import lambda_handler +from aws_lambda.create_event_tracker.handler import ( + lambda_handler, + RESOURCE, + STATUS, + CONFIG, +) -def test_create_event_tracker(): +def test_create_event_tracker(validate_handler_config): + validate_handler_config(RESOURCE, CONFIG, STATUS) with pytest.raises(ValueError): lambda_handler({}, None) diff --git a/source/tests/aws_lambda/create_filter/test_create_filter_handler.py b/source/tests/aws_lambda/create_filter/test_create_filter_handler.py index a5c093d..f605cdc 100644 --- a/source/tests/aws_lambda/create_filter/test_create_filter_handler.py +++ b/source/tests/aws_lambda/create_filter/test_create_filter_handler.py @@ -13,9 +13,15 @@ import pytest -from aws_lambda.create_filter.handler import lambda_handler +from aws_lambda.create_filter.handler import ( + lambda_handler, + RESOURCE, + STATUS, + CONFIG, +) -def test_create_filter(): +def test_create_filter(validate_handler_config): + validate_handler_config(RESOURCE, CONFIG, STATUS) with pytest.raises(ValueError): lambda_handler({}, None) diff --git a/source/tests/aws_lambda/create_recommender/test_create_recommender_handler.py b/source/tests/aws_lambda/create_recommender/test_create_recommender_handler.py new file mode 100644 index 0000000..4352784 --- /dev/null +++ b/source/tests/aws_lambda/create_recommender/test_create_recommender_handler.py @@ -0,0 +1,27 @@ +# ###################################################################################################################### +# Copyright Amazon.com, Inc. or its affiliates. All Rights Reserved. # +# # +# Licensed under the Apache License, Version 2.0 (the "License"). You may not use this file except in compliance # +# with the License. You may obtain a copy of the License at # +# # +# http://www.apache.org/licenses/LICENSE-2.0 # +# # +# Unless required by applicable law or agreed to in writing, software distributed under the License is distributed # +# on an "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. See the License for # +# the specific language governing permissions and limitations under the License. # +# ###################################################################################################################### + +import pytest + +from aws_lambda.create_recommender.handler import ( + lambda_handler, + RESOURCE, + STATUS, + CONFIG, +) + + +def test_create_recommender_handler(validate_handler_config): + validate_handler_config(RESOURCE, CONFIG, STATUS) + with pytest.raises(ValueError): + lambda_handler({}, None) diff --git a/source/tests/aws_lambda/create_schema/create_schema_handler.py b/source/tests/aws_lambda/create_schema/create_schema_handler.py index a1b957c..0d2184a 100644 --- a/source/tests/aws_lambda/create_schema/create_schema_handler.py +++ b/source/tests/aws_lambda/create_schema/create_schema_handler.py @@ -13,9 +13,14 @@ import pytest -from aws_lambda.create_schema.handler import lambda_handler +from aws_lambda.create_schema.handler import ( + lambda_handler, + RESOURCE, + CONFIG, +) -def test_create_schema_handler(): +def test_create_schema_handler(validate_handler_config): + validate_handler_config(RESOURCE, CONFIG) with pytest.raises(ValueError): lambda_handler({}, None) diff --git a/source/tests/aws_lambda/create_solution/test_create_solution_handler.py b/source/tests/aws_lambda/create_solution/test_create_solution_handler.py index 4d27ff1..e4b8b64 100644 --- a/source/tests/aws_lambda/create_solution/test_create_solution_handler.py +++ b/source/tests/aws_lambda/create_solution/test_create_solution_handler.py @@ -13,9 +13,15 @@ import pytest -from aws_lambda.create_solution.handler import lambda_handler +from aws_lambda.create_solution.handler import ( + lambda_handler, + RESOURCE, + STATUS, + CONFIG, +) -def test_create_solution(): +def test_create_solution(validate_handler_config): + validate_handler_config(RESOURCE, CONFIG, STATUS) with pytest.raises(ValueError): lambda_handler({}, None) diff --git a/source/tests/aws_lambda/create_solution_version/test_create_solution_version_handler.py b/source/tests/aws_lambda/create_solution_version/test_create_solution_version_handler.py index b9a1d7a..9e3c1d2 100644 --- a/source/tests/aws_lambda/create_solution_version/test_create_solution_version_handler.py +++ b/source/tests/aws_lambda/create_solution_version/test_create_solution_version_handler.py @@ -13,9 +13,15 @@ import pytest -from aws_lambda.create_solution_version.handler import lambda_handler +from aws_lambda.create_solution_version.handler import ( + lambda_handler, + RESOURCE, + STATUS, + CONFIG, +) -def test_create_solution_version_handler(): +def test_create_solution_version_handler(validate_handler_config): + validate_handler_config(RESOURCE, CONFIG, STATUS) with pytest.raises(ValueError): lambda_handler({}, None) diff --git a/source/tests/aws_lambda/test_personalize_service.py b/source/tests/aws_lambda/test_personalize_service.py index 32f0d46..eda04a3 100644 --- a/source/tests/aws_lambda/test_personalize_service.py +++ b/source/tests/aws_lambda/test_personalize_service.py @@ -201,6 +201,11 @@ def test_service_model(personalize_stubber): expected_params={"solutionArn": solution_arn_1}, service_response={"solutionVersions": []}, ) + personalize_stubber.add_response( + method="list_recommenders", + expected_params={"datasetGroupArn": dataset_group_arn_1}, + service_response={"recommenders": []}, + ) personalize_stubber.add_response( method="list_event_trackers", expected_params={"datasetGroupArn": dataset_group_arn_1}, @@ -238,6 +243,11 @@ def test_service_model(personalize_stubber): expected_params={"solutionArn": solution_arn_2}, service_response={"solutionVersions": []}, ) + personalize_stubber.add_response( + method="list_recommenders", + expected_params={"datasetGroupArn": dataset_group_arn_2}, + service_response={"recommenders": []}, + ) personalize_stubber.add_response( method="list_event_trackers", expected_params={"datasetGroupArn": dataset_group_arn_2}, diff --git a/source/tests/aws_lambda/test_sfn_middleware.py b/source/tests/aws_lambda/test_sfn_middleware.py index 17739ce..1e3a000 100644 --- a/source/tests/aws_lambda/test_sfn_middleware.py +++ b/source/tests/aws_lambda/test_sfn_middleware.py @@ -113,7 +113,7 @@ def test_set_defaults_1(): defaults = set_defaults({}) del defaults["currentDate"] del defaults["datasetGroup"] - assert defaults == {"solutions": []} + assert defaults == {"recommenders": [], "solutions": []} def test_set_defaults_2(): diff --git a/source/tests/cdk_solution_helper/aws_lambda/cfn_custom_resources/resource_hash/test_resource_name_cdk.py b/source/tests/cdk_solution_helper/aws_lambda/cfn_custom_resources/resource_hash/test_resource_name_cdk.py index 691ada7..8a9bae8 100644 --- a/source/tests/cdk_solution_helper/aws_lambda/cfn_custom_resources/resource_hash/test_resource_name_cdk.py +++ b/source/tests/cdk_solution_helper/aws_lambda/cfn_custom_resources/resource_hash/test_resource_name_cdk.py @@ -12,7 +12,7 @@ # ##################################################################################################################### import pytest -from aws_cdk.core import Stack, App +from aws_cdk import Stack, App from constructs import Construct from aws_solutions.cdk.aws_lambda.cfn_custom_resources.resource_name.name import ( @@ -31,7 +31,7 @@ def __init__(self, scope: Construct, construct_id: str, **kwargs) -> None: def resource_naming_stack(): app = App() SomeStack(app, "some-test-naming") - yield app.synth().get_stack("some-test-naming").template + yield app.synth().get_stack_by_name("some-test-naming").template def test_resource_service_tokens(resource_naming_stack): diff --git a/source/tests/cdk_solution_helper/aws_lambda/cfn_custom_resources/resource_name/test_resource_hash_cdk.py b/source/tests/cdk_solution_helper/aws_lambda/cfn_custom_resources/resource_name/test_resource_hash_cdk.py index 691ada7..8a9bae8 100644 --- a/source/tests/cdk_solution_helper/aws_lambda/cfn_custom_resources/resource_name/test_resource_hash_cdk.py +++ b/source/tests/cdk_solution_helper/aws_lambda/cfn_custom_resources/resource_name/test_resource_hash_cdk.py @@ -12,7 +12,7 @@ # ##################################################################################################################### import pytest -from aws_cdk.core import Stack, App +from aws_cdk import Stack, App from constructs import Construct from aws_solutions.cdk.aws_lambda.cfn_custom_resources.resource_name.name import ( @@ -31,7 +31,7 @@ def __init__(self, scope: Construct, construct_id: str, **kwargs) -> None: def resource_naming_stack(): app = App() SomeStack(app, "some-test-naming") - yield app.synth().get_stack("some-test-naming").template + yield app.synth().get_stack_by_name("some-test-naming").template def test_resource_service_tokens(resource_naming_stack): diff --git a/source/tests/cdk_solution_helper/aws_lambda/cfn_custom_resources/solution_metrics/test_metrics_cdk.py b/source/tests/cdk_solution_helper/aws_lambda/cfn_custom_resources/solution_metrics/test_metrics_cdk.py index 7c3a529..c797593 100644 --- a/source/tests/cdk_solution_helper/aws_lambda/cfn_custom_resources/solution_metrics/test_metrics_cdk.py +++ b/source/tests/cdk_solution_helper/aws_lambda/cfn_custom_resources/solution_metrics/test_metrics_cdk.py @@ -12,7 +12,7 @@ # ##################################################################################################################### import pytest -from aws_cdk.core import Stack, App +from aws_cdk import Stack, App from constructs import Construct from aws_solutions.cdk.aws_lambda.cfn_custom_resources.solutions_metrics.metrics import ( @@ -35,7 +35,7 @@ def __init__(self, scope: Construct, construct_id: str, **kwargs) -> None: def test_stack_metrics(): app = App() SomeStack(app, "some-test-metrics") - yield app.synth().get_stack("some-test-metrics").template + yield app.synth().get_stack_by_name("some-test-metrics").template def test_metrics_valid(test_stack_metrics): diff --git a/source/tests/cdk_solution_helper/aws_lambda/java/test_java_function.py b/source/tests/cdk_solution_helper/aws_lambda/java/test_java_function.py index 496770c..d2c64bf 100644 --- a/source/tests/cdk_solution_helper/aws_lambda/java/test_java_function.py +++ b/source/tests/cdk_solution_helper/aws_lambda/java/test_java_function.py @@ -14,11 +14,11 @@ from pathlib import Path import pytest -from aws_cdk.core import ( +from aws_cdk import ( Stack, - Construct, App, ) +from constructs import Construct from aws_solutions.cdk.aws_lambda.java.function import SolutionsJavaFunction @@ -53,7 +53,9 @@ def __init__(self, scope: Construct, construct_id: str, **kwargs) -> None: @pytest.mark.no_cdk_lambda_mock def test_java_function_synth(java_function_synth): - function_stack = java_function_synth.get_stack("test-function-lambda").template + function_stack = java_function_synth.get_stack_by_name( + "test-function-lambda" + ).template func = function_stack["Resources"]["TestFunction"] assert func["Type"] == "AWS::Lambda::Function" diff --git a/source/tests/cdk_solution_helper/aws_lambda/python/test_function.py b/source/tests/cdk_solution_helper/aws_lambda/python/test_function.py index 77386f7..bc27f00 100644 --- a/source/tests/cdk_solution_helper/aws_lambda/python/test_function.py +++ b/source/tests/cdk_solution_helper/aws_lambda/python/test_function.py @@ -15,7 +15,8 @@ from pathlib import Path import pytest -from aws_cdk.core import Construct, Stack, App +from aws_cdk import Stack, App +from constructs import Construct from aws_solutions.cdk.aws_lambda.python.function import ( SolutionsPythonFunction, @@ -72,7 +73,7 @@ def __init__(self, scope: Construct, construct_id: str, **kwargs) -> None: def test_function_has_default_role(function_synth): - function_stack = function_synth.get_stack("test-function").template + function_stack = function_synth.get_stack_by_name("test-function").template func = function_stack["Resources"]["TestFunction"] assert func["Type"] == "AWS::Lambda::Function" assert ( diff --git a/source/tests/cdk_solution_helper/aws_lambda/python/test_layer_version.py b/source/tests/cdk_solution_helper/aws_lambda/python/test_layer_version.py index efe55f0..7c5d9ed 100644 --- a/source/tests/cdk_solution_helper/aws_lambda/python/test_layer_version.py +++ b/source/tests/cdk_solution_helper/aws_lambda/python/test_layer_version.py @@ -16,7 +16,8 @@ from pathlib import Path import pytest -from aws_cdk.core import Construct, Stack, App +from aws_cdk import Stack, App +from constructs import Construct from aws_solutions.cdk.aws_lambda.python.layer import SolutionsPythonLayerVersion from aws_solutions.cdk.helpers.copytree import copytree @@ -62,16 +63,24 @@ def __init__(self, scope: Construct, construct_id: str, **kwargs) -> None: @pytest.mark.no_cdk_lambda_mock def test_layer_version(layer_synth): - layer_synth.get_stack("test-layer-version").template + layer_synth.get_stack_by_name("test-layer-version").template directory = Path(layer_synth.directory) manifest = json.loads((directory / "manifest.json").read_text(encoding="utf-8")) - asset_dir = ( - directory - / manifest["artifacts"]["test-layer-version"]["metadata"][ - "/test-layer-version" - ][0]["data"]["path"] - ) + asset_file = manifest["artifacts"]["test-layer-version.assets"]["properties"][ + "file" + ] + assets = json.loads((directory / asset_file).read_text(encoding="utf-8")) + asset_dir = next( + iter( + [ + v + for v in assets["files"].values() + if v.get("source", {}).get("packaging") == "zip" + ] + ) + )["source"]["path"] + asset_path = directory / asset_dir # check that the package was installed to the correct path - assert (asset_dir / "python" / "minimal").exists() + assert (asset_path / "python" / "minimal").exists() diff --git a/source/tests/cdk_solution_helper/helpers/test_load_cdk_app.py b/source/tests/cdk_solution_helper/helpers/test_load_cdk_app.py index 0376237..c4c9884 100644 --- a/source/tests/cdk_solution_helper/helpers/test_load_cdk_app.py +++ b/source/tests/cdk_solution_helper/helpers/test_load_cdk_app.py @@ -17,7 +17,8 @@ from aws_solutions.cdk.helpers.loader import load_cdk_app, CDKLoaderException CDK_APP = """ -from aws_cdk.core import App, Stack, Construct +from constructs import Construct +from aws_cdk import App, Stack class EmptyStack(Stack): def __init__(self, scope: Construct, construct_id: str, **kwargs): @@ -85,8 +86,16 @@ def test_load_cdk_app(cdk_app): assert callable(cdk_entrypoint) result = cdk_entrypoint() - stack = result.get_stack("empty-stack") - assert stack.template == {} # the stack generated should be empty + stack = result.get_stack_by_name("empty-stack") + + # CDK will include the bootstrap version Parameter and CheckBootstrapVersion Rule + assert not stack.template.get("Metadata") + assert not stack.template.get("Description") + assert not stack.template.get("Mappings") + assert not stack.template.get("Conditions") + assert not stack.template.get("Transform") + assert not stack.template.get("Resources") + assert not stack.template.get("Outputs") @pytest.mark.parametrize( diff --git a/source/tests/cdk_solution_helper/test_aspects.py b/source/tests/cdk_solution_helper/test_aspects.py index f694e91..a181f5e 100644 --- a/source/tests/cdk_solution_helper/test_aspects.py +++ b/source/tests/cdk_solution_helper/test_aspects.py @@ -12,8 +12,9 @@ # ###################################################################################################################### import pytest +from aws_cdk import App, Stack, Aspects, CfnCondition, Fn from aws_cdk.aws_sqs import Queue, CfnQueue -from aws_cdk.core import App, Stack, Construct, Aspects, CfnCondition, Fn +from constructs import Construct from aws_solutions.cdk.aspects import ConditionalResources @@ -44,7 +45,7 @@ def __init__(self, scope: Construct, construct_id: str, **kwargs) -> None: def stack_conditional(): app = App() SomeStack(app, "some-test-queues") - yield app.synth().get_stack("some-test-queues").template + yield app.synth().get_stack_by_name("some-test-queues").template def test_conditional_resources(stack_conditional): diff --git a/source/tests/cdk_solution_helper/test_cfn_nag_suppressions.py b/source/tests/cdk_solution_helper/test_cfn_nag_suppressions.py index 1bcb209..4b998b3 100644 --- a/source/tests/cdk_solution_helper/test_cfn_nag_suppressions.py +++ b/source/tests/cdk_solution_helper/test_cfn_nag_suppressions.py @@ -11,7 +11,7 @@ # the specific language governing permissions and limitations under the License. # # ##################################################################################################################### -from aws_cdk.core import CfnResource, App, Stack +from aws_cdk import CfnResource, App, Stack from aws_solutions.cdk.cfn_nag import add_cfn_nag_suppressions, CfnNagSuppression diff --git a/source/tests/cdk_solution_helper/test_interfaces.py b/source/tests/cdk_solution_helper/test_interfaces.py index ce8ba08..8e088f4 100644 --- a/source/tests/cdk_solution_helper/test_interfaces.py +++ b/source/tests/cdk_solution_helper/test_interfaces.py @@ -15,7 +15,7 @@ from pathlib import Path import pytest -from aws_cdk.core import App, Stack, NestedStack, CfnParameter +from aws_cdk import App, Stack, NestedStack, CfnParameter from aws_solutions.cdk.interfaces import ( _TemplateParameter, diff --git a/source/tests/cdk_solution_helper/test_mappings.py b/source/tests/cdk_solution_helper/test_mappings.py index f7588bd..966d0c3 100644 --- a/source/tests/cdk_solution_helper/test_mappings.py +++ b/source/tests/cdk_solution_helper/test_mappings.py @@ -11,7 +11,7 @@ # the specific language governing permissions and limitations under the License. # # ###################################################################################################################### import pytest -from aws_cdk.core import App, Stack +from aws_cdk import App, Stack from aws_solutions.cdk.mappings import Mappings diff --git a/source/tests/cdk_solution_helper/test_stack.py b/source/tests/cdk_solution_helper/test_stack.py index 5334d47..46a2cd1 100644 --- a/source/tests/cdk_solution_helper/test_stack.py +++ b/source/tests/cdk_solution_helper/test_stack.py @@ -14,7 +14,7 @@ import re import pytest -from aws_cdk.core import App, CfnParameter +from aws_cdk import App, CfnParameter from aws_solutions.cdk.stack import ( SolutionStack, @@ -89,16 +89,18 @@ def test_validate_re_exception(): def test_solution_stack(): stack_id = "S00123" + solution_version = "v0.0.1" stack_description = "stack description" stack_filename = "stack-name.template" - app = App(context={"SOLUTION_ID": stack_id, "SOLUTION_VERSION": "v0.0.1"}) + app = App(context={"SOLUTION_ID": stack_id, "SOLUTION_VERSION": solution_version}) SolutionStack(app, "stack", stack_description, stack_filename) template = app.synth().stacks[0].template assert ( - template["Description"] == f"({stack_id}) - {stack_description}. Version v0.0.1" + template["Description"] + == f"({stack_id}) - {stack_description}. Version {solution_version}" ) assert template["Metadata"] == { "AWS::CloudFormation::Interface": { @@ -106,6 +108,8 @@ def test_solution_stack(): "ParameterLabels": {}, }, "aws:solutions:templatename": "stack-name.template", + "aws:solutions:solution_id": stack_id, + "aws:solutions:solution_version": solution_version, } assert template["Conditions"] == { "SendAnonymousUsageData": { diff --git a/source/tests/cdk_solution_helper/test_synthesizers.py b/source/tests/cdk_solution_helper/test_synthesizers.py index ab7e46b..fd512df 100644 --- a/source/tests/cdk_solution_helper/test_synthesizers.py +++ b/source/tests/cdk_solution_helper/test_synthesizers.py @@ -14,7 +14,7 @@ import os import pytest -from aws_cdk.core import App, Stack +from aws_cdk import App, Stack from aws_solutions.cdk.interfaces import TemplateOptions from aws_solutions.cdk.mappings import Mappings diff --git a/source/tests/conftest.py b/source/tests/conftest.py index 2ba35a6..2cbece8 100644 --- a/source/tests/conftest.py +++ b/source/tests/conftest.py @@ -14,8 +14,9 @@ import sys from pathlib import Path from tempfile import TemporaryDirectory -from typing import Dict +from typing import Dict, Optional +import boto3 import jsii import pytest from aws_cdk.aws_lambda import ( @@ -26,8 +27,8 @@ LayerVersionProps, LayerVersion, ) -from aws_cdk.core import Construct from botocore.stub import Stubber +from constructs import Construct from aws_solutions.core import get_service_client @@ -188,3 +189,38 @@ def notifier_stubber(mocker): notifier = NotifierStub() mocker.patch("shared.events.NOTIFY_LIST", [notifier]) yield notifier + + +@pytest.fixture +def validate_handler_config(): + """Validates a handler configuration against the installed botocore shapes""" + + def _validate_handler_config( + resource: str, config: Dict, status: Optional[str] = None + ): + cli = boto3.client("personalize") + + shape = resource[0].upper() + resource[1:] + request_shape = cli.meta.service_model.shape_for(f"Create{shape}Request") + response_shape = cli.meta.service_model.shape_for(f"Describe{shape}Response") + + # check config parameter + for k in config.keys(): + if "workflowConfig" not in config[k].get("path"): + assert ( + k in request_shape.members.keys() + ), f"invalid key {k} not in Create{shape} API call" + + for k in request_shape.members.keys(): + assert k in config.keys(), "missing {k} in config" + + # check status parameter + if status: + m = response_shape + for k in status.split("."): + assert ( + k in m.members.keys() + ), f"status component {k} not found in {m.keys()}" + m = m.members[k] + + return _validate_handler_config diff --git a/source/tests/fixtures/config/sample_config.json b/source/tests/fixtures/config/sample_config.json index 67209e7..626f0ee 100644 --- a/source/tests/fixtures/config/sample_config.json +++ b/source/tests/fixtures/config/sample_config.json @@ -83,6 +83,34 @@ } }, "solutions": [ + { + "serviceConfig": { + "name": "affinity_item", + "recipeArn": "arn:aws:personalize:::recipe/aws-item-affinity" + }, + "batchSegmentJobs": [ + { + "serviceConfig": {}, + "workflowConfig": { + "schedule": "cron(0 3 * * ? *)" + } + } + ] + }, + { + "serviceConfig": { + "name": "affinity_item_attribute", + "recipeArn": "arn:aws:personalize:::recipe/aws-item-attribute-affinity" + }, + "batchSegmentJobs": [ + { + "serviceConfig": {}, + "workflowConfig": { + "schedule": "cron(0 3 * * ? *)" + } + } + ] + }, { "serviceConfig": { "name": "unit_test_sims_new", diff --git a/source/tests/test_deploy.py b/source/tests/test_deploy.py index f67e949..7441851 100644 --- a/source/tests/test_deploy.py +++ b/source/tests/test_deploy.py @@ -32,7 +32,7 @@ def test_deploy(solution, cdk_entrypoint): extra_context = "EXTRA_CONTEXT" source_bucket = "SOURCE_BUCKET" synth = build_app({extra_context: extra_context, "BUCKET_NAME": source_bucket}) - stack = synth.get_stack("PersonalizeStack") + stack = synth.get_stack_by_name("PersonalizeStack") assert solution.id in stack.template["Description"] assert ( source_bucket == stack.template["Mappings"]["SourceCode"]["General"]["S3Bucket"] @@ -56,7 +56,7 @@ def test_parameters(solution, cdk_entrypoint): extra_context = "EXTRA_CONTEXT" source_bucket = "SOURCE_BUCKET" synth = build_app({extra_context: extra_context, "BUCKET_NAME": source_bucket}) - stack = synth.get_stack("PersonalizeStack").template + stack = synth.get_stack_by_name("PersonalizeStack").template assert ( stack["Metadata"]["AWS::CloudFormation::Interface"]["ParameterGroups"][0][ diff --git a/source/tests/test_personalize_stack.py b/source/tests/test_personalize_stack.py index 17002ff..760797f 100644 --- a/source/tests/test_personalize_stack.py +++ b/source/tests/test_personalize_stack.py @@ -11,13 +11,13 @@ # the specific language governing permissions and limitations under the License. # # ###################################################################################################################### -import aws_cdk.core as cdk +from aws_cdk import App from infrastructure.personalize.stack import PersonalizeStack def test_personalize_stack_email(solution): - app = cdk.App(context=solution.context) + app = App(context=solution.context) PersonalizeStack( app, "PersonalizeStack", @@ -27,4 +27,4 @@ def test_personalize_stack_email(solution): synth = app.synth() # ensure the email parameter is present - assert synth.get_stack("PersonalizeStack").template["Parameters"]["Email"] + assert synth.get_stack_by_name("PersonalizeStack").template["Parameters"]["Email"] diff --git a/source/tests/test_resources.py b/source/tests/test_resources.py index e77860e..b978586 100644 --- a/source/tests/test_resources.py +++ b/source/tests/test_resources.py @@ -22,6 +22,8 @@ SolutionVersion, Campaign, EventTracker, + BatchSegmentJob, + BatchInferenceJob, ) @@ -41,6 +43,13 @@ (SolutionVersion, "solutionVersion", "solution-version", "solution_version"), (Campaign, "campaign", "campaign", "campaign"), (EventTracker, "eventTracker", "event-tracker", "event_tracker"), + ( + BatchInferenceJob, + "batchInferenceJob", + "batch-inference-job", + "batch_inference_job", + ), + (BatchSegmentJob, "batchSegmentJob", "batch-segment-job", "batch_segment_job"), ], ids=[ "DatasetGroup", @@ -51,6 +60,8 @@ "SolutionVersion", "Campaign", "EventTracker", + "BatchInferenceJob", + "BatchSegmentJob,", ], ) def test_resource_naming(klass, camel, dash, snake): diff --git a/source/tests/test_scheduler_cli.py b/source/tests/test_scheduler_cli.py index 1151762..f5ae07d 100644 --- a/source/tests/test_scheduler_cli.py +++ b/source/tests/test_scheduler_cli.py @@ -21,30 +21,37 @@ from aws_solutions.scheduler.common.scripts.scheduler_cli import ( get_stack_output_value, get_stack_tag_value, + get_stack_metadata_value, setup_cli_env, get_payload, ) +TEMPLATE = { + "AWSTemplateFormatVersion": "2010-09-09", + "Metadata": { + "aws:solutions:templatename": "maintaining-personalized-experiences-with-machine-learning.template", + "aws:solutions:solution_version": "solution_version_value", + "aws:solutions:solution_id": "solution_id_value", + }, + "Resources": { + "QueueResource": { + "Type": "AWS::SQS::Queue", + "Properties": {"QueueName": "my-queue"}, + } + }, + "Outputs": {"QueueOutput": {"Description": "The Queue Name", "Value": "my-queue"}}, +} + + @pytest.fixture -def stack(): - template = { - "AWSTemplateFormatVersion": "2010-09-09", - "Resources": { - "QueueResource": { - "Type": "AWS::SQS::Queue", - "Properties": {"QueueName": "my-queue"}, - } - }, - "Outputs": { - "QueueOutput": {"Description": "The Queue Name", "Value": "my-queue"} - }, - } +def stack(mocker): + global TEMPLATE with mock_cloudformation(): cli = boto3.client("cloudformation") cli.create_stack( StackName="TestStack", - TemplateBody=json.dumps(template), + TemplateBody=json.dumps(TEMPLATE), Tags=[ { "Key": "TestTag", @@ -54,7 +61,11 @@ def stack(): {"Key": "SOLUTION_VERSION", "Value": "SOLUTION_VERSION_VALUE"}, ], ) - yield boto3.resource("cloudformation").Stack("TestStack") + resource = boto3.resource("cloudformation").Stack("TestStack") + resource.meta.client.get_template_summary = mocker.MagicMock( + return_value=TEMPLATE | {"Metadata": json.dumps(TEMPLATE["Metadata"])} + ) + yield resource def test_get_stack_output_value(stack): @@ -75,12 +86,28 @@ def test_get_stack_tag_value_not_present(stack): get_stack_tag_value(stack, "missing") +def test_get_stack_metadata(stack, mocker): + assert ( + get_stack_metadata_value(stack, "aws:solutions:solution_id") + == "solution_id_value" + ) + assert ( + get_stack_metadata_value(stack, "aws:solutions:solution_version") + == "solution_version_value" + ) + + +def test_get_stack_metadata_not_present(stack, mocker): + with pytest.raises(ValueError): + get_stack_metadata_value(stack, "missing") + + def test_setup_cli_env(stack): with mock.patch.dict(os.environ, {}): setup_cli_env(stack, "eu-central-1") assert os.environ.get("AWS_REGION") == "eu-central-1" - assert os.environ.get("SOLUTION_ID") == "SOLUTION_ID_VALUE" - assert os.environ.get("SOLUTION_VERSION") == "SOLUTION_VERSION_VALUE" + assert os.environ.get("SOLUTION_ID") == "solution_id_value" + assert os.environ.get("SOLUTION_VERSION") == "solution_version_value" def test_get_payload():