diff --git a/pyproject.toml b/pyproject.toml index 4caa4bcd..98a11f9b 100644 --- a/pyproject.toml +++ b/pyproject.toml @@ -3,7 +3,7 @@ [tool.poetry] name = "aws-service-catalog-puppet" -version = "0.251.0" +version = "0.252.0" description = "Making it easier to deploy ServiceCatalog products" classifiers = ["Development Status :: 5 - Production/Stable", "Intended Audience :: Developers", "Programming Language :: Python :: 3", "License :: OSI Approved :: Apache Software License", "Operating System :: OS Independent", "Natural Language :: English"] homepage = "https://service-catalog-tools-workshop.com/" diff --git a/servicecatalog_puppet/commands/task_reference_helpers/complete_generator.py b/servicecatalog_puppet/commands/task_reference_helpers/complete_generator.py index 5268b7ca..c1963f14 100644 --- a/servicecatalog_puppet/commands/task_reference_helpers/complete_generator.py +++ b/servicecatalog_puppet/commands/task_reference_helpers/complete_generator.py @@ -21,6 +21,9 @@ from servicecatalog_puppet.commands.task_reference_helpers.generators.ssm_parameter_handler import ( ssm_parameter_handler, ) +from servicecatalog_puppet.commands.task_reference_helpers.generators.s3_parameter_handler import ( + s3_parameter_handler, +) from servicecatalog_puppet.waluigi.shared_tasks.task_topological_generations_without_scheduler_unit_test import ( dependency_task_reference, ) @@ -209,6 +212,14 @@ def generate(puppet_account_id, manifest, output_file_directory_path): puppet_account_id, task, ) + s3_parameter_handler( + all_tasks, + default_region, + new_tasks, + parameter_details, + puppet_account_id, + task, + ) # # Second pass - replacing dependencies with dependencies_by_reference and adding resources @@ -301,7 +312,6 @@ def generate(puppet_account_id, manifest, output_file_directory_path): # wire up dependencies for boto3 parameters boto3_parameters = task.get("boto3_parameters_tasks_references", {}).items() - dependencies_to_add = [] for parameter_name, parameter_task_reference in boto3_parameters: parameter_task = all_tasks.get(parameter_task_reference) for dependency_task_reference in task.get("dependencies_by_reference"): @@ -320,6 +330,26 @@ def generate(puppet_account_id, manifest, output_file_directory_path): if task_reference in parameter_task["dependencies_by_reference"]: parameter_task["dependencies_by_reference"].remove(task_reference) + # wire up dependencies for s3 parameters + s3_parameters = task.get("s3_parameters_tasks_references", {}).items() + for parameter_name, parameter_task_reference in s3_parameters: + parameter_task = all_tasks.get(parameter_task_reference) + for dependency_task_reference in task.get("dependencies_by_reference"): + if dependency_task_reference not in parameter_task.get( + "dependencies_by_reference" + ): + if dependency_task_reference != parameter_task_reference: + if ( + all_tasks[dependency_task_reference]["section_name"] + in constants.ALL_SECTION_NAMES + ): + parameter_task["dependencies_by_reference"].append( + dependency_task_reference + ) + + if task_reference in parameter_task["dependencies_by_reference"]: + parameter_task["dependencies_by_reference"].remove(task_reference) + # # Fourth pass - removing cyclic dependencies caused by a->b->c when using boto3 parameters # setting up deploy in spoke collections diff --git a/servicecatalog_puppet/commands/task_reference_helpers/generators/s3_parameter_handler.py b/servicecatalog_puppet/commands/task_reference_helpers/generators/s3_parameter_handler.py new file mode 100644 index 00000000..b76d4791 --- /dev/null +++ b/servicecatalog_puppet/commands/task_reference_helpers/generators/s3_parameter_handler.py @@ -0,0 +1,73 @@ +# Copyright 2023 Amazon.com, Inc. or its affiliates. All Rights Reserved. +# SPDX-License-Identifier: Apache-2.0 +from servicecatalog_puppet import constants, task_reference_constants + + +def s3_parameter_handler( + all_tasks, home_region, new_tasks, parameter_details, puppet_account_id, task +): + if parameter_details.get("s3"): + s3_parameter_details = parameter_details.get("s3") + key = s3_parameter_details.get("key") + jmespath = s3_parameter_details.get("jmespath") + + task_account_id = task.get("account_id") + task_region = task.get("region") + + key = ( + key.replace("${AWS::Region}", task_region) + .replace("${AWS::AccountId}", task_account_id) + .replace("${AWS::PuppetAccountId}", puppet_account_id) + ) + jmespath = ( + jmespath.replace("${AWS::Region}", task_region) + .replace("${AWS::AccountId}", task_account_id) + .replace("${AWS::PuppetAccountId}", puppet_account_id) + ) + + parameter_task_reference = f"{constants.S3_PARAMETERS}-{key}-{jmespath}" + + if all_tasks.get(parameter_task_reference): + s3_task_params = all_tasks.get(parameter_task_reference) + else: + s3_task_params = { + "task_reference": parameter_task_reference, + "account_id": puppet_account_id, + "region": home_region, + "key": key, + "jmespath": jmespath, + task_reference_constants.MANIFEST_SECTION_NAMES: dict(), + task_reference_constants.MANIFEST_ITEM_NAMES: dict(), + task_reference_constants.MANIFEST_ACCOUNT_IDS: dict(), + "dependencies": [], + "dependencies_by_reference": [], + "execution": constants.EXECUTION_MODE_HUB, + "section_name": constants.S3_PARAMETERS, + } + new_tasks[parameter_task_reference] = s3_task_params + + s3_task_params[task_reference_constants.MANIFEST_SECTION_NAMES].update( + **task.get(task_reference_constants.MANIFEST_SECTION_NAMES) + ) + s3_task_params[task_reference_constants.MANIFEST_ITEM_NAMES].update( + **task.get(task_reference_constants.MANIFEST_ITEM_NAMES) + ) + s3_task_params[task_reference_constants.MANIFEST_ACCOUNT_IDS].update( + **task.get(task_reference_constants.MANIFEST_ACCOUNT_IDS) + ) + + task["dependencies_by_reference"].append(parameter_task_reference) + + if not task.get("s3_parameters_tasks_references"): + task["s3_parameters_tasks_references"] = dict() + + parameter_name = ( + str(s3_parameter_details.get("name")) + .replace("${AWS::Region}", task_region) + .replace("${AWS::AccountId}", task_account_id) + .replace("${AWS::PuppetAccountId}", puppet_account_id) + ) + + task["s3_parameters_tasks_references"][ + parameter_name + ] = parameter_task_reference diff --git a/servicecatalog_puppet/constants.py b/servicecatalog_puppet/constants.py index 3eb3cc01..9156b1b9 100644 --- a/servicecatalog_puppet/constants.py +++ b/servicecatalog_puppet/constants.py @@ -331,6 +331,7 @@ SSM_OUTPUTS = "ssm_outputs" SSM_PARAMETERS = "ssm_parameters" +S3_PARAMETERS = "s3_parameters" BOTO3_PARAMETERS = "boto3_parameters" SSM_PARAMETERS_WITH_A_PATH = "ssm_parameters_with_a_path" PORTFOLIO_ASSOCIATIONS = "portfolio-associations" diff --git a/servicecatalog_puppet/template_builder/hub/bootstrap.py b/servicecatalog_puppet/template_builder/hub/bootstrap.py index 2bb3d21b..8912d3fd 100644 --- a/servicecatalog_puppet/template_builder/hub/bootstrap.py +++ b/servicecatalog_puppet/template_builder/hub/bootstrap.py @@ -176,6 +176,30 @@ def get_template( ) ) + template.add_resource( + s3.Bucket( + "ParameterRepository", + BucketName=t.Sub("sc-puppet-parameters-${AWS::AccountId}"), + VersioningConfiguration=s3.VersioningConfiguration(Status="Enabled"), + BucketEncryption=s3.BucketEncryption( + ServerSideEncryptionConfiguration=[ + s3.ServerSideEncryptionRule( + ServerSideEncryptionByDefault=s3.ServerSideEncryptionByDefault( + SSEAlgorithm="AES256" + ) + ) + ] + ), + PublicAccessBlockConfiguration=s3.PublicAccessBlockConfiguration( + BlockPublicAcls=True, + BlockPublicPolicy=True, + IgnorePublicAcls=True, + RestrictPublicBuckets=True, + ), + Tags=t.Tags({"ServiceCatalogPuppet:Actor": "Framework"}), + ) + ) + log_bucket = template.add_resource( s3.Bucket( "LogStore", diff --git a/servicecatalog_puppet/workflow/dependencies/resources_factory.py b/servicecatalog_puppet/workflow/dependencies/resources_factory.py index 992e752e..880e7f8d 100644 --- a/servicecatalog_puppet/workflow/dependencies/resources_factory.py +++ b/servicecatalog_puppet/workflow/dependencies/resources_factory.py @@ -144,6 +144,8 @@ "SSM_GET_PARAMETER_BY_PATH" + PER_REGION_OF_ACCOUNT ) +S3_GET_OBJECT_PER_REGION_OF_ACCOUNT = "S3_GET_OBJECT" + PER_REGION_OF_ACCOUNT + ORGANIZATIONS_ATTACH_POLICY_PER_REGION = "ORGANIZATIONS_ATTACH_POLICY" + PER_REGION ORGANIZATIONS_DETACH_POLICY_PER_REGION = "ORGANIZATIONS_DETACH_POLICY" + PER_REGION @@ -227,6 +229,9 @@ def create(section_name, parameters_to_use, puppet_account_id): elif section_name == constants.BOTO3_PARAMETERS: resources = [] + elif section_name == constants.S3_PARAMETERS: + resources = [S3_GET_OBJECT_PER_REGION_OF_ACCOUNT] + elif section_name == constants.SSM_PARAMETERS_WITH_A_PATH: resources = [ SSM_GET_PARAMETER_BY_PATH_PER_REGION_OF_ACCOUNT, diff --git a/servicecatalog_puppet/workflow/dependencies/task_factory.py b/servicecatalog_puppet/workflow/dependencies/task_factory.py index 57598a6e..00260e84 100644 --- a/servicecatalog_puppet/workflow/dependencies/task_factory.py +++ b/servicecatalog_puppet/workflow/dependencies/task_factory.py @@ -161,6 +161,15 @@ def create( jmespath_location=parameters_to_use.get("jmespath"), ) + elif section_name == constants.S3_PARAMETERS: + from servicecatalog_puppet.workflow.s3 import get_s3_parameter_task + + return get_s3_parameter_task.GetS3ParameterTask( + **common_parameters, + key=parameters_to_use.get("key"), + jmespath_location=parameters_to_use.get("jmespath"), + ) + elif section_name == constants.SSM_OUTPUTS: from servicecatalog_puppet.workflow.ssm import ssm_outputs_task diff --git a/servicecatalog_puppet/workflow/dependencies/tasks.py b/servicecatalog_puppet/workflow/dependencies/tasks.py index 1d943d00..50edf70b 100644 --- a/servicecatalog_puppet/workflow/dependencies/tasks.py +++ b/servicecatalog_puppet/workflow/dependencies/tasks.py @@ -231,6 +231,28 @@ def get_parameter_values(self): ) all_params[param_name] = parameter_task_output + if param_details.get("s3"): + requested_param_details = param_details.get("s3") + task_ref = ( + f"{constants.S3_PARAMETERS}" + f"-{requested_param_details.get('key')}" + f"-{requested_param_details.get('jmespath')}" + ) + + task_ref = ( + str(task_ref.replace("${AWS::AccountId}", self.account_id)) + .replace("${AWS::PuppetAccountId}", self.puppet_account_id) + .replace("${AWS::Region}", self.region) + ) + + print( + f"Getting output for task: {task_ref} within task {self.task_reference}" + ) + parameter_task_output = self.get_output_from_reference_dependency( + task_ref + ) + all_params[param_name] = parameter_task_output + if param_details.get("default"): all_params[param_name] = ( str(param_details.get("default")) diff --git a/servicecatalog_puppet/workflow/s3/get_s3_parameter_task.py b/servicecatalog_puppet/workflow/s3/get_s3_parameter_task.py new file mode 100644 index 00000000..23b2d3b3 --- /dev/null +++ b/servicecatalog_puppet/workflow/s3/get_s3_parameter_task.py @@ -0,0 +1,39 @@ +# Copyright 2023 Amazon.com, Inc. or its affiliates. All Rights Reserved. +# SPDX-License-Identifier: Apache-2.0 +import json +from copy import deepcopy + +import luigi + +from servicecatalog_puppet import constants +from servicecatalog_puppet.workflow.dependencies import tasks +import jmespath + + +class GetS3ParameterTask(tasks.TaskWithReference): + account_id = luigi.Parameter() + key = luigi.Parameter() + jmespath_location = luigi.Parameter() + region = luigi.Parameter() + cachable_level = constants.CACHE_LEVEL_RUN + + def params_for_results_display(self): + return { + "task_reference": self.task_reference, + "account_id": self.account_id, + "region": self.region, + "key": self.key, + "jmespath_location": self.jmespath_location, + } + + def run(self): + with self.spoke_regional_client("s3") as s3: + object = ( + s3.get_object( + Bucket=f"sc-puppet-parameters-{self.account_id}", Key=self.key + ) + .get("Body") + .read() + ) + result = jmespath.search(self.jmespath_location, json.loads(object)) + self.write_output(result) diff --git a/setup.py b/setup.py index 22ee33e4..b562593c 100644 --- a/setup.py +++ b/setup.py @@ -36,6 +36,7 @@ 'servicecatalog_puppet.workflow.portfolio.constraints_management', 'servicecatalog_puppet.workflow.portfolio.portfolio_management', 'servicecatalog_puppet.workflow.portfolio.sharing_management', + 'servicecatalog_puppet.workflow.s3', 'servicecatalog_puppet.workflow.service_control_policies', 'servicecatalog_puppet.workflow.simulate_policies', 'servicecatalog_puppet.workflow.ssm', @@ -73,7 +74,7 @@ setup_kwargs = { 'name': 'aws-service-catalog-puppet', - 'version': '0.251.0', + 'version': '0.252.0', 'description': 'Making it easier to deploy ServiceCatalog products', 'long_description': '# aws-service-catalog-puppet\n\n![logo](./docs/logo.png) \n\n## Badges\n\n[![codecov](https://codecov.io/gh/awslabs/aws-service-catalog-puppet/branch/master/graph/badge.svg?token=e8M7mdsmy0)](https://codecov.io/gh/awslabs/aws-service-catalog-puppet)\n\n\n## What is it?\nThis is a python3 framework that makes it easier to share multi region AWS Service Catalog portfolios and makes it \npossible to provision products into accounts declaratively using a metadata based rules engine.\n\nWith this framework you define your accounts in a YAML file. You give each account a set of tags, a default region and \na set of enabled regions.\n\nOnce you have done this you can define portfolios should be shared with each set of accounts using the tags and you \ncan specify which regions the shares occur in.\n\nIn addition to this, you can also define products that should be provisioned into accounts using the same tag based \napproach. The framework will assume role into the target account and provision the product on your behalf.\n\n\n## Getting started\n\nYou can read the [installation how to](https://service-catalog-tools-workshop.com/30-how-tos/10-installation/30-service-catalog-puppet.html)\nor you can read through the [every day use](https://service-catalog-tools-workshop.com/30-how-tos/50-every-day-use.html)\nguides.\n\nYou can read the [documentation](https://aws-service-catalog-puppet.readthedocs.io/en/latest/) to understand the inner \nworkings. \n\n\n## Going further\n\nThe framework is one of a pair. The other is [aws-service-catalog-factory](https://github.com/awslabs/aws-service-catalog-factory).\nWith Service Catalog Factory you can create pipelines that deploy multi region portfolios very easily. \n\n## License\n\nThis library is licensed under the Apache 2.0 License. \n \n', 'author': 'Eamonn Faherty',