launch role constraints, resource update constraints and associations…

… are now grouped by and managed by portfolio not by item
awslabs · Jul 5, 2024 · f651ce9 · f651ce9
1 parent 418f1b2
commit f651ce9
Show file tree

Hide file tree

Showing 9 changed files with 154 additions and 42 deletions.
diff --git a/servicecatalog_puppet/commands/task_reference_helpers/generators/spoke_local_portfolios.py b/servicecatalog_puppet/commands/task_reference_helpers/generators/spoke_local_portfolios.py
@@ -1,7 +1,8 @@
 #  Copyright 2023 Amazon.com, Inc. or its affiliates. All Rights Reserved.
 #  SPDX-License-Identifier: Apache-2.0
-from servicecatalog_puppet import config, constants, task_reference_constants
+from servicecatalog_puppet import config, constants, task_reference_constants, utils
 from servicecatalog_puppet.commands.task_reference_helpers.generators import portfolios
+from servicecatalog_puppet.workflow.portfolio.associations import association_utils
 
 
 def get_spoke_local_portfolio_common_args(
@@ -563,17 +564,61 @@ def handle_spoke_local_portfolios(
             dependencies_for_constraints.append(portfolio_import_or_copy_ref)
 
         if task_to_add.get("associations"):
-            shared_ref = f"{section_name}-{item_name}-{task_to_add.get('account_id')}-{task_to_add.get('region')}"
-            ref = f"portfolio_associations-{shared_ref}"
-            all_tasks[ref] = dict(
-                **get_spoke_local_portfolio_common_args(
-                    task_to_add, all_tasks_task_reference, [constants.CREATE_POLICIES],
-                ),
-                task_reference=ref,
-                spoke_local_portfolio_name=item_name,
-                section_name=constants.PORTFOLIO_ASSOCIATIONS,
-                associations=task_to_add.get("associations"),
+            v1_stack_name = f"associations-for-{utils.slugify_for_cloudformation_stack_name(item_name)}"
+            task_ref_for_v1_stack = f"{constants.TERMINATE_CLOUDFORMATION_STACK_TASK}-{v1_stack_name}-{task_to_add.get('account_id')}-{task_to_add.get('region')}"
+            if not all_tasks.get(task_ref_for_v1_stack):
+                all_tasks[task_ref_for_v1_stack] = dict(
+                    account_id=task_to_add.get("account_id"),
+                    region=task_to_add.get("region"),
+                    execution=task_to_add.get("execution"),
+                    task_reference=task_ref_for_v1_stack,
+                    section_name=constants.TERMINATE_CLOUDFORMATION_STACK_TASK,
+                    dependencies_by_reference=[],
+                    stack_name=v1_stack_name,
+                )
+
+            v2_stack_name = association_utils.generate_stack_name_for_associations_by_item_name(
+                item_name
             )
+            task_ref_for_v2_stack = f"{constants.TERMINATE_CLOUDFORMATION_STACK_TASK}-{v2_stack_name}-{task_to_add.get('account_id')}-{task_to_add.get('region')}"
+            if not all_tasks.get(task_ref_for_v2_stack):
+                all_tasks[task_ref_for_v2_stack] = dict(
+                    account_id=task_to_add.get("account_id"),
+                    region=task_to_add.get("region"),
+                    execution=task_to_add.get("execution"),
+                    task_reference=task_ref_for_v2_stack,
+                    section_name=constants.TERMINATE_CLOUDFORMATION_STACK_TASK,
+                    dependencies_by_reference=[],
+                    stack_name=v2_stack_name,
+                )
+
+            ref = f"portfolio_associations-{task_to_add.get('portfolio')}-{task_to_add.get('account_id')}-{task_to_add.get('region')}"
+            if all_tasks.get(ref):
+                for association in task_to_add.get("associations"):
+                    if association not in all_tasks[ref]["associations"]:
+                        all_tasks[ref]["associations"].append(association)
+                        all_tasks[ref]["dependencies_by_reference"].append(
+                            task_ref_for_v1_stack
+                        )
+                        all_tasks[ref]["dependencies_by_reference"].append(
+                            task_ref_for_v2_stack
+                        )
+            else:
+                all_tasks[ref] = dict(
+                    **get_spoke_local_portfolio_common_args(
+                        task_to_add,
+                        all_tasks_task_reference,
+                        [
+                            constants.CREATE_POLICIES,
+                            task_ref_for_v1_stack,
+                            task_ref_for_v2_stack,
+                        ],
+                    ),
+                    task_reference=ref,
+                    spoke_local_portfolio_name=item_name,
+                    section_name=constants.PORTFOLIO_ASSOCIATIONS,
+                    associations=task_to_add.get("associations"),
+                )
             all_tasks[ref][task_reference_constants.MANIFEST_SECTION_NAMES] = dict(
                 **task_to_add.get(task_reference_constants.MANIFEST_SECTION_NAMES)
             )
@@ -623,6 +668,19 @@ def handle_spoke_local_portfolios(
 
         shared_ref = f"{task_to_add.get('portfolio')}-{task_to_add.get('account_id')}-{task_to_add.get('region')}"
         if task_to_add.get("launch_constraints"):
+            stack_to_terminate = f"launch-constraints-for-{utils.slugify_for_cloudformation_stack_name(item_name)}"
+            task_ref_for_stack_to_terminate = f"{constants.TERMINATE_CLOUDFORMATION_STACK_TASK}-{stack_to_terminate}-{task_to_add.get('account_id')}-{task_to_add.get('region')}"
+            if not all_tasks.get(task_ref_for_stack_to_terminate):
+                all_tasks[task_ref_for_stack_to_terminate] = dict(
+                    account_id=task_to_add.get("account_id"),
+                    region=task_to_add.get("region"),
+                    execution=task_to_add.get("execution"),
+                    task_reference=task_ref_for_stack_to_terminate,
+                    section_name=constants.TERMINATE_CLOUDFORMATION_STACK_TASK,
+                    dependencies_by_reference=[],
+                    stack_name=stack_to_terminate,
+                )
+
             ref = f"launch_constraints-{shared_ref}"
             if all_tasks.get(ref):
                 task = all_tasks[ref]
@@ -649,22 +707,31 @@ def handle_spoke_local_portfolios(
                     launch_constraints=dict(products=dict()),
                     portfolio_get_all_products_and_their_versions_ref=spoke_portfolio_all_products_and_versions_after_ref,
                 )
+            all_tasks[ref]["dependencies_by_reference"].append(
+                task_ref_for_stack_to_terminate
+            )
 
             for launch_constraint in task_to_add["launch_constraints"]:
                 if launch_constraint.get("product"):
                     product = launch_constraint.get("product")
                     if not task["launch_constraints"]["products"].get(product):
                         task["launch_constraints"]["products"][product] = []
-                    task["launch_constraints"]["products"][
-                        product
-                    ] += launch_constraint.get("roles")
+
+                    for role in launch_constraint.get("roles"):
+                        if role not in task["launch_constraints"]["products"][product]:
+                            task["launch_constraints"]["products"][product].append(role)
                 if launch_constraint.get("products"):
                     for product in launch_constraint["products"]:
                         if not task["launch_constraints"]["products"].get(product):
                             task["launch_constraints"]["products"][product] = []
-                        task["launch_constraints"]["products"][
-                            product
-                        ] += launch_constraint.get("roles")
+                        for role in launch_constraint.get("roles"):
+                            if (
+                                role
+                                not in task["launch_constraints"]["products"][product]
+                            ):
+                                task["launch_constraints"]["products"][product].append(
+                                    role
+                                )
 
             task[task_reference_constants.MANIFEST_SECTION_NAMES] = dict(
                 **task_to_add.get(task_reference_constants.MANIFEST_SECTION_NAMES)
@@ -677,6 +744,19 @@ def handle_spoke_local_portfolios(
             )
 
         if task_to_add.get("resource_update_constraints"):
+            stack_to_terminate = f"update-resource-constraints-for-{utils.slugify_for_cloudformation_stack_name(item_name)}"
+            task_ref_for_stack_to_terminate = f"{constants.TERMINATE_CLOUDFORMATION_STACK_TASK}-{stack_to_terminate}-{task_to_add.get('account_id')}-{task_to_add.get('region')}"
+            if not all_tasks.get(task_ref_for_stack_to_terminate):
+                all_tasks[task_ref_for_stack_to_terminate] = dict(
+                    account_id=task_to_add.get("account_id"),
+                    region=task_to_add.get("region"),
+                    execution=task_to_add.get("execution"),
+                    task_reference=task_ref_for_stack_to_terminate,
+                    section_name=constants.TERMINATE_CLOUDFORMATION_STACK_TASK,
+                    dependencies_by_reference=[],
+                    stack_name=stack_to_terminate,
+                )
+
             ref = f"resource_update_constraints-{shared_ref}"
             if all_tasks.get(ref):
                 task = all_tasks[ref]
@@ -703,6 +783,9 @@ def handle_spoke_local_portfolios(
                     resource_update_constraints=dict(products=dict()),
                     portfolio_get_all_products_and_their_versions_ref=spoke_portfolio_all_products_and_versions_after_ref,
                 )
+            all_tasks[ref]["dependencies_by_reference"].append(
+                task_ref_for_stack_to_terminate
+            )
 
             for resource_update_constraint in task_to_add[
                 "resource_update_constraints"

diff --git a/...ecatalog_puppet/commands/task_reference_helpers/generators/spoke_local_portfolios_test.py b/...ecatalog_puppet/commands/task_reference_helpers/generators/spoke_local_portfolios_test.py
@@ -281,6 +281,8 @@ def test_for_accounts(self):
                 "dependencies_by_reference": [
                     f"imported-portfolios_{item_name}_{account_id}_{region}",
                     "create-policies",
+                    f"{constants.TERMINATE_CLOUDFORMATION_STACK_TASK}-associations-for-{item_name}-{account_id}-{region}",
+                    f"{constants.TERMINATE_CLOUDFORMATION_STACK_TASK}-associations-v2-for-{item_name}-{account_id}-{region}",
                 ],
                 "execution": "hub",
                 "spoke_local_portfolio_name": item_name,
@@ -294,11 +296,9 @@ def test_for_accounts(self):
                 "region": region,
                 "section_name": "portfolio-associations",
                 "status": None,
-                "task_reference": f"portfolio_associations-imported-portfolios-{item_name}-{account_id}-{region}",
+                "task_reference": f"portfolio_associations-{portfolio}-{account_id}-{region}",
             },
-            all_tasks[
-                f"portfolio_associations-imported-portfolios-{item_name}-{account_id}-{region}"
-            ],
+            all_tasks[f"portfolio_associations-{portfolio}-{account_id}-{region}"],
         )
 
         self.assertEqual(
@@ -326,4 +326,4 @@ def test_for_accounts(self):
                 f"portfolio-get-all-products-and-their-versions-after-{account_id}-{region}-{portfolio}"
             ],
         )
-        self.assertEqual(11, n_all_tasks)
+        self.assertEqual(13, n_all_tasks)
diff --git a/servicecatalog_puppet/constants.py b/servicecatalog_puppet/constants.py
@@ -427,3 +427,4 @@
     "arn:aws:iam::${AWS::Partition}:policy/AdministratorAccess"
 ]
 C7N_VERSION_DEFAULT = "0.9.28"
+TERMINATE_CLOUDFORMATION_STACK_TASK = "terminate-CloudFormation-stack"
diff --git a/servicecatalog_puppet/workflow/dependencies/resources_factory.py b/servicecatalog_puppet/workflow/dependencies/resources_factory.py
@@ -520,6 +520,8 @@ def create(section_name, parameters_to_use, puppet_account_id):
         resources = [
             # TODO choose what goes in here
         ]
+    elif section_name == constants.TERMINATE_CLOUDFORMATION_STACK_TASK:
+        resources = [CLOUDFORMATION_ENSURE_DELETED_PER_REGION_OF_ACCOUNT]
     else:
         print(f"WARNING: {section_name} did not generate any resources")
 

diff --git a/servicecatalog_puppet/workflow/dependencies/task_factory.py b/servicecatalog_puppet/workflow/dependencies/task_factory.py
@@ -877,5 +877,16 @@ def create(
             uses_orgs=parameters_to_use.get("uses_orgs",),
         )
 
+    elif section_name == constants.TERMINATE_CLOUDFORMATION_STACK_TASK:
+        from servicecatalog_puppet.workflow.general import (
+            terminate_cloudformation_stack_task,
+        )
+
+        return terminate_cloudformation_stack_task.TerminateCloudFormationStackTask(
+            **common_parameters,
+            stack_name=parameters_to_use.get("stack_name"),
+            execution=parameters_to_use.get("execution"),
+        )
+
     else:
         raise Exception(f"Unknown section_name: {section_name}")
diff --git a/servicecatalog_puppet/workflow/general/terminate_cloudformation_stack_task.py b/servicecatalog_puppet/workflow/general/terminate_cloudformation_stack_task.py
@@ -0,0 +1,31 @@
+#  Copyright 2023 Amazon.com, Inc. or its affiliates. All Rights Reserved.
+#  SPDX-License-Identifier: Apache-2.0
+
+import luigi
+
+from servicecatalog_puppet import constants
+from servicecatalog_puppet.workflow.dependencies import tasks
+
+
+class TerminateCloudFormationStackTask(tasks.TaskWithReference):
+    stack_name = luigi.Parameter()
+
+    region = luigi.Parameter()
+    account_id = luigi.Parameter()
+
+    execution = luigi.Parameter()
+
+    cachable_level = constants.CACHE_LEVEL_RUN
+
+    def params_for_results_display(self):
+        return {
+            "puppet_account_id": self.puppet_account_id,
+            "stack_name": self.stack_name,
+            "account_id": self.account_id,
+            "region": self.region,
+        }
+
+    def run(self):
+        with self.spoke_regional_client("cloudformation") as cloudformation:
+            cloudformation.ensure_deleted(StackName=self.stack_name)
+        self.write_empty_output()
diff --git a/...pet/workflow/portfolio/associations/create_associations_for_spoke_local_portfolio_task.py b/...pet/workflow/portfolio/associations/create_associations_for_spoke_local_portfolio_task.py
@@ -88,13 +88,9 @@ def run(self):
             associations_to_use = self.associations
 
         with self.spoke_regional_client("cloudformation") as cloudformation:
-            v1_stack_name = f"associations-for-{utils.slugify_for_cloudformation_stack_name(self.spoke_local_portfolio_name)}"
-            cloudformation.ensure_deleted(StackName=v1_stack_name)
-
-            v2_stack_name = association_utils.generate_stack_name_for_associations_by_item_name(
-                self.spoke_local_portfolio_name
+            stack_name = association_utils.generate_stack_name_for_associations_by_item_name(
+                self.portfolio
             )
-
             if associations_to_use:
                 tpl = t.Template()
                 tpl.description = f"Associations for {self.portfolio}"
@@ -115,7 +111,7 @@ def run(self):
                     )
                 template = tpl.to_yaml()
                 cloudformation.create_or_update(
-                    StackName=v2_stack_name,
+                    StackName=stack_name,
                     TemplateBody=template,
                     NotificationARNs=[
                         f"arn:{config.get_partition()}:sns:{self.region}:{self.puppet_account_id}:servicecatalog-puppet-cloudformation-regional-events"
@@ -126,5 +122,5 @@ def run(self):
                     Tags=self.initialiser_stack_tags,
                 )
             else:
-                cloudformation.ensure_deleted(StackName=v2_stack_name)
+                cloudformation.ensure_deleted(StackName=stack_name)
         self.write_empty_output()
diff --git a/...o/constraints_management/create_launch_role_constraints_for_spoke_local_portfolio_task.py b/...o/constraints_management/create_launch_role_constraints_for_spoke_local_portfolio_task.py
@@ -71,10 +71,6 @@ def run(self):
         template_to_use = tpl.to_yaml()
 
         with self.spoke_regional_client("cloudformation") as cloudformation:
-            ## TODO move into own task
-            cloudformation.ensure_deleted(
-                StackName=f"launch-constraints-for-{utils.slugify_for_cloudformation_stack_name(self.spoke_local_portfolio_name)}"
-            )
             stack_name = f"launch-constraints-for-{utils.slugify_for_cloudformation_stack_name(self.portfolio)}"
             cloudformation.create_or_update(
                 StackName=stack_name,
@@ -87,7 +83,4 @@ def run(self):
                 ShouldDeleteRollbackComplete=self.should_delete_rollback_complete_stacks,
                 Tags=self.initialiser_stack_tags,
             )
-            result = cloudformation.describe_stacks(StackName=stack_name,).get(
-                "Stacks"
-            )[0]
             self.write_empty_output()
diff --git a/...nstraints_management/create_resource_update_constraints_for_spoke_local_portfolio_task.py b/...nstraints_management/create_resource_update_constraints_for_spoke_local_portfolio_task.py
@@ -72,11 +72,6 @@ def run(self):
         template_to_use = tpl.to_yaml()
 
         with self.spoke_regional_client("cloudformation") as cloudformation:
-            # TODO move to another task
-            cloudformation.ensure_deleted(
-                f"update-resource-constraints-for-{utils.slugify_for_cloudformation_stack_name(self.spoke_local_portfolio_name)}"
-            )
-
             stack_name = f"update-resource-constraints-for-{utils.slugify_for_cloudformation_stack_name(self.portfolio)}"
             cloudformation.create_or_update(
                 StackName=stack_name,