Adding in creation complete event to state machine

Ignoring check I3042 for cfn-lint

Adding in EventBus to deployment account

Adding in x-ray tracing to pipeline management lambda functions

Changing event content, adding in xray layer to pipeline management lambda functions

Documentation

Mega Lint Fixes

Forgot to hit save :(

README.md

@@ -36,3 +36,4 @@ within the AWS Console.
 - Refer to the [User Guide](docs/user-guide.md) for using ADF once it is setup.
 - Refer to the [Samples Guide](docs/samples-guide.md) for a detailed walk
   through of the provided samples.
+- Refer to the [Integrations Guide](docs/integrations-guide.md) for information on events produced by the ADF.

docs/integrations-guide.md

@@ -0,0 +1,38 @@
+# Integrations Guide
+## Introduction
+The AWS Deployment Framework enables integrations with external workflows via an Event Bus deployed into the organisational root account.
+
+## Account Management Events
+The account management events are emitted at various stages during an execution of the Account Management State Machine.
+Currently - events are emitted for the following states:
+- ACCOUNT_PROVISIONED
+    Emitted when an AWS account is created.
+    Contains the account definition from the .yml file as well as the account_id.
+- ENTERPRISE_SUPPORT_REQUESTED
+    Emitted when the support ticket to AWS Support is raised.
+    Contains the account definition from the .yml file as well as the account_id.
+- ACCOUNT_ALIAS_CONFIGURED
+    Emitted when the accounts alias is configured by ADF.
+    The details section contains the account id and the alias value. The resource field also contains the account id
+- ACCOUNT_TAGS_CONFIGURED
+    Emitted when the accounts tags are updated by ADF.
+    The details section contains the account id and the tags. The resource field also contains the account id
+- DEFAULT_VPC_DELETED
+    Emitted when the default VPC in a region is deleted.
+    The details section contains the account id and the region of the VPC. The resource field contains the deleted VPC id.
+- ACCOUNT_CREATION_COMPLETE
+    Emitted when the state machine completes successfully.
+    Contains the account definition from the .yml file as well as the account_id in the resource field.
+
+
+
+
+## Pipeline Management Events
+- CROSS_ACCOUNT_RULE_CREATED_OR_UPDATED
+    Emitted when a rule is created to trigger pipelines from a different account.
+    The details sections contains the source_account_id (The account where the CodeCommit repository is located) and the resource sections contains the deployment account Id (The account where the CodePipeline is located)
+- REPOSITORY_CREATED_OR_UPDATED
+    Emitted when a codecommit repository is created in a different account than the deployment account.
+    The details sections contains the repository_account_id (The account where the CodeCommit repository is located) as well as the stack_name (The CloudFormation stack that creates the repository) and the resource sections contains the repository account Id and the pipeline name
+
+

src/lambda_codebase/account_processing/configure_account_alias.py

@@ -18,7 +18,7 @@
 LOGGER = configure_logger(__name__)
 ADF_ROLE_NAME = os.getenv("ADF_ROLE_NAME")
 AWS_PARTITION = os.getenv("AWS_PARTITION")
-EVENTS =  ADFEvents(boto3.client("events"), "AccountManagement.Alias")
+EVENTS =  ADFEvents(boto3.client("events"), "AccountManagement")
 
 
 def delete_account_aliases(account, iam_client, current_aliases):
@@ -80,7 +80,7 @@ def lambda_handler(event, _):
             "adf_account_alias_config",
         )
         ensure_account_has_alias(event, role.client("iam"))
-        EVENTS.put_event(detail=json.dumps(event), detailType="ACCOUNT_ALIAS_CONFIGURED", resources=[account_id])
+        EVENTS.put_event(detail=json.dumps({"account_id": account_id, "alias_value": event.get("alias")}), detailType="ACCOUNT_ALIAS_CONFIGURED", resources=[account_id])
     else:
         LOGGER.info(
             "Account: %s does not need an alias",

src/lambda_codebase/account_processing/configure_account_tags.py

@@ -19,7 +19,7 @@
 from events import ADFEvents
 
 patch_all()
-EVENTS =  ADFEvents(boto3.client("events"), "AccountManagement.Tags")
+EVENTS =  ADFEvents(boto3.client("events"), "AccountManagement")
 LOGGER = configure_logger(__name__)
 
 
@@ -40,7 +40,7 @@ def lambda_handler(event, _):
             event.get("tags"),
             organizations,
         )
-        EVENTS.put_event(detail=json.dumps(event), detailType="ACCOUNT_TAGS_CONFIGURED", resources=[event.get('account_id')])
+        EVENTS.put_event(detail=json.dumps({"tags": event.get("tags"), "account_id": event.get("account_id")}), detailType="ACCOUNT_TAGS_CONFIGURED", resources=[event.get('account_id')])
     else:
         LOGGER.info(
             "Account: %s does not need tags configured",

src/lambda_codebase/account_processing/create_account.py

@@ -18,7 +18,7 @@
 
 LOGGER = configure_logger(__name__)
 ADF_ROLE_NAME = os.getenv("ADF_ROLE_NAME")
-EVENTS =  ADFEvents(boto3.client("events"), "AccountManagement.AccountProvisioning")
+EVENTS =  ADFEvents(boto3.client("events"), "AccountManagement")
 
 
 

src/lambda_codebase/account_processing/delete_default_vpc.py

@@ -17,7 +17,7 @@
 LOGGER = configure_logger(__name__)
 ADF_ROLE_NAME = os.getenv("ADF_ROLE_NAME")
 AWS_PARTITION = os.getenv("AWS_PARTITION")
-EVENTS =  ADFEvents(boto3.client("events"), "AccountManagement.VPC")
+EVENTS =  ADFEvents(boto3.client("events"), "AccountManagement")
 
 
 
@@ -84,7 +84,7 @@ def lambda_handler(event, _):
         )
         ec2_resource = role.resource("ec2", region_name=event.get("region"))
         delete_default_vpc(ec2_resource, ec2_client, default_vpc_id)
-        EVENTS.put_event(detail=json.dumps(event), detailType="DEFAULT_VPC_DELETED", resources=[event.get("account_id"), default_vpc_id])
+        EVENTS.put_event(detail=json.dumps({"region": event.get("region"), "account_id":event.get("account_id")}), detailType="DEFAULT_VPC_DELETED", resources=[default_vpc_id])
 
 
     return {"Payload": event}

src/lambda_codebase/initial_commit/bootstrap_repository/adf-bootstrap/deployment/global.yml

@@ -69,6 +69,13 @@ Globals:
     CodeUri: lambda_codebase
     Runtime: python3.9
 
+Mappings:
+  OrganisationPartitionRegionMapping:
+    aws:
+      region: "us-east-1"
+    aws-us-gov:
+      region: "us-gov-west-1"
+
 Resources:
   LambdaLayerVersion:
     Type: "AWS::Serverless::LayerVersion"
@@ -183,6 +190,7 @@ Resources:
         CrossAccountAccessRole: !Ref CrossAccountAccessRole
         PipelineBucket: !Ref PipelineBucket
         RootAccountId: !Ref MasterAccountId
+        RootAccountRegion: !FindInMap [OrganisationPartitionRegionMapping, !Ref "AWS::Partition", "region"]
         CodeBuildImage: !Ref Image
         CodeBuildComputeType: !Ref ComputeType
         SharedModulesBucket: !Ref SharedModulesBucket

src/lambda_codebase/initial_commit/bootstrap_repository/adf-bootstrap/deployment/lambda_codebase/pipeline_management/create_or_update_rule.py

@@ -5,20 +5,25 @@
 """
 
 import os
+import json
 import boto3
 
 from cache import Cache
 from rule import Rule
 from logger import configure_logger
 from cloudwatch import ADFMetrics
+from events import ADFEvents
+from aws_xray_sdk.core import patch_all
 
 
+patch_all()
 LOGGER = configure_logger(__name__)
 DEPLOYMENT_ACCOUNT_REGION = os.environ["AWS_REGION"]
 DEPLOYMENT_ACCOUNT_ID = os.environ["ACCOUNT_ID"]
 PIPELINE_MANAGEMENT_STATEMACHINE = os.getenv("PIPELINE_MANAGEMENT_STATEMACHINE_ARN")
 CLOUDWATCH = boto3.client("cloudwatch")
 METRICS = ADFMetrics(CLOUDWATCH, "PIPELINE_MANAGEMENT/RULE")
+EVENTS = ADFEvents(boto3.client("events", region_name=os.getenv("ADF_EVENTBUS_REGION")), "PipelineManagement")
 
 _cache = None
 
@@ -56,5 +61,6 @@ def lambda_handler(pipeline, _):
         METRICS.put_metric_data(
             {"MetricName": "CreateOrUpdate", "Value": 1, "Unit": "Count"}
         )
+        EVENTS.put_event(detail=json.dumps({"source_account_id": _source_account_id}), detailType="CROSS_ACCOUNT_RULE_CREATED_OR_UPDATED", resources=[DEPLOYMENT_ACCOUNT_ID])
 
     return pipeline

src/lambda_codebase/initial_commit/bootstrap_repository/adf-bootstrap/deployment/lambda_codebase/pipeline_management/create_repository.py

@@ -4,19 +4,23 @@
 """
 
 import os
+import json
 import boto3
 from repo import Repo
 
 from logger import configure_logger
 from cloudwatch import ADFMetrics
 from parameter_store import ParameterStore
+from events import ADFEvents
 
 
 CLOUDWATCH = boto3.client("cloudwatch")
 METRICS = ADFMetrics(CLOUDWATCH, "PIPELINE_MANAGEMENT/REPO")
 LOGGER = configure_logger(__name__)
 DEPLOYMENT_ACCOUNT_REGION = os.environ["AWS_REGION"]
 DEPLOYMENT_ACCOUNT_ID = os.environ["ACCOUNT_ID"]
+EVENTS = ADFEvents(boto3.client("events", region_name=os.getenv("ADF_EVENTBUS_REGION")), "PipelineManagement")
+
 
 
 def lambda_handler(pipeline, _):
@@ -52,5 +56,15 @@ def lambda_handler(pipeline, _):
             METRICS.put_metric_data(
                 {"MetricName": "CreateOrUpdate", "Value": 1, "Unit": "Count"}
             )
+            EVENTS.put_event(
+                detail=json.dumps({
+                "repository_account_id": code_account_id,
+                "stack_name": repo.stack_name
+                }),
+                detailType="REPOSITORY_CREATED_OR_UPDATED",
+                resources=[
+                    f'{code_account_id}:{pipeline.get("name")}'
+                    ]
+                )
 
     return pipeline

src/lambda_codebase/initial_commit/bootstrap_repository/adf-bootstrap/deployment/lambda_codebase/pipeline_management/generate_pipeline_inputs.py

@@ -13,8 +13,10 @@
 from sts import STS
 from logger import configure_logger
 from partition import get_partition
+from aws_xray_sdk.core import patch_all
 
 
+patch_all()
 LOGGER = configure_logger(__name__)
 DEPLOYMENT_ACCOUNT_REGION = os.environ["AWS_REGION"]
 DEPLOYMENT_ACCOUNT_ID = os.environ["ACCOUNT_ID"]

src/lambda_codebase/initial_commit/bootstrap_repository/adf-bootstrap/deployment/lambda_codebase/pipeline_management/identify_out_of_date_pipelines.py

@@ -14,8 +14,10 @@
 from logger import configure_logger
 from deployment_map import DeploymentMap
 from parameter_store import ParameterStore
+from aws_xray_sdk.core import patch_all
 
 
+patch_all()
 LOGGER = configure_logger(__name__)
 S3_BUCKET_NAME = os.environ["S3_BUCKET_NAME"]
 DEPLOYMENT_ACCOUNT_ID = os.environ["ACCOUNT_ID"]

src/lambda_codebase/initial_commit/bootstrap_repository/adf-bootstrap/deployment/lambda_codebase/pipeline_management/process_deployment_map.py

@@ -14,8 +14,10 @@
 import boto3
 from botocore.exceptions import ClientError
 from logger import configure_logger
+from aws_xray_sdk.core import patch_all
 
 
+patch_all()
 LOGGER = configure_logger(__name__)
 DEPLOYMENT_ACCOUNT_REGION = os.environ["AWS_REGION"]
 DEPLOYMENT_ACCOUNT_ID = os.environ["ACCOUNT_ID"]

src/lambda_codebase/initial_commit/bootstrap_repository/adf-bootstrap/deployment/lambda_codebase/pipeline_management/store_pipeline_definition.py

@@ -9,8 +9,10 @@
 import boto3
 
 from logger import configure_logger
+from aws_xray_sdk.core import patch_all
 
 
+patch_all()
 LOGGER = configure_logger(__name__)
 DEPLOYMENT_ACCOUNT_REGION = os.environ["AWS_REGION"]
 DEPLOYMENT_ACCOUNT_ID = os.environ["ACCOUNT_ID"]

src/lambda_codebase/initial_commit/bootstrap_repository/adf-bootstrap/deployment/pipeline_management.yml

@@ -27,6 +27,10 @@ Parameters:
     Type: String
     MinLength: "1"
 
+  RootAccountRegion:
+    Type: String
+    MinLength: "1"
+
   CodeBuildImage:
     Type: String
     MinLength: "1"
@@ -61,8 +65,19 @@ Globals:
     Tracing: Active
     Layers:
       - !Ref LambdaLayer
+      - !Ref PipelineManagementLayerVersion
 
 Resources:
+  PipelineManagementLayerVersion:
+    Type: "AWS::Serverless::LayerVersion"
+    Properties:
+      ContentUri: "../../adf-build/shared/"
+      CompatibleRuntimes:
+        - python3.9
+      Description: "Common dependencies for ADF Pipeline Management Functions"
+      LayerName: pipeline_management_layer
+    Metadata:
+      BuildMethod: python3.9
   ADFPipelineMangementLambdaBasePolicy:
     Type: "AWS::IAM::ManagedPolicy"
     Properties:
@@ -79,6 +94,10 @@ Resources:
               - "xray:PutTraceSegments"
               - "cloudwatch:PutMetricData"
             Resource: "*"
+          - Effect: Allow
+            Action:
+              - "events:PutEvents"
+            Resource: !Sub "arn:${AWS::Partition}:events:*:${RootAccountId}:event-bus/ADF-Event-Bus"
       Roles:
         - !Ref DeploymentMapProcessingLambdaRole
         - !Ref CreateOrUpdateRuleLambdaRole
@@ -823,6 +842,8 @@ Resources:
           ADF_LOG_LEVEL: !Ref ADFLogLevel
           PIPELINE_MANAGEMENT_STATE_MACHINE: !Sub "arn:${AWS::Partition}:states:${AWS::Region}:${AWS::AccountId}:stateMachine:ADFPipelineManagementStateMachine"
           ADF_ROLE_NAME: !Ref CrossAccountAccessRole
+          ADF_EVENTBUS_ARN: !Sub "arn:${AWS::Partition}:events:${RootAccountRegion}:${RootAccountId}:event-bus/ADF-Event-Bus"
+          ADF_EVENTBUS_REGION: !Ref RootAccountRegion
       FunctionName: DeploymentMapProcessorFunction
       Role: !GetAtt DeploymentMapProcessingLambdaRole.Arn
       Events:
@@ -861,6 +882,8 @@ Resources:
           ADF_LOG_LEVEL: !Ref ADFLogLevel
           ADF_ROLE_NAME: !Ref CrossAccountAccessRole
           S3_BUCKET_NAME: !Ref PipelineBucket
+          ADF_EVENTBUS_ARN: !Sub "arn:${AWS::Partition}:events:${RootAccountRegion}:${RootAccountId}:event-bus/ADF-Event-Bus"
+          ADF_EVENTBUS_REGION: !Ref RootAccountRegion
       FunctionName: ADFPipelineCreateOrUpdateRuleFunction
       Role: !GetAtt CreateOrUpdateRuleLambdaRole.Arn
 
@@ -877,6 +900,8 @@ Resources:
           ADF_LOG_LEVEL: !Ref ADFLogLevel
           ADF_ROLE_NAME: !Ref CrossAccountAccessRole
           S3_BUCKET_NAME: !Ref PipelineBucket
+          ADF_EVENTBUS_ARN: !Sub "arn:${AWS::Partition}:events:${RootAccountRegion}:${RootAccountId}:event-bus/ADF-Event-Bus"
+          ADF_EVENTBUS_REGION: !Ref RootAccountRegion
       FunctionName: ADFPipelineCreateRepositoryFunction
       Role: !GetAtt CreateRepositoryLambdaRole.Arn
 

src/lambda_codebase/initial_commit/bootstrap_repository/adf-build/shared/python/events.py

@@ -37,5 +37,5 @@ def put_event(self, detailType, detail, resources=[]): # pylint: disable=W0102
         }
         trace_id = os.getenv("_X_AMZN_TRACE_ID")
         if trace_id:
-            payload["TraceHeader"] = trace_id
+            payload["TraceHeader"] = trace_id.split(";")[0]
         self.events.put_events(Entries=[payload])

src/lambda_codebase/initial_commit/bootstrap_repository/adf-build/shared/requirements.txt

@@ -59,3 +59,4 @@ pyyaml>=5.4.1
 schema~=0.7.5
 tenacity==8.0.1
 urllib3~=1.26.12
+aws-xray-sdk==2.10.0