From 47df7fdd2a033aebe9887f167114536afae6203c Mon Sep 17 00:00:00 2001 From: Edgar Costa Date: Tue, 4 Jun 2024 11:32:07 -0300 Subject: [PATCH] Adds e2e workflow with automated resource cleanup (#197) * adding the e2e test for bootstrap with terraform * fixing versions comment --- .github/scripts/e2e-delete-lbs.py | 33 +++++++ .github/scripts/e2e-delete-sgs.py | 45 +++++++++ .github/workflows/e2e-parallel-full.yml | 120 ++++++++++++++++++++++++ bootstrap/terraform/versions.tf | 2 +- docs/_partials/destroy.md | 1 + 5 files changed, 200 insertions(+), 1 deletion(-) create mode 100644 .github/scripts/e2e-delete-lbs.py create mode 100644 .github/scripts/e2e-delete-sgs.py create mode 100644 .github/workflows/e2e-parallel-full.yml diff --git a/.github/scripts/e2e-delete-lbs.py b/.github/scripts/e2e-delete-lbs.py new file mode 100644 index 00000000..eb3eed8a --- /dev/null +++ b/.github/scripts/e2e-delete-lbs.py @@ -0,0 +1,33 @@ +import os +import boto3 + +REGION = os.environ.get('AWS_DEFAULT_REGION', 'us-east-1') +ELB_CLIENT = boto3.client('elbv2', region_name=REGION) + +def delete_target_groups(target_group_arns): + for tg_arn in target_group_arns: + ELB_CLIENT.delete_target_group(TargetGroupArn=tg_arn) + +def delete_listeners(listener_arns): + for listener_arn in listener_arns: + ELB_CLIENT.delete_listener(ListenerArn=listener_arn) + +def delete_load_balancers(): + response = ELB_CLIENT.describe_load_balancers() + + for lb in response['LoadBalancers']: + lb_arn = lb['LoadBalancerArn'] + listeners = ELB_CLIENT.describe_listeners(LoadBalancerArn=lb_arn) + listener_arns = [listener['ListenerArn'] for listener in listeners['Listeners']] + + delete_listeners(listener_arns) + + target_groups = ELB_CLIENT.describe_target_groups(LoadBalancerArn=lb_arn) + target_group_arns = [tg['TargetGroupArn'] for tg in target_groups['TargetGroups']] + + delete_target_groups(target_group_arns) + + ELB_CLIENT.delete_load_balancer(LoadBalancerArn=lb_arn) + +if __name__ == '__main__': + delete_load_balancers() diff --git a/.github/scripts/e2e-delete-sgs.py b/.github/scripts/e2e-delete-sgs.py new file mode 100644 index 00000000..e8b2bf01 --- /dev/null +++ b/.github/scripts/e2e-delete-sgs.py @@ -0,0 +1,45 @@ +import os +import boto3 + +REGION = os.environ.get('AWS_DEFAULT_REGION', 'us-east-1') +EC2_CLIENT = boto3.client('ec2', region_name=REGION) + +def remove_security_group_rules(security_group_id): + try: + sg_details = EC2_CLIENT.describe_security_groups(GroupIds=[security_group_id]) + sg = sg_details['SecurityGroups'][0] + + if sg['IpPermissions']: + EC2_CLIENT.revoke_security_group_ingress( + GroupId=security_group_id, + IpPermissions=sg['IpPermissions'] + ) + + if sg['IpPermissionsEgress']: + EC2_CLIENT.revoke_security_group_egress( + GroupId=security_group_id, + IpPermissions=sg['IpPermissionsEgress'] + ) + except Exception as e: + print(f"Error removing rules from {security_group_id}: {str(e)}") + +def delete_all_security_groups(): + try: + response = EC2_CLIENT.describe_security_groups() + for sg in response['SecurityGroups']: + # Skip deleting default security groups or any critical system security group + if sg['GroupName'] == 'default' or 'default' in sg['GroupName']: + print(f"Skipping default security group: {sg['GroupId']} ({sg['GroupName']})") + continue + + try: + remove_security_group_rules(sg['GroupId']) + EC2_CLIENT.delete_security_group(GroupId=sg['GroupId']) + print(f"Deleted security group: {sg['GroupId']}") + except Exception as e: + print(f"Failed to delete {sg['GroupId']}: {str(e)}") + except Exception as e: + print(f"Failed to process security groups: {str(e)}") + +if __name__ == '__main__': + delete_all_security_groups() diff --git a/.github/workflows/e2e-parallel-full.yml b/.github/workflows/e2e-parallel-full.yml new file mode 100644 index 00000000..b16d25e1 --- /dev/null +++ b/.github/workflows/e2e-parallel-full.yml @@ -0,0 +1,120 @@ +name: e2e-parallel-full + +on: + workflow_dispatch: + inputs: + TFDestroy: + description: 'Destroy TF Automatically (false/true) - Default: true' + required: true + default: 'true' + +concurrency: e2e-parallel-full + +env: + BUCKET_NAME: terraform-crossplane-on-eks-github-actions-state + +permissions: + contents: read + +jobs: + prereq-cleanup: + name: Prerequisite Cleanup + runs-on: ubuntu-latest + # These permissions are needed to interact with GitHub's OIDC Token endpoint. + permissions: + id-token: write + contents: read + steps: + - name: Harden Runner + uses: step-security/harden-runner@v2 + with: + egress-policy: audit + + - name: Checkout + uses: actions/checkout@a5ac7e51b41094c92402da3b24376905380afc29 + + - name: Auth AWS + uses: aws-actions/configure-aws-credentials@v4.0.2 + with: + role-to-assume: ${{ secrets.ROLE_TO_ASSUME }} + aws-region: us-east-1 + role-duration-seconds: 3600 + role-session-name: GithubActions-Session + + - name: Ensure load balancers and sgs are removed + run: | + pip3 install boto3 + python3 .github/scripts/e2e-delete-sgs.py + python3 .github/scripts/e2e-delete-lbs.py + + deploy: + name: Run e2e test + runs-on: ubuntu-latest + needs: prereq-cleanup + + # These permissions are needed to interact with GitHub's OIDC Token endpoint. + permissions: + id-token: write + contents: read + strategy: + fail-fast: false + matrix: + include: + - example_path: bootstrap/terraform + # - example_path: bootstrap/terraform-fully-private + steps: + - name: Harden Runner + uses: step-security/harden-runner@v2 + with: + egress-policy: audit + + - name: Checkout + uses: actions/checkout@a5ac7e51b41094c92402da3b24376905380afc29 + + - name: Setup backend + # Un-comment remote backend for use in workflow + run: sed -i "s/# //g" ${{ matrix.example_path }}/versions.tf + + - name: Auth AWS + uses: aws-actions/configure-aws-credentials@v4.0.2 + with: + role-to-assume: ${{ secrets.ROLE_TO_ASSUME }} + aws-region: us-east-1 + role-duration-seconds: 3600 + role-session-name: GithubActions-Session + + - name: Setup Terraform + uses: hashicorp/setup-terraform@v3 + with: + terraform_version: 1.8.4 + + - name: Terraform Apply + id: apply + working-directory: ${{ matrix.example_path }} + run: | + terraform init -upgrade=true + terraform apply -target="module.vpc" -no-color -input=false -auto-approve + terraform apply -target="module.eks" -no-color -input=false -auto-approve + terraform apply -target="module.eks_blueprints_addons" -no-color -input=false -auto-approve + terraform apply -target="module.crossplane" -no-color -input=false -auto-approve + terraform apply -target="module.gatekeeper" -no-color -input=false -auto-approve + terraform apply -no-color -input=false -auto-approve + + - name: Terraform Destroy + if: github.event.inputs.TFDestroy == 'true' && (steps.apply.outcome == 'success' || steps.apply.outcome == 'failure') + working-directory: ${{ matrix.example_path }} + run: | + terraform init -upgrade=true + terraform destroy -target="module.crossplane" -no-color -auto-approve + terraform destroy -target="module.gatekeeper" -no-color -auto-approve + terraform destroy -target="module.eks_blueprints_addons" -no-color -auto-approve + terraform destroy -target="module.eks" -no-color -auto-approve + terraform destroy -target="module.vpc" -no-color -auto-approve + terraform destroy -no-color -auto-approve + + - name: Fail if TF apply failed + if: steps.apply.outcome == 'failure' + run: | + echo "Terraform Apply step failed...Please check the logs of the Terraform Apply step." + echo "Failing the job to avoid false positives." + exit 1 \ No newline at end of file diff --git a/bootstrap/terraform/versions.tf b/bootstrap/terraform/versions.tf index b82daba3..10952f9e 100644 --- a/bootstrap/terraform/versions.tf +++ b/bootstrap/terraform/versions.tf @@ -23,7 +23,7 @@ terraform { } } - # ## Used for end-to-end testing on project; update to suit your needs + # ## Used for end-to-end testing on project; update to suit your needs # backend "s3" { # bucket = "terraform-crossplane-on-eks-github-actions-state" # region = "us-east-1" diff --git a/docs/_partials/destroy.md b/docs/_partials/destroy.md index 7b95aafb..f44a3491 100644 --- a/docs/_partials/destroy.md +++ b/docs/_partials/destroy.md @@ -1,5 +1,6 @@ ```sh terraform destroy -target="module.crossplane" -auto-approve +terraform destroy -target="module.gatekeeper" -auto-approve terraform destroy -target="module.eks_blueprints_addons" -auto-approve terraform destroy -target="module.eks" -auto-approve terraform destroy -target="module.vpc" -auto-approve