diff --git a/.github/scripts/eksctl-install.sh b/.github/scripts/eksctl-install.sh new file mode 100644 index 0000000000..5f18168ffa --- /dev/null +++ b/.github/scripts/eksctl-install.sh @@ -0,0 +1,13 @@ +#!/bin/bash + +ARCH=amd64 +PLATFORM=$(uname -s)_$ARCH + +curl -sLO "https://github.com/eksctl-io/eksctl/releases/latest/download/eksctl_$PLATFORM.tar.gz" + +# (Optional) Verify checksum +curl -sL "https://github.com/eksctl-io/eksctl/releases/latest/download/eksctl_checksums.txt" | grep $PLATFORM | sha256sum --check + +tar -xzf eksctl_$PLATFORM.tar.gz -C /tmp && rm eksctl_$PLATFORM.tar.gz + +sudo mv /tmp/eksctl /usr/local/bin diff --git a/.github/scripts/gpg-install.sh b/.github/scripts/gpg-install.sh new file mode 100644 index 0000000000..cdeb5c27d3 --- /dev/null +++ b/.github/scripts/gpg-install.sh @@ -0,0 +1,3 @@ +#!/bin/bash + +sudo apt-get install gnupg \ No newline at end of file diff --git a/.github/workflows/cv-infrastructure.yml b/.github/workflows/cv-infrastructure.yml new file mode 100644 index 0000000000..da5ee170df --- /dev/null +++ b/.github/workflows/cv-infrastructure.yml @@ -0,0 +1,128 @@ +name: Create AWS infrastructure +on: + push: + branches: + - infrastructure +jobs: + setup_roles_and_policies: + if: false + runs-on: ubuntu-latest + environment: + name: develop + env: + AWS_ACCESS_KEY_ID: ${{ secrets.AWS_ACCESS_KEY_ID }} + AWS_SECRET_ACCESS_KEY: ${{ secrets.AWS_SECRET_ACCESS_KEY }} + CLUSTER_ROLE_NAME: cv-eks-cluster-role + CLUSTER_POLICY_NAME: cv-eks-cluster-policy + NODEGROUP_ROLE_NAME: cv-eks-nodegroup-role + NODEGROUP_POLICY_NAME: cv-eks-nodegroup-policy + CODEPIPELINE_ROLE_NAME: cv-codepipeline-role + CODEPIPELINE_POLICY_NAME: cv-codepipeline-policy + CODEBUILD_ROLE_NAME: cv-codebuild-role + CODEBUILD_POLICY_NAME: cv-codebuild-policy + steps: + - name: Checkout code + uses: actions/checkout@v3.5.3 + + - name: Setup AWS credentials + run: | + aws configure set aws_access_key_id $AWS_ACCESS_KEY_ID} + aws configure set aws_secret_access_key $AWS_SECRET_ACCESS_KEY} + aws configure set default.region us-west-1 + + - name: Creates the role that will assume the trust policy to deal with the cluster + run: aws iam create-role --role-name ${CLUSTER_ROLE_NAME} --assume-role-policy-document file://${GITHUB_WORKSPACE}/eksk8s/eks-cluster-trust-policy.json + + - name: Attaches the policy to the cluster role + run: aws iam put-role-policy --role-name ${CLUSTER_ROLE_NAME} --policy-name ${CLUSTER_POLICY_NAME} --policy-document file://${GITHUB_WORKSPACE}/eksk8s/eks-cluster-policy.json + + - name: Creates the role that will assume the trust policy to deal with the node-group + run: aws iam create-role --role-name ${NODEGROUP_ROLE_NAME} --assume-role-policy-document file://${GITHUB_WORKSPACE}/eksk8s/eks-nodegroup-trust-policy.json + + - name: Attaches the policy for the node-group role + run: aws iam put-role-policy --role-name ${NODEGROUP_ROLE_NAME} --policy-name ${NODEGROUP_POLICY_NAME} --policy-document file://${GITHUB_WORKSPACE}/eksk8s/eks-nodegroup-policy.json + + - name: Create the role that will assume the trust policy to deal with the codepipeline service + run: aws iam create-role --role-name ${CODEPIPELINE_ROLE_NAME} --assume-role-policy-document file://${GITHUB_WORKSPACE}/eksk8s/codepipeline-trust-policy.json + + - name: Attaches the policy for the codepipeline role + run: aws iam put-role-policy --role-name ${CODEPIPELINE_ROLE_NAME} --policy-name ${CODEPIPELINE_POLICY_NAME} --policy-document file://${GITHUB_WORKSPACE}/eksk8s/codepipeline-policy.json + + - name: Create the role that will assume the trust policy to deal with the codebuild service + run: aws iam create-role --role-name ${CODEBUILD_ROLE_NAME} --assume-role-policy-document file://${GITHUB_WORKSPACE}/eksk8s/codebuild-trust-policy.json + + - name: Attaches the policy for the codebuild role + run: aws iam put-role-policy --role-name ${CODEBUILD_ROLE_NAME} --policy-name ${CODEBUILD_POLICY_NAME} --policy-document file://${GITHUB_WORKSPACE}/eksk8s/codebuild-policy.json + + create_the_cluster: + runs-on: ubuntu-latest + # needs: setup_roles_and_policies + environment: + name: develop + env: + AWS_ACCESS_KEY_ID: ${{ secrets.AWS_ACCESS_KEY_ID }} + AWS_SECRET_ACCESS_KEY: ${{ secrets.AWS_SECRET_ACCESS_KEY }} + SSH_PRIVATE_KEY: ${{ secrets.SSH_PRIVATE_KEY }} + SSH_PUBLIC_KEY: ${{ secrets.SSH_PUBLIC_KEY }} + PASSPHRASE: ${{ secrets.PASSPHRASE }} + GPG_PUBLIC_KEY: ${{ secrets.GPG_PUBLIC_KEY }} + GPG_PRIVATE_KEY: ${{ secrets.GPG_PRIVATE_KEY }} + GPG_TRUST_OWNERS: ${{ secrets.GPG_TRUST_OWNERS }} + steps: + - name: Checkout code + uses: actions/checkout@v3.5.3 + + - name: Setup the runner + run: | + sh ${GITHUB_WORKSPACE}/.github/scripts/eksctl-install.sh + sh ${GITHUB_WORKSPACE}/.github/scripts/gpg-install.sh + + - name: Setup the keys used to create the k8s cluster + if: false + run: | + mkdir ${HOME}/.ssh/ + echo ${SSH_PRIVATE_KEY} > ${HOME}/.ssh/k8s + echo ${SSH_PUBLIC_KEY} > ${HOME}/.ssh/k8s.pub + + - name: Creates the k8s cluster using eksctl + if: false + run: eksctl create cluster -f ${GITHUB_WORKSPACE}/eksk8s/cluster-config.yaml + + - name: Import the gpg components + run: | + echo ${GPG_TRUST_OWNERS} | base64 --decode > trust-file + gpg --import-ownertrust < trust-file + echo ${GPG_PUBLIC_KEY} | base64 --decode | gpg --import + echo ${GPG_PRIVATE_KEY} | base64 --decode | gpg --import --batch + + - name: Creates fake kubeconfig file + run: | + mkdir ${HOME}/.kube/ + cat << EOF > ${HOME}/.kube/config + Line1 + EOF + + - name: Encrypts the kubeconfig file + run: gpg --encrypt -r runner@mygithub.com --output kube_config.gpg ${HOME}/.kube/config + + - name: Pushes kubeconfig artifact + uses: actions/upload-artifact@v3.1.2 + with: + name: kube_config.gpg + path: kube_config.gpg + + manages_aws_secrets: + runs-on: ubuntu-latest + needs: create_the_cluster + if: false + environment: + name: develop + env: + AWS_ACCESS_KEY_ID: ${{ secrets.AWS_ACCESS_KEY_ID }} + AWS_SECRET_ACCESS_KEY: ${{ secrets.AWS_SECRET_ACCESS_KEY }} + GH_TOKEN: ${{ secrets.GH_TOKEN }} + CODEPIPELINE_ROLE_NAME: cv-codepipeline-role + CODEPIPELINE_POLICY_NAME: cv-codepipeline-policy + steps: + - name: Stores the GH_TOKEN in AWS Systems Manager Parameter Store + run: aws ssm put-parameter --name "MySecureStringParameter" --value "MySuperSecretValue" --type "SecureString" --key-id "arn:aws:kms:us-east-1:123456789012:key/12345678-1234-1234-1234-123456789012" diff --git a/.github/workflows/manual.yml b/.github/workflows/manual.yml index 27d15f879d..b3f1779920 100644 --- a/.github/workflows/manual.yml +++ b/.github/workflows/manual.yml @@ -25,22 +25,3 @@ jobs: JIRA_BASE_URL: ${{ secrets.JIRA_BASE_URL }} JIRA_USER_EMAIL: ${{ secrets.JIRA_USER_EMAIL }} JIRA_API_TOKEN: ${{ secrets.JIRA_API_TOKEN }} - - - name: Create NEW JIRA ticket - id: create - uses: atlassian/gajira-create@master - with: - project: CONUPDATE - issuetype: Task - summary: | - Github PR - nd0044 - Full Stack Nanodegree C4 | Repo: ${{ github.repository }} | PR# ${{github.event.number}} - description: | - Repo link: https://github.com/${{ github.repository }} - PR no. ${{ github.event.pull_request.number }} - PR title: ${{ github.event.pull_request.title }} - PR description: ${{ github.event.pull_request.description }} - In addition, please resolve other issues, if any. - fields: '{"components": [{"name":"nd0044 - Full Stack Nanodegree"}], "customfield_16449":"https://classroom.udacity.com/nanodegrees/nd0044/dashboard/overview", "customfield_16450":"Resolve the PR", "labels": ["github"], "priority":{"id": "4"}}' - - - name: Log created issue - run: echo "Issue ${{ steps.create.outputs.issue }} was created" diff --git a/.github/workflows/secrets.yml b/.github/workflows/secrets.yml new file mode 100644 index 0000000000..d4b70c18bd --- /dev/null +++ b/.github/workflows/secrets.yml @@ -0,0 +1,41 @@ +name: Generate KMS (Key Management System) and store the ARN + +on: + push: + branches: + - secrets + +jobs: + generate_kms_key: + runs-on: ubuntu-latest + environment: + name: develop + env: + AWS_ACCESS_KEY_ID: ${{ secrets.AWS_ACCESS_KEY_ID }} + AWS_SECRET_ACCESS_KEY: ${{ secrets.AWS_SECRET_ACCESS_KEY }} + CLUSTER_ROLE_NAME: cv-eks-cluster-role + CLUSTER_POLICY_NAME: cv-eks-cluster-policy + NODEGROUP_ROLE_NAME: cv-eks-nodegroup-role + NODEGROUP_POLICY_NAME: cv-eks-nodegroup-policy + CODEPIPELINE_ROLE_NAME: cv-codepipeline-role + CODEPIPELINE_POLICY_NAME: cv-codepipeline-policy + CODEBUILD_ROLE_NAME: cv-codebuild-role + CODEBUILD_POLICY_NAME: cv-codebuild-policy + KVM_KEY_ARN: "" + steps: + - name: Configurar AWS CLI + uses: aws-actions/configure-aws-credentials@v3 + with: + aws-access-key-id: ${{ secrets.AWS_ACCESS_KEY_ID }} + aws-secret-access-key: ${{ secrets.AWS_SECRET_ACCESS_KEY }} + aws-region: us-east-1 + + - name: Generate KMS key and save the KVM key in GH secrets + run: | + KVM_KEY_ID=$(aws kms create-key --query KeyMetadata.KeyId --output text) + echo ${KVM_KEY_ID} + KVM_KEY_ARN=$(aws kms describe-key --key-id $KVM_KEY_ID --query KeyMetadata.Arn --output text) + echo ${KVM_KEY_ARN} >> secrets.KVM_KEY_ARN + + - name: Shows the ARN just created + run: echo $KVM_KEY_ARN diff --git a/.gitignore b/.gitignore index 9500e81546..87d03a0ad5 100644 --- a/.gitignore +++ b/.gitignore @@ -28,4 +28,3 @@ htmlcov/ .envrc .direnv -.github/** diff --git a/eksk8s/README.MD b/eksk8s/README.MD new file mode 100644 index 0000000000..b371a761bd --- /dev/null +++ b/eksk8s/README.MD @@ -0,0 +1,29 @@ +With regard to the file eks-cluster-policy.json: + +{ + "Version": "2012-10-17", + "Statement": [ + { + "Effect": "Allow", + "Action": [ + "eks:CreateCluster", + "eks:DescribeCluster", + "eks:UpdateClusterConfig", + "eks:UpdateClusterVersion", + "ssm:GetParameters" + ], + "Resource": "*" // * means that the already defined actions could be applied to any EKS cluster in the account + } + ] +} + +In the file eksk8s/cluster-config.yaml the key nodeGroups.iam.instanceRoleARN has the value=arn:aws:iam::62132xxxx821:role/cv-eks-nodegroup-role. In a real project this file should be created using jinja/cookicutter in a previous step. This file should never have hard-coded values. However, for educational purposes/save time these values are hardcoded in the file. + +This key pair was created in aws console. It is possible to do it using the command to create it. This step involves pushing (uploading) the keys to aws via aws cli or somehow. + +The subnets were also created by hand in the aws console. + +The command `eksctl create cluster -f ` creates the file `$HOME/.kube/config` This file is the one that has the context to connect to the remote cluster. Then to store this file securely, the file should be encrypted and published in the artifacts list of the github repo. Then, in a linux terminal generate the gpg key with the command `gpg --full-generate-key`. For educational purposes I have not included a passphrase nor expiration date for this gpg key set. After, get the ID of the just created `gpg` with the command: `gpg --list-keys` Then, using the commands: `gpg --export -a KEY_ID | base64 -w 0 > public.asc` and `gpg --export-secret-keys -a KEY_ID | base64 -w 0 > private.asc` export the gpg keys that will be stored as secrets in GitHub actions. After, in the workflow import the gpg key to encrypt the file and then upload as artifact to GitHub actions repo. + +#### Note: +For reference on how to deal with base64 and gpg keys use this link: https://dev.to/epam_india_python/how-we-set-up-gpg-keys-as-environment-variables-2f26 diff --git a/eksk8s/cluster-config.yaml b/eksk8s/cluster-config.yaml new file mode 100644 index 0000000000..050c0e9263 --- /dev/null +++ b/eksk8s/cluster-config.yaml @@ -0,0 +1,31 @@ +apiVersion: eksctl.io/v1alpha5 +kind: ClusterConfig + +metadata: + name: cv-eks-cluster + region: us-west-1 + +vpc: + id: vpc-04e8935eb17607398 + subnets: + public: + us-west-1a: + id: subnet-08375f54b10ac81a2 + us-west-1b: + id: subnet-0a8eea0bb2b8bd056 + clusterEndpoints: + publicAccess: true + securityGroup: sg-074fc7db95c8a8141 + +nodeGroups: + - name: cv-eks-nodegroup + iam: + instanceRoleARN: arn:aws:iam::621328949821:role/cv-eks-nodegroup-role + instanceType: t2.micro + desiredCapacity: 2 + ssh: + allow: true + publicKeyPath: ~/.ssh/k8s.pub + subnets: + - subnet-08375f54b10ac81a2 + - subnet-0a8eea0bb2b8bd056 diff --git a/eksk8s/codebuild-policy.json b/eksk8s/codebuild-policy.json new file mode 100644 index 0000000000..8e6155cd2c --- /dev/null +++ b/eksk8s/codebuild-policy.json @@ -0,0 +1,18 @@ +{ + "Version": "2012-10-17", + "Statement": [ + { + "Effect": "Allow", + "Action": [ + "codebuild:StartBuild", + "eks:Describe*", + "ssm:GetParameters", + "logs:CreateLogGroup", + "logs:CreateLogStream", + "logs:PutLogEvents" + ], + "Resource": "*" + } + ] + } + \ No newline at end of file diff --git a/eksk8s/codebuild-trust-policy.json b/eksk8s/codebuild-trust-policy.json new file mode 100644 index 0000000000..5d8f864928 --- /dev/null +++ b/eksk8s/codebuild-trust-policy.json @@ -0,0 +1,13 @@ +{ + "Version": "2012-10-17", + "Statement": [ + { + "Effect": "Allow", + "Principal": { + "Service": "codebuild.amazonaws.com" + }, + "Action": "sts:AssumeRole" + } + ] + } + \ No newline at end of file diff --git a/eksk8s/codepipeline-policy.json b/eksk8s/codepipeline-policy.json new file mode 100644 index 0000000000..0cc48f219d --- /dev/null +++ b/eksk8s/codepipeline-policy.json @@ -0,0 +1,15 @@ +{ + "Version": "2012-10-17", + "Statement": [ + { + "Effect": "Allow", + "Action": [ + "codebuild:StartBuild", + "codebuild:BatchGetBuilds", + "cloudformation:CreateStack" + ], + "Resource": "*" + } + ] + } + \ No newline at end of file diff --git a/eksk8s/codepipeline-trust-policy.json b/eksk8s/codepipeline-trust-policy.json new file mode 100644 index 0000000000..e297ab6bc6 --- /dev/null +++ b/eksk8s/codepipeline-trust-policy.json @@ -0,0 +1,13 @@ +{ + "Version": "2012-10-17", + "Statement": [ + { + "Effect": "Allow", + "Principal": { + "Service": "codepipeline.amazonaws.com" + }, + "Action": "sts:AssumeRole" + } + ] + } + \ No newline at end of file diff --git a/eksk8s/eks-cluster-policy.json b/eksk8s/eks-cluster-policy.json new file mode 100644 index 0000000000..b6d34dc2fc --- /dev/null +++ b/eksk8s/eks-cluster-policy.json @@ -0,0 +1,17 @@ +{ + "Version": "2012-10-17", + "Statement": [ + { + "Effect": "Allow", + "Action": [ + "eks:CreateCluster", + "eks:DescribeCluster", + "eks:UpdateClusterConfig", + "eks:UpdateClusterVersion", + "ssm:GetParameters" + ], + "Resource": "*" + } + ] +} + diff --git a/eksk8s/eks-cluster-trust-policy.json b/eksk8s/eks-cluster-trust-policy.json new file mode 100644 index 0000000000..fd952e62a2 --- /dev/null +++ b/eksk8s/eks-cluster-trust-policy.json @@ -0,0 +1,13 @@ +{ + "Version": "2012-10-17", + "Statement": [ + { + "Effect": "Allow", + "Principal": { + "Service": "eks.amazonaws.com" + }, + "Action": "sts:AssumeRole" + } + ] +} + diff --git a/eksk8s/eks-nodegroup-policy.json b/eksk8s/eks-nodegroup-policy.json new file mode 100644 index 0000000000..f0c52c1e56 --- /dev/null +++ b/eksk8s/eks-nodegroup-policy.json @@ -0,0 +1,70 @@ +{ + "Version": "2012-10-17", + "Statement": [ + { + "Effect": "Allow", + "Action": [ + "ec2:UnassignPrivateIpAddresses", + "ec2:AssignPrivateIpAddresses", + "ecr:GetAuthorizationToken", + "ec2:AuthorizeSecurityGroupIngress", + "ec2:DescribeInstances", + "ec2:AssociateRouteTable", + "ec2:DeleteVolume", + "logs:CreateLogStream", + "eks:DescribeNodegroup", + "autoscaling:DescribeAutoScalingGroups", + "ec2:CreateRoute", + "ec2:DescribeVolumes", + "ec2:DescribeRouteTables", + "ecr:BatchCheckLayerAvailability", + "ec2:DetachVolume", + "ec2:ModifyVolume", + "eks:ListNodegroups", + "ec2:CreateTags", + "autoscaling:DescribeTags", + "ecr:GetDownloadUrlForLayer", + "ec2:CreateRouteTable", + "ec2:RunInstances", + "ec2:DisassociateRouteTable", + "logs:CreateLogGroup", + "ec2:CreateVolume", + "ecr:PutImage", + "eks:DeleteNodegroup", + "ecr:BatchGetImage", + "eks:UpdateNodegroupConfig", + "ec2:DescribeSubnets", + "ecr:InitiateLayerUpload", + "ec2:AttachVolume", + "ec2:DeleteTags", + "logs:DescribeLogStreams", + "autoscaling:DescribeLaunchConfigurations", + "ecr:UploadLayerPart", + "ecr:ListImages", + "iam:PassRole", + "ec2:CreateNetworkInterface", + "ec2:DescribeNetworkInterfaces", + "ec2:DeleteNetworkInterface", + "ec2:CreateSecurityGroup", + "ec2:DetachNetworkInterface", + "ec2:ModifyNetworkInterfaceAttribute", + "ec2:AttachNetworkInterface", + "ecr:CompleteLayerUpload", + "ecr:DescribeRepositories", + "ec2:ModifyInstanceAttribute", + "ec2:RebootInstances", + "ec2:TerminateInstances", + "logs:DescribeLogGroups", + "ec2:DeleteRoute", + "eks:CreateNodegroup", + "logs:PutLogEvents", + "ec2:DescribeSecurityGroups", + "ec2:DeleteSecurityGroup", + "ecr:GetRepositoryPolicy" + + ], + "Resource": "*" + } + ] +} + diff --git a/eksk8s/eks-nodegroup-trust-policy.json b/eksk8s/eks-nodegroup-trust-policy.json new file mode 100644 index 0000000000..418fd594ff --- /dev/null +++ b/eksk8s/eks-nodegroup-trust-policy.json @@ -0,0 +1,20 @@ +{ + "Version": "2012-10-17", + "Statement": [ + { + "Effect": "Allow", + "Principal": { + "Service": "ec2.amazonaws.com" + }, + "Action": "sts:AssumeRole" + }, + { + "Effect": "Allow", + "Principal": { + "Service": "eks.amazonaws.com" + }, + "Action": "sts:AssumeRole" + } + ] +} +