diff --git a/.github/workflows/deploy.yml b/.github/workflows/deploy.yml new file mode 100644 index 0000000..82e6b5d --- /dev/null +++ b/.github/workflows/deploy.yml @@ -0,0 +1,42 @@ +name: Deploy code to Lambda +on: push + +env: + AWS_ACCESS_KEY_ID: ${{ secrets.AWS_ACCESS_KEY_ID }} + AWS_SECRET_ACCESS_KEY: ${{ secrets.AWS_SECRET_ACCESS_KEY }} + REPO_NAME: ${{ github.event.repository.name }} + +jobs: + deploy-lambda: + runs-on: ubuntu-latest + steps: + - name: 'Checkout' + uses: actions/checkout@v2 + - name: 'Setup python 3.8' + uses: actions/setup-python@v1 + with: + python-version: 3.8 + - name: 'Zip it all' + run: | + mkdir -p zip_folder + cp -R src/* zip_folder + pip install -r requirements.txt --target=zip_folder + cd zip_folder + zip -r ../${REPO_NAME}.zip . + - name: 'Setup terraform' + uses: hashicorp/setup-terraform@v1 + with: + terraform_version: '>=0.13.0 <0.14.0' + - name: 'Terraform init' + working-directory: terraform + run: terraform init + - name: 'Terraform validate' + working-directory: terraform + run: terraform validate + - name: 'Terraform plan' + working-directory: terraform + run: | + terraform plan -out="${REPO_NAME}.tfplan" + - name: 'Terraform apply' + working-directory: terraform + run: terraform apply -auto-approve "${REPO_NAME}.tfplan" \ No newline at end of file diff --git a/.github/workflows/push.yml b/.github/workflows/push.yml new file mode 100644 index 0000000..b4bbd0a --- /dev/null +++ b/.github/workflows/push.yml @@ -0,0 +1,38 @@ +name: Deploy code to Lambda +on: push + +env: + AWS_ACCESS_KEY_ID: ${{ secrets.AWS_ACCESS_KEY_ID }} + AWS_SECRET_ACCESS_KEY: ${{ secrets.AWS_SECRET_ACCESS_KEY }} + REPO_NAME: ${{ github.event.repository.name }} + +jobs: + deploy-lambda: + runs-on: ubuntu-latest + steps: + - name: 'Checkout' + uses: actions/checkout@v2 + - name: 'Setup python 3.8' + uses: actions/setup-python@v1 + with: + python-version: 3.8 + - name: 'Zip it all' + run: | + mkdir -p zip_folder + cp -R src/* zip_folder + pip install -r requirements.txt --target=zip_folder + cd zip_folder + zip -r ../${REPO_NAME}.zip . + - name: 'Setup terraform' + uses: hashicorp/setup-terraform@v1 + with: + terraform_version: '>=0.13.0 <0.14.0' + - name: 'Terraform init' + working-directory: terraform + run: terraform init + - name: 'Terraform validate' + working-directory: terraform + run: terraform validate + - name: Lint Terraform + working-directory: terraform + uses: actionshub/terraform-lint@master diff --git a/.gitignore b/.gitignore index be50c1e..5035770 100644 --- a/.gitignore +++ b/.gitignore @@ -20,3 +20,18 @@ s3transfer-0.1.9.dist-info botocore-1.4.68.dist-info docutils-0.12.dist-info .idea +DS_Store + +### Terraform ### +# Local .terraform directories +**/.terraform/* + +# .tfstate files +*.tfstate +*.tfstate.* + +# Crash log files +crash.log + +zip_folder +*.zip diff --git a/LICENSE b/LICENSE deleted file mode 100644 index ba625fa..0000000 --- a/LICENSE +++ /dev/null @@ -1,10 +0,0 @@ -Copyright 2016 Amazon.com, Inc. or its affiliates. All Rights Reserved. - -Licensed under the Apache License, Version 2.0 (the "License"). You may not use this file except in compliance with -the License. A copy of the License is located at - - http://aws.amazon.com/apache2.0/ - -or in the "license" file accompanying this file. This file is distributed on an "AS IS" BASIS, WITHOUT WARRANTIES OR -CONDITIONS OF ANY KIND, either express or implied. See the License for the specific language governing permissions and -limitations under the License. \ No newline at end of file diff --git a/NOTICE b/NOTICE deleted file mode 100644 index 93f0586..0000000 --- a/NOTICE +++ /dev/null @@ -1,2 +0,0 @@ -ECR_CLEANUP -Copyright 2016 Amazon.com, Inc. or its affiliates. All Rights Reserved. \ No newline at end of file diff --git a/README.md b/README.md index 9d22478..b284c30 100644 --- a/README.md +++ b/README.md @@ -1,65 +1,40 @@ # Automated Image Cleanup for Amazon ECR -The Python script and Lambda function described here help clean up images in [Amazon ECR](https://aws.amazon.com/ecr). The script looks for images that are not used in running [Amazon ECS](https://aws.amazon.com/ecs) tasks that can be deleted. You can configure the script to print the image list first to confirm deletions, specify a region, or specify a number of images to keep for potential rollbacks. +The Python script and Lambda function described here help clean up images in [Amazon ECR](https://aws.amazon.com/ecr). +- The script looks for images that are not used in running [Amazon ECS](https://aws.amazon.com/ecs) tasks that can be deleted. +- Only looks images in repos specified in GitHubActions env var ECR_REPOS_LIFECYCLE ex: ECR_REPOS_LIFECYCLE='apache, nginx' +- You can configure the script to print the image list first to confirm deletions, specify a region, or specify a number of images to keep for potential rollbacks with DRYRUN=true +- You can skip images followin a REGEX in their tag. -## Authenticate with AWS -[Configuring the AWS Command Line Interface.](http://docs.aws.amazon.com/cli/latest/userguide/cli-chap-getting-started.html) - -## Use virtualenv for Python execution - -To prevent any problems with your system Python version conflicting with the application, we recommend using virtualenv. +## Usage in local Install Python: `pip install python 3` -Install virtualenv: - - $ pip install virtualenv - $ virtualenv -p PATH_TO_YOUR_PYTHON_3 cloudformtion - $ virtualenv ~/.virtualenvs/cloudformtion - $ source ~/.virtualenvs/cloudformtion/bin/activate - -## Generate the Lambda package +Getting info about local env: + `which python3` -1. CD to the folder that contains main.py. -1. Run the following command: -`pip install -r requirements.txt -t `pwd`` -1. Compress the contents of folder (not the folder). - -## Upload the package to Lambda - -1. Run the following command: -`aws lambda create-function --function-name {NAME_OF_FUNCTION} --runtime python2.7 ---role {ARN_NUMBER} --handler main.handler --timeout 15 ---zip-file fileb://{ZIP_FILE_PATH}` - -## Send the package update to Lambda +Install virtualenv: -1. Run the following command: + pip install virtualenv + virtualenv --python=/usr/local/bin/python3 ~/venv-ecr-cleanup-lambda + source ~/venv-ecr-cleanup-lambda/bin/activate - `aws lambda update-function-code --function-name {NAME_OF_FUNCTION} --zip-file fileb://{ZIP_FILE_PATH}` - - -## Examples -Prints the images that are not used by running tasks and which are older than the last 100 versions, in all regions: - -`python main.py` - - -Deletes the images that are not used by running tasks and which are older than the last 100 versions, in all regions: - -`python main.py –dryrun False` - - -Deletes the images that are not used by running tasks and which are older than the last 20 versions (in each repository), in all regions: +## Install dependencies -`python main.py –dryrun False –imagestokeep 20` +`pip install -r requirements.txt` +## Example -Deletes the images that are not used by running tasks and which are older than the last 20 versions (in each repository), in Oregon only: +Deletes the images that: +- are not used by ECS running tasks . +- are older than the last 20 versions (in each repository). +- in Oregon only. +- ignore image tags that contains `release` or `archive`. +- skip images with tags that contain alpine and apache. -`python main.py –dryrun False –imagestokeep 20 –region us-west-2` +`python main.py –dryrun False –imagestokeep 20 –region us-west-2 -ignoretagsregex 'release|archive' -ecr_repos_lifecycle 'alpine,apache'` -Deletes the images that are not used by running tasks and which are older than the last 20 versions (in each repository), in Oregon only, and ignore image tags that contains `release` or `archive`: -`python main.py –dryrun False –imagestokeep 20 –region us-west-2 -ignoretagsregex release|archive` +## Credits +Fork of awslabs/ecr-cleanup-lambda adding functionality of ECR_REPOS_LIFECYCLE var. \ No newline at end of file diff --git a/lambda-cloudformation.yaml b/lambda-cloudformation.yaml deleted file mode 100644 index d2c39eb..0000000 --- a/lambda-cloudformation.yaml +++ /dev/null @@ -1,78 +0,0 @@ -# Copyright 2017 Amazon.com, Inc. or its affiliates. All Rights Reserved. -# Licensed under the Apache License, Version 2.0 (the "License"). You may not use this file except in compliance with -# the License. A copy of the License is located at -# http://aws.amazon.com/apache2.0/ -# or in the "license" file accompanying this file. This file is distributed on an "AS IS" BASIS, WITHOUT WARRANTIES OR -# CONDITIONS OF ANY KIND, either express or implied. See the License for the specific language governing permissions and -# limitations under the License. - -AWSTemplateFormatVersion: '2010-09-09' -Transform: AWS::Serverless-2016-10-31 -Description: automated cleanup of ECR repositories -Resources: - LambdaFunction: - Type: AWS::Serverless::Function - Properties: - Handler: main.handler - Runtime: python2.7 - Description: ECR Cleanup Lambda - CodeUri: ./ - MemorySize: 128 - Timeout: 300 - Role: !GetAtt LAMBDAROLE.Arn - Events: - Schedule: - Type: Schedule - Properties: - Schedule: cron(0 22 * * ? *) - Environment: - Variables: - REGION: None - DRYRUN: True - IMAGES_TO_KEEP: 100 - LAMBDAROLE: - Type: AWS::IAM::Role - Properties: - #RoleName: !Sub ${AWS::StackName}-Role-${AWS::Region} - AssumeRolePolicyDocument: - Version: 2012-10-17 - Statement: - - - Effect: Allow - Principal: - Service: - - lambda.amazonaws.com - Action: - - sts:AssumeRole - Path: / - LAMBDAPOLICY: - Type: AWS::IAM::Policy - Properties: - PolicyName: !Sub ${AWS::StackName}-Policy-${AWS::Region} - PolicyDocument: - Version: 2012-10-17 - Statement: - - - Effect: Allow - Action: - - logs:CreateLogGroup - - logs:CreateLogStream - - logs:PutLogEvents - Resource: arn:aws:logs:*:*:* - - - Effect: Allow - Action: - - ecr:BatchDeleteImage - - ecr:DescribeRepositories - - ecr:ListImages - - ecr:DescribeImages - - ecs:DescribeClusters - - ecs:DescribeTaskDefinition - - ecs:DescribeTasks - - ecs:ListClusters - - ecs:ListTaskDefinitions - - ecs:ListTasks - Resource: "*" - Roles: - - - !Ref LAMBDAROLE \ No newline at end of file diff --git a/main.py b/src/main.py similarity index 93% rename from main.py rename to src/main.py index 458a1e4..8920ef6 100644 --- a/main.py +++ b/src/main.py @@ -1,15 +1,3 @@ -''' -Copyright 2016 Amazon.com, Inc. or its affiliates. All Rights Reserved. - -Licensed under the Apache License, Version 2.0 (the "License"). You may not use this file except in compliance with -the License. A copy of the License is located at - - http://aws.amazon.com/apache2.0/ - -or in the "license" file accompanying this file. This file is distributed on an "AS IS" BASIS, WITHOUT WARRANTIES OR -CONDITIONS OF ANY KIND, either express or implied. See the License for the specific language governing permissions and -limitations under the License. -''' from __future__ import print_function import argparse @@ -21,6 +9,7 @@ DRYRUN = None IMAGES_TO_KEEP = None IGNORE_TAGS_REGEX = None +ECR_REPOS_LIFECYCLE = None def initialize(): @@ -28,6 +17,7 @@ def initialize(): global DRYRUN global IMAGES_TO_KEEP global IGNORE_TAGS_REGEX + global ECR_REPOS_LIFECYCLE REGION = os.environ.get('REGION', "None") DRYRUN = os.environ.get('DRYRUN', "false").lower() @@ -37,6 +27,8 @@ def initialize(): DRYRUN = True IMAGES_TO_KEEP = int(os.environ.get('IMAGES_TO_KEEP', 100)) IGNORE_TAGS_REGEX = os.environ.get('IGNORE_TAGS_REGEX', "^$") + ECR_REPOS_LIFECYCLE = os.environ.get('ECR_REPOS_LIFECYCLE', "None") + def handler(event, context): initialize() @@ -57,7 +49,8 @@ def discover_delete_images(regionname): describe_repo_paginator = ecr_client.get_paginator('describe_repositories') for response_listrepopaginator in describe_repo_paginator.paginate(): for repo in response_listrepopaginator['repositories']: - repositories.append(repo) + if repo['repositoryName'] in ECR_REPOS_LIFECYCLE.split(','): + repositories.append(repo) ecs_client = boto3.client('ecs', region_name=regionname) diff --git a/terraform/main.tf b/terraform/main.tf new file mode 100644 index 0000000..ec303a7 --- /dev/null +++ b/terraform/main.tf @@ -0,0 +1,92 @@ +resource "aws_iam_role_policy" "ecr-cleanup-lambda_role_policy" { + name = "ecr-cleanup-lambda" + role = aws_iam_role.ecr-cleanup-lambda_role.id + + policy = <<-EOF + { + "Version": "2012-10-17", + "Statement": [ + { + "Action": [ + "logs:CreateLogGroup", + "logs:CreateLogStream", + "logs:PutLogEvents" + ], + "Effect": "Allow", + "Resource": "arn:aws:logs:*:*:*" + }, + { + "Action": [ + "ecr:BatchDeleteImage", + "ecr:DescribeRepositories", + "ecr:ListImages", + "ecr:DescribeImages" + ], + "Effect": "Allow", + "Resource": "*" + }, + { + "Action": [ + "ecs:DescribeClusters", + "ecs:DescribeTaskDefinition", + "ecs:DescribeTasks", + "ecs:ListClusters", + "ecs:ListTaskDefinitions", + "ecs:ListTasks" + ], + "Effect": "Allow", + "Resource": "*" + } + ] + } + EOF +} + +resource "aws_iam_role" "ecr-cleanup-lambda_role" { + name = "ecr-cleanup-lambda" + + assume_role_policy = <<-EOF + { + "Version": "2012-10-17", + "Statement": [ + { + "Action": "sts:AssumeRole", + "Principal": { + "Service": "lambda.amazonaws.com" + }, + "Effect": "Allow", + "Sid": "" + } + ] + } + EOF +} + +module "lambda_function_existing_package_local" { + source = "terraform-aws-modules/lambda/aws" + version = "~> 1.0" + + function_name = var.NAME_OF_FUNCTION + description = "Lifecycle for ECR images" + handler = "main.handler" + runtime = "python3.8" + + lambda_role = aws_iam_role.ecr-cleanup-lambda_role.arn + create_package = false + create_role = false + local_existing_package = "../${var.NAME_OF_FUNCTION}.zip" + + environment_variables = { + IMAGES_TO_KEEP = tonumber(var.IMAGES_TO_KEEP), + IGNORE_TAGS_REGEX = var.IGNORE_TAGS_REGEX, + REGION = var.REGION + ECR_REPOS_LIFECYCLE = var.ECR_REPOS_LIFECYCLE + } + + tags ={ + Environment = "evil" + Terraform = "true" + } +} + + diff --git a/terraform/meta.tf b/terraform/meta.tf new file mode 100644 index 0000000..c2d1727 --- /dev/null +++ b/terraform/meta.tf @@ -0,0 +1,21 @@ +terraform { + required_version = ">= 0.13.0, < 0.14.0" + + required_providers { + aws = { + version = "~> 3.14.1" + source = "hashicorp/aws" + } + } + + backend "s3" { + bucket = "jportasa-terraform-remote-state" + key = "evil/eu-west-1/ecr-cleanup-lambda/terraform.tfstate" + region = "eu-west-1" + encrypt = true + } +} + +provider "aws" { + region = "eu-west-1" +} \ No newline at end of file diff --git a/terraform/vars.tf b/terraform/vars.tf new file mode 100644 index 0000000..5f57a56 --- /dev/null +++ b/terraform/vars.tf @@ -0,0 +1,21 @@ +variable "DRYRUN" { + default = true +} + +variable "IMAGES_TO_KEEP" { + default = 50 +} + +variable "IGNORE_TAGS_REGEX" { + default = "release|archive" +} + +variable "NAME_OF_FUNCTION" { + default = "ecr-cleanup-lambda" +} + +variable "REGION" {} + +variable "ECR_REPOS_LIFECYCLE" { + default = "alpine,apache" +} \ No newline at end of file