chore!: Deprecate python3.6 runtime

baolsen · baolsen · commit c8378f089b52 · 2023-02-13T10:45:01.000+02:00
diff --git a/.github/DISABLED.pre-commit-config.yaml b/.github/DISABLED.pre-commit-config.yaml
diff --git a/.github/workflows/check-deployments.yml b/.github/workflows/check-deployments.yml
@@ -52,19 +52,20 @@ jobs:
       - run: |
           NAME=${{ matrix.name }}
           RUNTIME=${{ matrix.runtime }}
+          DEPLOYMENT=deployment-$NAME-$RUNTIME.zip
           #
           echo NAME=$NAME RUNTIME=$RUNTIME
           #
-          cp $NAME-$RUNTIME.zip $NAME-$RUNTIME.zip_original
+          cp $DEPLOYMENT ${DEPLOYMENT}_original
           cd $NAME
-          zip -r ../$NAME-$RUNTIME.zip .
+          zip -r ../$DEPLOYMENT .
           cd ..
           #
           # Compare length and names of files in Zip.
           # Building in Docker doesn't work, some files are still different.
-          [[ -f $NAME-$RUNTIME.zip ]] || { echo "Deployment file not found."; exit 1; }
+          [[ -f $DEPLOYMENT ]] || { echo "Deployment file not found."; exit 1; }
           diff \
-            <(unzip -vqq $NAME-$RUNTIME.zip  | awk '{$2=""; $3=""; $4=""; $5=""; $6=""; print}' | sort -k3 -f) \
-            <(unzip -vqq $NAME-$RUNTIME.zip_original  | awk '{$2=""; $3=""; $4=""; $5=""; $6=""; print}' | sort -k3 -f)
+            <(unzip -vqq $DEPLOYMENT  | awk '{$2=""; $3=""; $4=""; $5=""; $6=""; print}' | sort -k3 -f) \
+            <(unzip -vqq ${DEPLOYMENT}_original  | awk '{$2=""; $3=""; $4=""; $5=""; $6=""; print}' | sort -k3 -f)
           FILES_CHANGED=$?
           echo FILES_CHANGED=$FILES_CHANGED
diff --git a/.github/workflows/pre-commit-and-tests.yml b/.github/workflows/pre-commit-and-tests.yml
@@ -59,7 +59,7 @@ jobs:
         if: failure()
         run: |
           LOG_FILE=~/.cache/pre-commit/pre-commit.log
-          [ -f $LOG_FILE ] || cat $LOG_FILE
+          [ -f $LOG_FILE ] && cat $LOG_FILE
 
   #--------------------------------------------------------------
   # TESTS
diff --git a/.github/workflows/release.yml b/.github/workflows/release.yml
@@ -65,4 +65,4 @@ jobs:
           [[ -z "$TAG_NAME" ]] && { echo "TAG_NAME is empty" ; exit 1; }
           #
           gh config set prompt disabled
-          gh release upload $TAG_NAME *.zip
+          gh release upload $TAG_NAME deployment-*.zip
diff --git a/.tfsec-config.yml b/.tfsec-config.yml
@@ -14,5 +14,7 @@
 
 ---
 exclude:
-# Add IDs of checks to ignore them
-#  - aws-sns-topic-encryption-use-cmk
+  # Add IDs of checks to ignore them
+  #  - aws-sns-topic-encryption-use-cmk
+  # Without full tracing enabled it is difficult to trace the flow of logs
+  - aws-lambda-enable-tracing
diff --git a/README.md b/README.md
@@ -50,7 +50,7 @@ Full contributing [guidelines are covered here](.github/contributing.md).
 | <a name="input_included_accounts"></a> [included\_accounts](#input\_included\_accounts) | List of accounts that be scanned to manual actions. If empty will scan all accounts. | `list(string)` | `[]` | no |
 | <a name="input_included_users"></a> [included\_users](#input\_included\_users) | List of emails that be scanned to manual actions. If empty will scan all emails. | `list(string)` | `[]` | no |
 | <a name="input_lambda_memory_size"></a> [lambda\_memory\_size](#input\_lambda\_memory\_size) | The amount of memory for Lambda to use | `number` | `"128"` | no |
-| <a name="input_lambda_runtime"></a> [lambda\_runtime](#input\_lambda\_runtime) | The lambda runtime to use | `string` | `"python3.8"` | no |
+| <a name="input_lambda_runtime"></a> [lambda\_runtime](#input\_lambda\_runtime) | The lambda runtime to use. One of: `["python3.9", "python3.8", "python3.7"]` | `string` | `"python3.8"` | no |
 | <a name="input_log_retention_in_days"></a> [log\_retention\_in\_days](#input\_log\_retention\_in\_days) | Number of days to keep CloudWatch logs | `number` | `14` | no |
 | <a name="input_message_format"></a> [message\_format](#input\_message\_format) | Where do you want to send this message? slack or msteams | `string` | `"slack"` | no |
 | <a name="input_naming_prefix"></a> [naming\_prefix](#input\_naming\_prefix) | Resources will be prefixed with this | `string` | `"clickops-notifier"` | no |
diff --git a/build_one.sh b/build_one.sh
@@ -11,15 +11,16 @@ NAME=$1
 LAMBDA_RUNTIME=$2
 echo "*** NAME=$NAME  LAMBDA_RUNTIME=$LAMBDA_RUNTIME "
 
-DEPLOYMENT_FILE=$NAME-$LAMBDA_RUNTIME.zip
+DEPLOYMENT_FILE=deployment-$NAME-$LAMBDA_RUNTIME.zip
+
+[[ -d $DEPLOYMENT_FILE ]] && rm -f $DEPLOYMENT_FILE
 
 cd $NAME
 
 [[ -d .package ]] && rm -rf .package
-[[ -d $DEPLOYMENT_FILE ]] && rm -f $DEPLOYMENT_FILE
 
 # Build requirements in Lambda environment
-docker run -v "$PWD":/var/task "public.ecr.aws/sam/build-$LAMBDA_RUNTIME" /bin/sh -c "pip install -r requirements.txt -t .package; exit"
+docker run --platform linux/amd64 -v "$PWD":/var/task "public.ecr.aws/sam/build-$LAMBDA_RUNTIME" /bin/sh -c "pip install -r requirements.txt -t .package; exit"
 
 # Create deployment package
 echo "*** Adding requirements to $DEPLOYMENT_FILE"
diff --git a/build_targets.json b/build_targets.json
@@ -1,9 +1,5 @@
 {
   "include": [
-    {
-      "name": "clickopsnotifier",
-      "runtime": "python3.6"
-    },
     {
       "name": "clickopsnotifier",
       "runtime": "python3.7"
diff --git a/clickopsnotifier-python3.6.zip b/clickopsnotifier-python3.6.zip
diff --git a/clickopsnotifier/app.py b/clickopsnotifier/app.py
@@ -7,27 +7,31 @@
 import base64
 import os
 from typing import Tuple
+import logging
 
 from clickops import ClickOpsEventChecker, CloudTrailEvent
 from messenger import Messenger
 from delivery_stream import DeliveryStream
 
-s3 = boto3.client("s3")
-ssm = boto3.client("ssm")
+logger = logging.getLogger("clickopsnotifier")
+LOG_LEVEL = os.environ.get("LOG_LEVEL", "INFO")
+logger.setLevel(LOG_LEVEL)
 
-WEBHOOK_PARAMETER = os.environ["WEBHOOK_PARAMETER"]
-EXCLUDED_ACCOUNTS = json.loads(os.environ["EXCLUDED_ACCOUNTS"])
-INCLUDED_ACCOUNTS = json.loads(os.environ["INCLUDED_ACCOUNTS"])
-EXCLUDED_USERS = json.loads(os.environ["EXCLUDED_USERS"])
-INCLUDED_USERS = json.loads(os.environ["INCLUDED_USERS"])
-EXCLUDED_SCOPED_ACTIONS = json.loads(os.environ["EXCLUDED_SCOPED_ACTIONS"])
-MESSAGE_FORMAT = os.environ["MESSAGE_FORMAT"]
-LOG_LEVEL = os.environ["LOG_LEVEL"]
+WEBHOOK_PARAMETER = os.environ.get("WEBHOOK_PARAMETER", "")
+EXCLUDED_ACCOUNTS = json.loads(os.environ.get("EXCLUDED_ACCOUNTS", "[]"))
+INCLUDED_ACCOUNTS = json.loads(os.environ.get("INCLUDED_ACCOUNTS", "[]"))
+EXCLUDED_USERS = json.loads(os.environ.get("EXCLUDED_USERS", "[]"))
+INCLUDED_USERS = json.loads(os.environ.get("INCLUDED_USERS", "[]"))
+EXCLUDED_SCOPED_ACTIONS = json.loads(os.environ.get("EXCLUDED_SCOPED_ACTIONS", "[]"))
+MESSAGE_FORMAT = os.environ.get("MESSAGE_FORMAT", "slack")
 
 FIREHOSE_DELIVERY_STREAM_NAME = os.environ.get("FIREHOSE_DELIVERY_STREAM_NAME")
 if FIREHOSE_DELIVERY_STREAM_NAME == "__NONE__":
     FIREHOSE_DELIVERY_STREAM_NAME = None
 
+s3 = boto3.client("s3")
+ssm = boto3.client("ssm")
+
 WEBHOOK_URL = None
 
 
@@ -91,6 +95,9 @@ def handler_organizational(event, context) -> None:  # noqa: C901
     :return: None
     """
 
+    if event is None:
+        raise KeyError("event is None")
+
     webhook_url = get_webhook()
 
     messenger = Messenger(format=MESSAGE_FORMAT, webhook=webhook_url)
@@ -104,7 +111,6 @@ def handler_organizational(event, context) -> None:  # noqa: C901
         records = s3_events.get("Records", [])
 
         for record in records:
-
             # Get the object from the event and show its content type
             bucket = record["s3"]["bucket"]["name"]
             key = urllib.parse.unquote_plus(
@@ -127,7 +133,6 @@ def handler_organizational(event, context) -> None:  # noqa: C901
                 event_json = json.load(fh)
 
                 for event in event_json["Records"]:
-
                     event_origin = f"{bucket}/{key}"
 
                     success = success and __handle_event(
@@ -146,6 +151,8 @@ def handler_organizational(event, context) -> None:  # noqa: C901
 
 
 def handler_standalone(event, context) -> None:
+    if event is None:
+        raise KeyError("event is None")
 
     webhook_url = get_webhook()
 
@@ -161,7 +168,6 @@ def handler_standalone(event, context) -> None:
 
     success = True
     for e in event_json["logEvents"]:
-
         event_origin = (
             event_json["logGroup"]
             + ":"
diff --git a/deployment-clickopsnotifier-python3.7.zip b/deployment-clickopsnotifier-python3.7.zip
diff --git a/deployment-clickopsnotifier-python3.8.zip b/deployment-clickopsnotifier-python3.8.zip
diff --git a/deployment-clickopsnotifier-python3.9.zip b/deployment-clickopsnotifier-python3.9.zip
diff --git a/deployment_organization.tf b/deployment_organization.tf
@@ -10,6 +10,8 @@ resource "aws_sqs_queue" "bucket_notifications" {
 
   name                       = var.naming_prefix
   visibility_timeout_seconds = var.event_processing_timeout + 5
+
+  sqs_managed_sse_enabled = true
 }
 
 data "aws_iam_policy_document" "bucket_notifications" {
diff --git a/examples/delivery_stream/README.md b/examples/delivery_stream/README.md
@@ -168,12 +168,25 @@ resource "aws_kinesis_firehose_delivery_stream" "extended_s3_stream" {
 }
 
 resource "aws_s3_bucket" "firehose" {
+  #tfsec:ignore:aws-s3-encryption-customer-key
+  #tfsec:ignore:aws-s3-enable-bucket-logging
+  #tfsec:ignore:aws-s3-enable-versioning
   bucket = local.naming_prefix_firehose
 }
 
-resource "aws_s3_bucket_acl" "firehose_bucket_acl" {
-  bucket = aws_s3_bucket.firehose.id
-  acl    = "private"
+resource "aws_s3_bucket_server_side_encryption_configuration" "firehose" {
+  bucket = aws_s3_bucket.firehose.bucket
+
+  rule {
+    apply_server_side_encryption_by_default {
+      sse_algorithm = "AES256"
+    }
+  }
+}
+
+resource "aws_s3_bucket_public_access_block" "firehose" {
+  bucket            = aws_s3_bucket.firehose.id
+  block_public_acls = true
 }
 
 resource "aws_iam_role" "firehose" {
@@ -201,6 +214,7 @@ resource "aws_iam_role_policy" "firehose" {
 }
 
 data "aws_iam_policy_document" "firehose" {
+  #tfsec:ignore:aws-iam-no-policy-wildcards
   statement {
     actions = [
       "s3:AbortMultipartUpload",
@@ -269,7 +283,8 @@ No outputs.
 | [aws_iam_role_policy.firehose](https://registry.terraform.io/providers/hashicorp/aws/latest/docs/resources/iam_role_policy) | resource |
 | [aws_kinesis_firehose_delivery_stream.extended_s3_stream](https://registry.terraform.io/providers/hashicorp/aws/latest/docs/resources/kinesis_firehose_delivery_stream) | resource |
 | [aws_s3_bucket.firehose](https://registry.terraform.io/providers/hashicorp/aws/latest/docs/resources/s3_bucket) | resource |
-| [aws_s3_bucket_acl.firehose_bucket_acl](https://registry.terraform.io/providers/hashicorp/aws/latest/docs/resources/s3_bucket_acl) | resource |
+| [aws_s3_bucket_public_access_block.firehose](https://registry.terraform.io/providers/hashicorp/aws/latest/docs/resources/s3_bucket_public_access_block) | resource |
+| [aws_s3_bucket_server_side_encryption_configuration.firehose](https://registry.terraform.io/providers/hashicorp/aws/latest/docs/resources/s3_bucket_server_side_encryption_configuration) | resource |
 | [random_pet.run_id](https://registry.terraform.io/providers/hashicorp/random/latest/docs/resources/pet) | resource |
 | [aws_iam_policy_document.firehose](https://registry.terraform.io/providers/hashicorp/aws/latest/docs/data-sources/iam_policy_document) | data source |
 
diff --git a/examples/delivery_stream/main.tf b/examples/delivery_stream/main.tf
@@ -164,12 +164,25 @@ resource "aws_kinesis_firehose_delivery_stream" "extended_s3_stream" {
 }
 
 resource "aws_s3_bucket" "firehose" {
+  #tfsec:ignore:aws-s3-encryption-customer-key
+  #tfsec:ignore:aws-s3-enable-bucket-logging
+  #tfsec:ignore:aws-s3-enable-versioning
   bucket = local.naming_prefix_firehose
 }
 
-resource "aws_s3_bucket_acl" "firehose_bucket_acl" {
-  bucket = aws_s3_bucket.firehose.id
-  acl    = "private"
+resource "aws_s3_bucket_server_side_encryption_configuration" "firehose" {
+  bucket = aws_s3_bucket.firehose.bucket
+
+  rule {
+    apply_server_side_encryption_by_default {
+      sse_algorithm = "AES256"
+    }
+  }
+}
+
+resource "aws_s3_bucket_public_access_block" "firehose" {
+  bucket            = aws_s3_bucket.firehose.id
+  block_public_acls = true
 }
 
 resource "aws_iam_role" "firehose" {
@@ -197,6 +210,7 @@ resource "aws_iam_role_policy" "firehose" {
 }
 
 data "aws_iam_policy_document" "firehose" {
+  #tfsec:ignore:aws-iam-no-policy-wildcards
   statement {
     actions = [
       "s3:AbortMultipartUpload",
diff --git a/main.tf b/main.tf
@@ -69,11 +69,9 @@ data "aws_iam_policy_document" "lambda_permissions" {
 }
 
 locals {
-
-  deployment_filename = "clickopsnotifier-${var.lambda_runtime}.zip"
+  deployment_filename = "deployment-clickopsnotifier-${var.lambda_runtime}.zip"
   deployment_path     = "${path.module}/${local.deployment_filename}"
-
-  s3_key = coalesce(var.s3_key, join("/", [var.naming_prefix, local.deployment_filename]))
+  s3_key              = coalesce(var.s3_key, join("/", [var.naming_prefix, local.deployment_filename]))
 }
 
 resource "aws_s3_object" "deployment" {
@@ -92,10 +90,10 @@ module "clickops_notifier_lambda" {
   function_name = var.naming_prefix
   description   = "ClickOps Notifier Lambda"
 
-  handler = var.standalone ? "app.handler_standalone" : "app.handler_organizational"
-  runtime = var.lambda_runtime
-
-  # publish = true
+  handler     = var.standalone ? "app.handler_standalone" : "app.handler_organizational"
+  runtime     = var.lambda_runtime
+  timeout     = var.event_processing_timeout
+  memory_size = var.lambda_memory_size
 
   # Where should we get the package from?
   create_package         = false
@@ -109,9 +107,15 @@ module "clickops_notifier_lambda" {
     }
   )
 
-  timeout     = var.event_processing_timeout
-  memory_size = var.lambda_memory_size
+  # Publish creation/changes as a new Lambda Function Version
+  publish = true
 
+  create_lambda_function_url = false
+
+  # Logs
+  cloudwatch_logs_retention_in_days = var.log_retention_in_days
+
+  # IAM
   create_role = var.create_iam_role
   lambda_role = var.iam_role_arn
 
@@ -121,8 +125,6 @@ module "clickops_notifier_lambda" {
   attach_policy_statements = length(var.additional_iam_policy_statements) > 0
   policy_statements        = var.additional_iam_policy_statements
 
-  cloudwatch_logs_retention_in_days = var.log_retention_in_days
-
   environment_variables = {
     WEBHOOK_PARAMETER = aws_ssm_parameter.slack_webhook.name
 
@@ -158,6 +160,10 @@ module "clickops_notifier_lambda" {
   }
 
   tags = var.tags
+
+  depends_on = [
+    aws_s3_object.deployment
+  ]
 }
 
 resource "aws_ssm_parameter" "slack_webhook" {
diff --git a/tests/test_examples.py b/tests/test_examples.py
@@ -1,6 +1,6 @@
 import pytest
 
-from conftest import terraform_apply, terraform_plan
+from tests.conftest import terraform_apply, terraform_plan
 
 
 @pytest.mark.slow
diff --git a/tests/test_lambda_handler.py b/tests/test_lambda_handler.py
@@ -1,17 +1,19 @@
-from clickops import app
+import pytest
+
+from clickopsnotifier import app
 
 
 def test_lambda_handler_standalone():
     """
     This test ensures that the lambda handler_standalone function is callable.
     """
-    test_event = None
-    app.handler_standalone(test_event, None)
+    with pytest.raises(KeyError):
+        app.handler_standalone(None, None)
 
 
 def test_lambda_handler_organizational():
     """
     This test ensures that the lambda handler_organizational function is callable.
     """
-    test_event = None
-    app.handler_organizational(test_event, None)
+    with pytest.raises(KeyError):
+        app.handler_organizational(None, None)
diff --git a/variables.tf b/variables.tf
@@ -134,15 +134,14 @@ variable "log_retention_in_days" {
 
 variable "lambda_runtime" {
   type        = string
-  description = "The lambda runtime to use"
+  description = "The lambda runtime to use. One of: `[\"python3.9\", \"python3.8\", \"python3.7\"]`"
   default     = "python3.8"
 
   validation {
     condition = contains([
       "python3.9",
       "python3.8",
-      "python3.7",
-      "python3.6"
+      "python3.7"
     ], var.lambda_runtime)
     error_message = "Invalid lambda_runtime provided."
   }

Original file line number	Diff line number	Diff line change
`@@ -65,4 +65,4 @@ jobs:`
`65`	`65`	`[[ -z "$TAG_NAME" ]] && { echo "TAG_NAME is empty" ; exit 1; }`
`66`	`66`	`#`
`67`	`67`	`gh config set prompt disabled`
`68`		`- gh release upload $TAG_NAME *.zip`
	`68`	`+ gh release upload $TAG_NAME deployment-*.zip`
Original file line number	Diff line number	Diff line change
`@@ -1,9 +1,5 @@`
`1`	`1`	`{`
`2`	`2`	`"include": [`
`3`		`- {`
`4`		`- "name": "clickopsnotifier",`
`5`		`- "runtime": "python3.6"`
`6`		`- },`
`7`	`3`	`{`
`8`	`4`	`"name": "clickopsnotifier",`
`9`	`5`	`"runtime": "python3.7"`