Fix missing IAM permissions for Fluent Bit

The service account for Fluent Bit currently does not have permission to
create the log streams with the new log stream template name. This means
that non-system logs won't be sent to Cloudwatch.

This updates the IAM policy to allow creating these log groups, set
retention policies, and put log events within the Flightdeck namespace.

Author: jferris

.github/workflows/tests.yaml

@@ -43,6 +43,7 @@ jobs:
       run: |
         CLUSTER=$(echo 'flightdeck-${{ github.ref_name }}' | cut -c1-20)
         CLUSTER="$CLUSTER-sandbox-v1"
+        echo "CLUSTER=$CLUSTER" >> "$GITHUB_ENV"
         aws \
           --region us-east-1 \
           eks \
@@ -64,4 +65,6 @@ jobs:
 
     - name: Run tests
       run: |
-        make tests ADDRESS=https://${{ github.ref_name }}.flightdeck-test.thoughtbot.com
+        make tests \
+          ADDRESS=https://${{ github.ref_name }}.flightdeck-test.thoughtbot.com \
+          CLUSTER="$CLUSTER"

aws/platform/main.tf

@@ -370,6 +370,7 @@ locals {
         [OUTPUT]
             Name cloudwatch_logs
             Match *
+            auto_create_group true
             region ${data.aws_region.current.name}
             log_group_name ${module.cloudwatch_logs.log_group_name}
             log_group_template ${var.logs_prefix}/$kubernetes['namespace_name']

aws/platform/modules/cloudwatch-logs/main.tf

@@ -27,36 +27,17 @@ resource "aws_iam_role_policy_attachment" "this" {
 
 data "aws_iam_policy_document" "this" {
   statement {
-    sid = "AllowCreateLogEvents"
+    sid = "AllowWriteLogs"
     actions = [
-      "logs:DescribeLogStreams",
-      "logs:PutLogEvents"
-    ]
-    resources = [
-      "${aws_cloudwatch_log_group.this.arn}:log-stream:*"
-    ]
-  }
-
-  statement {
-    sid = "AllowCreateLogGroup"
-    actions = [
-      "logs:CreateLogGroup"
+      "logs:CreateLogGroup",
+      "logs:CreateLogStream",
+      "logs:PutLogEvents",
+      "logs:PutRetentionPolicy",
     ]
     resources = [
       "${local.arn_prefix}:log-group:${var.log_group_prefix}/*"
     ]
   }
-
-  statement {
-    sid = "AllowCreateLogStream"
-    actions = [
-      "logs:CreateLogStream"
-    ]
-    resources = [
-      aws_cloudwatch_log_group.this.arn,
-      "${aws_cloudwatch_log_group.this.arn}:log-stream:*"
-    ]
-  }
 }
 
 data "aws_caller_identity" "current" {}

tests/fluentbit.bats

@@ -29,3 +29,33 @@
     false
   fi
 }
+
+@test "creates log streams within groups for Kubernetes namespaces" {
+  expected="$RANDOM"
+  curl -v "$ADDRESS/echo?log=$expected"
+  pod=$(kubectl \
+    get pod \
+    --field-selector=status.phase=Running \
+    --selector=app=echoserver \
+    -n acceptance \
+    --output=name \
+    | cut -d'/' -f2)
+  logs=$(aws \
+    --region us-east-1 \
+    logs \
+    get-log-events \
+    --log-group-name "/flightdeck/acceptance" \
+    --log-stream-name "$pod.echoserver" \
+    --query 'events[*].[message]' \
+      --output text)
+
+  if ! echo "$logs" | grep -q "log=$expected"; then
+    echo "Failed to find log for test request." >&2
+    echo >&2
+    echo "Test request was: GET /echo?log=$expected" >&2
+    echo >&2
+    echo "Found log entries" >&2
+    echo "$logs" >&2
+    false
+  fi
+}