diff --git a/test/cloudwatchlogs/publish_logs_test.go b/test/cloudwatchlogs/publish_logs_test.go index 7a5ca8831..383acfd7e 100644 --- a/test/cloudwatchlogs/publish_logs_test.go +++ b/test/cloudwatchlogs/publish_logs_test.go @@ -6,6 +6,7 @@ package cloudwatchlogs import ( + "context" "fmt" "log" "os" @@ -15,7 +16,11 @@ import ( "testing" "time" + "github.com/aws/aws-sdk-go-v2/aws" "github.com/aws/aws-sdk-go-v2/service/cloudwatchlogs/types" + "github.com/aws/aws-sdk-go-v2/service/ec2" + ec2Types "github.com/aws/aws-sdk-go-v2/service/ec2/types" + "github.com/google/uuid" "github.com/stretchr/testify/assert" "github.com/aws/amazon-cloudwatch-agent-test/environment" @@ -29,9 +34,18 @@ const ( logLineId2 = "bar" logFilePath = "/tmp/cwagent_log_test.log" // TODO: not sure how well this will work on Windows sleepForFlush = 20 * time.Second // default flush interval is 5 seconds + sleepForExtendedFlush = 180 * time.Second // increase flush time for the two main tests + retryWaitTime = 30 * time.Second configPathAutoRemoval = "resources/config_auto_removal.json" standardLogGroupClass = "STANDARD" infrequentAccessLogGroupClass = "INFREQUENT_ACCESS" + + entityType = "@entity.KeyAttributes.Type" + entityName = "@entity.KeyAttributes.Name" + entityEnvironment = "@entity.KeyAttributes.Environment" + entityPlatform = "@entity.Attributes.PlatformType" + entityInstanceId = "@entity.Attributes.EC2.InstanceId" + queryString = "fields @message, @entity.KeyAttributes.Type, @entity.KeyAttributes.Name, @entity.KeyAttributes.Environment, @entity.Attributes.PlatformType, @entity.Attributes.EC2.InstanceId" ) var ( @@ -70,6 +84,7 @@ var ( logGroupClass: types.LogGroupClassInfrequentAccess, }, } + resourceNotFoundException *types.ResourceNotFoundException ) type writeToCloudWatchTestInput struct { @@ -119,9 +134,9 @@ func TestWriteLogsToCloudWatch(t *testing.T) { // ensure that there is enough time from the "start" time and the first log line, // so we don't miss it in the GetLogEvents call - time.Sleep(sleepForFlush) + time.Sleep(sleepForExtendedFlush) writeLogLines(t, f, param.iterations) - time.Sleep(sleepForFlush) + time.Sleep(sleepForExtendedFlush) common.StopAgent() end := time.Now() @@ -139,6 +154,128 @@ func TestWriteLogsToCloudWatch(t *testing.T) { } } +// TestWriteLogsWithEntityInfo writes logs and validates that the +// log events are associated with entities from CloudWatch Logs +func TestWriteLogsWithEntityInfo(t *testing.T) { + instanceId := awsservice.GetInstanceId() + log.Printf("Found instance id %s", instanceId) + + // Define tags to create for EC2 test case + tagsToCreate := []ec2Types.Tag{ + { + Key: aws.String("service"), + Value: aws.String("service-test"), + }, + } + + testCases := map[string]struct { + agentConfigPath string + iterations int + useEC2Tag bool + expectedEntity common.ExpectedEntity + }{ + "IAMRole": { + agentConfigPath: filepath.Join("resources", "config_log.json"), + iterations: 1000, + expectedEntity: common.ExpectedEntity{ + EntityType: "Service", + Name: "cwa-e2e-iam-role", //should match the name of the IAM role used in our testing + Environment: "ec2:default", + PlatformType: "AWS::EC2", + InstanceId: instanceId, + }, + }, + "ServiceInConfig": { + agentConfigPath: filepath.Join("resources", "config_log_service_name.json"), + iterations: 1000, + expectedEntity: common.ExpectedEntity{ + EntityType: "Service", + Name: "service-in-config", //should match the service.name value in the config file + Environment: "environment-in-config", //should match the deployment.environment value in the config file + PlatformType: "AWS::EC2", + InstanceId: instanceId, + }, + }, + "EC2Tags": { + agentConfigPath: filepath.Join("resources", "config_log.json"), + iterations: 1000, + useEC2Tag: true, + expectedEntity: common.ExpectedEntity{ + EntityType: "Service", + Name: "service-test", //should match the value in tagsToCreate + Environment: "ec2:default", + PlatformType: "AWS::EC2", + InstanceId: instanceId, + }, + }, + } + for name, testCase := range testCases { + t.Run(name, func(t *testing.T) { + t.Cleanup(func() { + // delete the log group/stream after each test case + awsservice.DeleteLogGroupAndStream(instanceId, instanceId) + + // delete EC2 tags added to the instance for the test + if testCase.useEC2Tag { + input := &ec2.DeleteTagsInput{ + Resources: []string{instanceId}, + Tags: tagsToCreate, + } + _, err := awsservice.Ec2Client.DeleteTags(context.TODO(), input) + assert.NoError(t, err) + // Add a short delay to ensure tag deletion propagates + time.Sleep(5 * time.Second) + } + }) + if testCase.useEC2Tag { + // enable instance metadata tags + modifyInput := &ec2.ModifyInstanceMetadataOptionsInput{ + InstanceId: aws.String(instanceId), + InstanceMetadataTags: ec2Types.InstanceMetadataTagsStateEnabled, + } + _, modifyErr := awsservice.Ec2Client.ModifyInstanceMetadataOptions(context.TODO(), modifyInput) + assert.NoError(t, modifyErr) + + input := &ec2.CreateTagsInput{ + Resources: []string{instanceId}, + Tags: tagsToCreate, + } + _, createErr := awsservice.Ec2Client.CreateTags(context.TODO(), input) + assert.NoError(t, createErr) + } + id := uuid.New() + f, err := os.Create(logFilePath + "-" + id.String()) + if err != nil { + t.Fatalf("Error occurred creating log file for writing: %v", err) + } + + // Defer file closing and removal with error handling + defer func() { + if err := f.Close(); err != nil { + t.Errorf("Error occurred closing log file: %v", err) + } + if err := os.Remove(logFilePath + "-" + id.String()); err != nil { + t.Errorf("Error occurred removing log file: %v", err) + } + }() + + common.DeleteFile(common.AgentLogFile) + common.TouchFile(common.AgentLogFile) + + common.CopyFile(testCase.agentConfigPath, configOutputPath) + + common.StartAgent(configOutputPath, true, false) + time.Sleep(sleepForExtendedFlush) + writeLogLines(t, f, testCase.iterations) + time.Sleep(sleepForExtendedFlush) + common.StopAgent() + end := time.Now() + + common.ValidateLogEntity(t, instanceId, instanceId, &end, queryString, testCase.expectedEntity, "EC2") + }) + } +} + // TestAutoRemovalStopAgent configures agent to monitor a file with auto removal on. // Then it restarts the agent. // Verify the file is NOT removed. diff --git a/test/cloudwatchlogs/resources/config_log.json b/test/cloudwatchlogs/resources/config_log.json index 0cb8fbb4f..37b963b94 100644 --- a/test/cloudwatchlogs/resources/config_log.json +++ b/test/cloudwatchlogs/resources/config_log.json @@ -8,7 +8,7 @@ "files": { "collect_list": [ { - "file_path": "/tmp/cwagent_log_test.log", + "file_path": "/tmp/cwagent_log_test.log*", "log_group_name": "{instance_id}", "log_stream_name": "{instance_id}", "timezone": "UTC" diff --git a/test/cloudwatchlogs/resources/config_log_service_name.json b/test/cloudwatchlogs/resources/config_log_service_name.json new file mode 100644 index 000000000..0354796a6 --- /dev/null +++ b/test/cloudwatchlogs/resources/config_log_service_name.json @@ -0,0 +1,22 @@ +{ + "agent": { + "run_as_user": "root", + "debug": true + }, + "logs": { + "logs_collected": { + "files": { + "collect_list": [ + { + "file_path": "/tmp/cwagent_log_test.log*", + "log_group_name": "{instance_id}", + "log_stream_name": "{instance_id}", + "timezone": "UTC", + "service.name": "service-in-config", + "deployment.environment": "environment-in-config" + } + ] + } + } + } +} diff --git a/test/entity/entity_test.go b/test/entity/entity_test.go index a45b7464b..04aa39694 100644 --- a/test/entity/entity_test.go +++ b/test/entity/entity_test.go @@ -11,29 +11,17 @@ import ( "testing" "time" - "github.com/aws/aws-sdk-go-v2/aws" "github.com/stretchr/testify/assert" "github.com/aws/amazon-cloudwatch-agent-test/environment" "github.com/aws/amazon-cloudwatch-agent-test/util/awsservice" + "github.com/aws/amazon-cloudwatch-agent-test/util/common" ) const ( sleepForFlush = 240 * time.Second - entityType = "@entity.KeyAttributes.Type" - entityName = "@entity.KeyAttributes.Name" - entityEnvironment = "@entity.KeyAttributes.Environment" - - entityPlatform = "@entity.Attributes.PlatformType" - entityInstanceId = "@entity.Attributes.EC2.InstanceId" - entityEKSCluster = "@entity.Attributes.EKS.Cluster" - entityK8sNode = "@entity.Attributes.K8s.Node" - entityK8sNamespace = "@entity.Attributes.K8s.Namespace" - entityK8sWorkload = "@entity.Attributes.K8s.Workload" - entityServiceNameSource = "@entity.Attributes.AWS.ServiceNameSource" - - // Constants for possible vaues for entity attributes + // Constants for possible values for entity attributes eksServiceEntityType = "Service" entityEKSPlatform = "AWS::EKS" k8sDefaultNamespace = "default" @@ -41,96 +29,6 @@ const ( entityServiceNameSourceK8sWorkload = "K8sWorkload" ) -type EntityValidator struct { - requiredFields map[string]bool - expectedEntity expectedEntity - platformType string - fieldValidators map[string]func(fieldValue string) bool -} - -// NewEntityValidator initializes the validator based on the platform type. -func NewEntityValidator(platformType string, expected expectedEntity) *EntityValidator { - ev := &EntityValidator{ - expectedEntity: expected, - platformType: platformType, - requiredFields: make(map[string]bool), - fieldValidators: make(map[string]func(fieldValue string) bool), - } - - // Define platform-specific required fields and validators - if platformType == "EC2" { - ev.requiredFields = map[string]bool{ - entityType: false, - entityName: false, - entityEnvironment: false, - entityPlatform: false, - entityInstanceId: false, - } - ev.fieldValidators = map[string]func(fieldValue string) bool{ - entityType: func(v string) bool { return v == ev.expectedEntity.entityType }, - entityName: func(v string) bool { return v == ev.expectedEntity.name }, - entityEnvironment: func(v string) bool { return v == ev.expectedEntity.environment }, - entityPlatform: func(v string) bool { return v == ev.expectedEntity.platformType }, - entityInstanceId: func(v string) bool { return v == ev.expectedEntity.instanceId }, - } - } else if platformType == "EKS" { - ev.requiredFields = map[string]bool{ - entityType: false, - entityName: false, - entityEnvironment: false, - entityPlatform: false, - entityEKSCluster: false, - entityK8sNode: false, - entityK8sNamespace: false, - entityK8sWorkload: false, - entityServiceNameSource: false, - } - ev.fieldValidators = map[string]func(fieldValue string) bool{ - entityType: func(v string) bool { return v == ev.expectedEntity.entityType }, - entityName: func(v string) bool { return v == ev.expectedEntity.name }, - entityEnvironment: func(v string) bool { return v == ev.expectedEntity.environment }, - entityPlatform: func(v string) bool { return v == ev.expectedEntity.platformType }, - entityEKSCluster: func(v string) bool { return v == ev.expectedEntity.eksCluster }, - entityK8sNode: func(v string) bool { return v == ev.expectedEntity.k8sNode }, - entityK8sNamespace: func(v string) bool { return v == ev.expectedEntity.k8sNamespace }, - entityK8sWorkload: func(v string) bool { return v == ev.expectedEntity.k8sWorkload }, - entityServiceNameSource: func(v string) bool { return v == ev.expectedEntity.serviceNameSource }, - } - } - return ev -} - -// ValidateField checks if a field is expected and matches the expected value. -func (ev *EntityValidator) ValidateField(field, value string, t *testing.T) { - if validator, ok := ev.fieldValidators[field]; ok { - ev.requiredFields[field] = true - assert.True(t, validator(value), "Validation failed for field %s", field) - } -} - -// AllFieldsPresent ensures all required fields are found. -func (ev *EntityValidator) AllFieldsPresent() bool { - for _, present := range ev.requiredFields { - if !present { - return false - } - } - return true -} - -type expectedEntity struct { - entityType string - name string - environment string - platformType string - k8sWorkload string - k8sNode string - k8sNamespace string - eksCluster string - serviceNameSource string - instanceId string -} - func init() { environment.RegisterEnvironmentMetaDataFlags() } @@ -161,58 +59,58 @@ func TestPutLogEventEntityEKS(t *testing.T) { agentConfigPath string podName string useEC2Tag bool - expectedEntity expectedEntity + expectedEntity common.ExpectedEntity }{ "Entity/K8sWorkloadServiceNameSource": { agentConfigPath: filepath.Join("resources", "compass_default_log.json"), podName: "log-generator", - expectedEntity: expectedEntity{ - entityType: eksServiceEntityType, - name: "log-generator", - environment: "eks:" + env.EKSClusterName + "/" + k8sDefaultNamespace, - platformType: entityEKSPlatform, - k8sWorkload: "log-generator", - k8sNode: *instancePrivateDNS, - k8sNamespace: k8sDefaultNamespace, - eksCluster: env.EKSClusterName, - instanceId: env.InstanceId, - serviceNameSource: entityServiceNameSourceK8sWorkload, + expectedEntity: common.ExpectedEntity{ + EntityType: eksServiceEntityType, + Name: "log-generator", + Environment: "eks:" + env.EKSClusterName + "/" + k8sDefaultNamespace, + PlatformType: entityEKSPlatform, + K8sWorkload: "log-generator", + K8sNode: *instancePrivateDNS, + K8sNamespace: k8sDefaultNamespace, + EksCluster: env.EKSClusterName, + InstanceId: env.InstanceId, + ServiceNameSource: entityServiceNameSourceK8sWorkload, }, }, "Entity/InstrumentationServiceNameSource": { agentConfigPath: filepath.Join("resources", "compass_default_log.json"), podName: "petclinic-instrumentation-default-env", - expectedEntity: expectedEntity{ - entityType: eksServiceEntityType, + expectedEntity: common.ExpectedEntity{ + EntityType: eksServiceEntityType, // This service name comes from OTEL_SERVICE_NAME which is // customized in the terraform code when creating the pod - name: "petclinic-custom-service-name", - environment: "eks:" + env.EKSClusterName + "/" + k8sDefaultNamespace, - platformType: entityEKSPlatform, - k8sWorkload: "petclinic-instrumentation-default-env", - k8sNode: *instancePrivateDNS, - k8sNamespace: k8sDefaultNamespace, - eksCluster: env.EKSClusterName, - instanceId: env.InstanceId, - serviceNameSource: entityServiceNameSourceInstrumentation, + Name: "petclinic-custom-service-name", + Environment: "eks:" + env.EKSClusterName + "/" + k8sDefaultNamespace, + PlatformType: entityEKSPlatform, + K8sWorkload: "petclinic-instrumentation-default-env", + K8sNode: *instancePrivateDNS, + K8sNamespace: k8sDefaultNamespace, + EksCluster: env.EKSClusterName, + InstanceId: env.InstanceId, + ServiceNameSource: entityServiceNameSourceInstrumentation, }, }, "Entity/InstrumentationServiceNameSourceCustomEnvironment": { agentConfigPath: filepath.Join("resources", "compass_default_log.json"), podName: "petclinic-instrumentation-custom-env", - expectedEntity: expectedEntity{ - entityType: eksServiceEntityType, + expectedEntity: common.ExpectedEntity{ + EntityType: eksServiceEntityType, // This service name comes from OTEL_SERVICE_NAME which is // customized in the terraform code when creating the pod - name: "petclinic-custom-service-name", - environment: "petclinic-custom-environment", - platformType: entityEKSPlatform, - k8sWorkload: "petclinic-instrumentation-custom-env", - k8sNode: *instancePrivateDNS, - k8sNamespace: k8sDefaultNamespace, - eksCluster: env.EKSClusterName, - instanceId: env.InstanceId, - serviceNameSource: entityServiceNameSourceInstrumentation, + Name: "petclinic-custom-service-name", + Environment: "petclinic-custom-environment", + PlatformType: entityEKSPlatform, + K8sWorkload: "petclinic-instrumentation-custom-env", + K8sNode: *instancePrivateDNS, + K8sNamespace: k8sDefaultNamespace, + EksCluster: env.EKSClusterName, + InstanceId: env.InstanceId, + ServiceNameSource: entityServiceNameSourceInstrumentation, }, }, } @@ -232,34 +130,7 @@ func TestPutLogEventEntityEKS(t *testing.T) { assert.NotEmpty(t, podApplicationLogStream) // check CWL to ensure we got the expected entities in the log group queryString := fmt.Sprintf("fields @message, @entity.KeyAttributes.Type, @entity.KeyAttributes.Name, @entity.KeyAttributes.Environment, @entity.Attributes.PlatformType, @entity.Attributes.EKS.Cluster, @entity.Attributes.K8s.Node, @entity.Attributes.K8s.Namespace, @entity.Attributes.K8s.Workload, @entity.Attributes.AWS.ServiceNameSource, @entity.Attributes.EC2.InstanceId | filter @logStream == \"%s\"", podApplicationLogStream) - ValidateLogEntity(t, appLogGroup, podApplicationLogStream, &end, queryString, testCase.expectedEntity, string(env.ComputeType)) + common.ValidateLogEntity(t, appLogGroup, podApplicationLogStream, &end, queryString, testCase.expectedEntity, string(env.ComputeType)) }) } } - -// ValidateLogEntity performs the entity validation for PutLogEvents. -func ValidateLogEntity(t *testing.T, logGroup, logStream string, end *time.Time, queryString string, expectedEntity expectedEntity, entityPlatformType string) { - log.Printf("Checking log group/stream: %s/%s", logGroup, logStream) - if !awsservice.IsLogGroupExists(logGroup) { - t.Fatalf("application log group used for entity validation doesn't exist: %s", logGroup) - } - - begin := end.Add(-2 * time.Minute) - log.Printf("Start time is %s and end time is %s", begin.String(), end.String()) - - result, err := awsservice.GetLogQueryResults(logGroup, begin.Unix(), end.Unix(), queryString) - assert.NoError(t, err) - if !assert.NotZero(t, len(result)) { - return - } - - validator := NewEntityValidator(entityPlatformType, expectedEntity) - for _, field := range result[0] { - fieldName := aws.ToString(field.Field) - fieldValue := aws.ToString(field.Value) - validator.ValidateField(fieldName, fieldValue, t) - fmt.Printf("%s: %s\n", fieldName, fieldValue) - } - - assert.True(t, validator.AllFieldsPresent(), "Not all required fields were found") -} diff --git a/util/common/entity.go b/util/common/entity.go new file mode 100644 index 000000000..70d34c8cd --- /dev/null +++ b/util/common/entity.go @@ -0,0 +1,147 @@ +// Copyright Amazon.com, Inc. or its affiliates. All Rights Reserved. +// SPDX-License-Identifier: MIT + +package common + +import ( + "fmt" + "log" + "testing" + "time" + + "github.com/aws/aws-sdk-go-v2/aws" + "github.com/stretchr/testify/assert" + + "github.com/aws/amazon-cloudwatch-agent-test/util/awsservice" +) + +const ( + entityType = "@entity.KeyAttributes.Type" + entityName = "@entity.KeyAttributes.Name" + entityEnvironment = "@entity.KeyAttributes.Environment" + + entityPlatform = "@entity.Attributes.PlatformType" + entityInstanceId = "@entity.Attributes.EC2.InstanceId" + entityEKSCluster = "@entity.Attributes.EKS.Cluster" + entityK8sNode = "@entity.Attributes.K8s.Node" + entityK8sNamespace = "@entity.Attributes.K8s.Namespace" + entityK8sWorkload = "@entity.Attributes.K8s.Workload" + entityServiceNameSource = "@entity.Attributes.AWS.ServiceNameSource" +) + +type ExpectedEntity struct { + EntityType string + Name string + Environment string + PlatformType string + K8sWorkload string + K8sNode string + K8sNamespace string + EksCluster string + ServiceNameSource string + InstanceId string +} + +type EntityValidator struct { + requiredFields map[string]bool + expectedEntity ExpectedEntity + platformType string + fieldValidators map[string]func(fieldValue string) bool +} + +// NewEntityValidator initializes the validator based on the platform type. +func NewEntityValidator(platformType string, expected ExpectedEntity) *EntityValidator { + ev := &EntityValidator{ + expectedEntity: expected, + platformType: platformType, + requiredFields: make(map[string]bool), + fieldValidators: make(map[string]func(fieldValue string) bool), + } + + // Define platform-specific required fields and validators + if platformType == "EC2" { + ev.requiredFields = map[string]bool{ + entityType: false, + entityName: false, + entityEnvironment: false, + entityPlatform: false, + entityInstanceId: false, + } + ev.fieldValidators = map[string]func(fieldValue string) bool{ + entityType: func(v string) bool { return v == ev.expectedEntity.EntityType }, + entityName: func(v string) bool { return v == ev.expectedEntity.Name }, + entityEnvironment: func(v string) bool { return v == ev.expectedEntity.Environment }, + entityPlatform: func(v string) bool { return v == ev.expectedEntity.PlatformType }, + entityInstanceId: func(v string) bool { return v == ev.expectedEntity.InstanceId }, + } + } else if platformType == "EKS" { + ev.requiredFields = map[string]bool{ + entityType: false, + entityName: false, + entityEnvironment: false, + entityPlatform: false, + entityEKSCluster: false, + entityK8sNode: false, + entityK8sNamespace: false, + entityK8sWorkload: false, + entityServiceNameSource: false, + } + ev.fieldValidators = map[string]func(fieldValue string) bool{ + entityType: func(v string) bool { return v == ev.expectedEntity.EntityType }, + entityName: func(v string) bool { return v == ev.expectedEntity.Name }, + entityEnvironment: func(v string) bool { return v == ev.expectedEntity.Environment }, + entityPlatform: func(v string) bool { return v == ev.expectedEntity.PlatformType }, + entityEKSCluster: func(v string) bool { return v == ev.expectedEntity.EksCluster }, + entityK8sNode: func(v string) bool { return v == ev.expectedEntity.K8sNode }, + entityK8sNamespace: func(v string) bool { return v == ev.expectedEntity.K8sNamespace }, + entityK8sWorkload: func(v string) bool { return v == ev.expectedEntity.K8sWorkload }, + entityServiceNameSource: func(v string) bool { return v == ev.expectedEntity.ServiceNameSource }, + } + } + return ev +} + +// ValidateField checks if a field is expected and matches the expected value. +func (ev *EntityValidator) ValidateField(field, value string, t *testing.T) { + if validator, ok := ev.fieldValidators[field]; ok { + ev.requiredFields[field] = true + assert.True(t, validator(value), "Validation failed for field %s", field) + } +} + +// AllFieldsPresent ensures all required fields are found. +func (ev *EntityValidator) AllFieldsPresent() bool { + for _, present := range ev.requiredFields { + if !present { + return false + } + } + return true +} + +// ValidateLogEntity validates the entity data for both EC2 and EKS +func ValidateLogEntity(t *testing.T, logGroup, logStream string, end *time.Time, queryString string, expectedEntity ExpectedEntity, entityPlatformType string) { + log.Printf("Checking log group/stream: %s/%s", logGroup, logStream) + if !awsservice.IsLogGroupExists(logGroup) { + t.Fatalf("application log group used for entity validation doesn't exist: %s", logGroup) + } + + begin := end.Add(-12 * time.Minute) + log.Printf("Start time is %s and end time is %s", begin.String(), end.String()) + + result, err := awsservice.GetLogQueryResults(logGroup, begin.Unix(), end.Unix(), queryString) + assert.NoError(t, err) + if !assert.NotZero(t, len(result)) { + return + } + + validator := NewEntityValidator(entityPlatformType, expectedEntity) + for _, field := range result[0] { + fieldName := aws.ToString(field.Field) + fieldValue := aws.ToString(field.Value) + validator.ValidateField(fieldName, fieldValue, t) + fmt.Printf("%s: %s\n", fieldName, fieldValue) + } + + assert.True(t, validator.AllFieldsPresent(), "Not all required fields were found") +}