diff --git a/confgenerator/confgenerator.go b/confgenerator/confgenerator.go index c3bd8b0f6b..378109d61c 100644 --- a/confgenerator/confgenerator.go +++ b/confgenerator/confgenerator.go @@ -20,6 +20,7 @@ import ( "crypto/md5" "encoding/hex" "fmt" + "log" "path" "regexp" "sort" @@ -28,6 +29,7 @@ import ( "github.com/GoogleCloudPlatform/ops-agent/confgenerator/fluentbit" "github.com/GoogleCloudPlatform/ops-agent/confgenerator/otel" + "github.com/GoogleCloudPlatform/ops-agent/confgenerator/resourcedetector" "github.com/GoogleCloudPlatform/ops-agent/internal/platform" ) @@ -293,6 +295,57 @@ func processUserDefinedMultilineParser(i int, pID string, receiver LoggingReceiv return nil } +func sliceContains(s []string, v string) bool { + for _, e := range s { + if e == v { + return true + } + } + return false +} + +const ( + attributeLabelPrefix string = "compute.googleapis.com/attributes/" +) + +// addGceMetadataAttributesComponents annotates logs with labels corresponding +// to instance attributes from the GCE metadata server. +func addGceMetadataAttributesComponents(ctx context.Context, attributes []string, tag, uid string) []fluentbit.Component { + processorName := fmt.Sprintf("%s.%s.gce_metadata", tag, uid) + resource, err := platform.FromContext(ctx).GetResource() + if err != nil { + log.Printf("can't get resource metadata: %v", err) + return nil + } + gceMetadata, ok := resource.(resourcedetector.GCEResource) + if !ok { + // Not on GCE; no attributes to detect. + log.Printf("ignoring the gce_metadata_attributes processor outside of GCE: %T", resource) + return nil + } + modifications := map[string]*ModifyField{} + var attributeKeys []string + for k, _ := range gceMetadata.Metadata { + attributeKeys = append(attributeKeys, k) + } + sort.Strings(attributeKeys) + for _, k := range attributeKeys { + if !sliceContains(attributes, k) { + continue + } + v := gceMetadata.Metadata[k] + modifications[fmt.Sprintf(`labels."%s%s"`, attributeLabelPrefix, k)] = &ModifyField{ + StaticValue: &v, + } + } + if len(modifications) == 0 { + return nil + } + return LoggingProcessorModifyFields{ + Fields: modifications, + }.Components(ctx, tag, processorName) +} + // generateFluentbitComponents generates a slice of fluentbit config sections to represent l. func (uc *UnifiedConfig) generateFluentbitComponents(ctx context.Context, userAgent string) ([]fluentbit.Component, error) { l := uc.Logging @@ -396,6 +449,11 @@ func (uc *UnifiedConfig) generateFluentbitComponents(ctx context.Context, userAg out = append(out, stackdriverOutputComponent(ctx, strings.Join(tags, "|"), userAgent, "2G")) } out = append(out, uc.generateSelfLogsComponents(ctx, userAgent)...) + out = append(out, addGceMetadataAttributesComponents(ctx, []string{ + "dataproc-cluster-name", + "dataproc-cluster-uuid", + "dataproc-region", + }, "*", "default-dataproc")...) } out = append(out, fluentbit.MetricsOutputComponent()) diff --git a/confgenerator/confgenerator_test.go b/confgenerator/confgenerator_test.go index 7185fbbf1c..45bdf93f2b 100644 --- a/confgenerator/confgenerator_test.go +++ b/confgenerator/confgenerator_test.go @@ -57,20 +57,38 @@ var winlogv1channels = []string{ } var ( - testPlatforms = []platformConfig{ - { - name: "linux", - defaultLogsDir: "/var/log/google-cloud-ops-agent", - defaultStateDir: "/var/lib/google-cloud-ops-agent/fluent-bit", - platform: platform.Platform{ - Type: platform.Linux, - HostInfo: &host.InfoStat{ - OS: "linux", - Platform: "linux_platform", - PlatformVersion: "linux_platform_version", - }, + // Set up the test environment with mocked data. + testResource = resourcedetector.GCEResource{ + Project: "test-project", + Zone: "test-zone", + Network: "test-network", + Subnetwork: "test-subnetwork", + PublicIP: "test-public-ip", + PrivateIP: "test-private-ip", + InstanceID: "test-instance-id", + InstanceName: "test-instance-name", + Tags: "test-tag", + MachineType: "test-machine-type", + Metadata: map[string]string{"test-key": "test-value", "test-escape": "${foo:bar}"}, + Label: map[string]string{"test-label-key": "test-label-value"}, + InterfaceIPv4: map[string]string{"test-interface": "test-interface-ipv4"}, + } + linuxTestPlatform = platformConfig{ + name: "linux", + defaultLogsDir: "/var/log/google-cloud-ops-agent", + defaultStateDir: "/var/lib/google-cloud-ops-agent/fluent-bit", + platform: platform.Platform{ + Type: platform.Linux, + HostInfo: &host.InfoStat{ + OS: "linux", + Platform: "linux_platform", + PlatformVersion: "linux_platform_version", }, + TestGCEResourceOverride: testResource, }, + } + testPlatforms = []platformConfig{ + linuxTestPlatform, { name: "linux-gpu", defaultLogsDir: "/var/log/google-cloud-ops-agent", @@ -82,7 +100,8 @@ var ( Platform: "linux_platform", PlatformVersion: "linux_platform_version", }, - HasNvidiaGpu: true, + TestGCEResourceOverride: testResource, + HasNvidiaGpu: true, }, }, { @@ -98,6 +117,7 @@ var ( Platform: "win_platform", PlatformVersion: "win_platform_version", }, + TestGCEResourceOverride: testResource, }, }, { @@ -113,6 +133,7 @@ var ( Platform: "win_platform", PlatformVersion: "win_platform_version", }, + TestGCEResourceOverride: testResource, }, }, } @@ -149,6 +170,42 @@ func TestGoldens(t *testing.T) { } } +func TestDataprocDefaults(t *testing.T) { + t.Parallel() + + goldensDir := "goldens" + testName := "builtin" + dataprocMetadata := map[string]string{ + "dataproc-cluster-name": "test-cluster", + "dataproc-cluster-uuid": "test-uuid", + "dataproc-region": "test-region", + } + + t.Run(testName, func(t *testing.T) { + t.Parallel() + pc := linuxTestPlatform + // Update mocked resource to include Dataproc labels. + dataprocResource := testResource + newMetadata := map[string]string{} + for k, v := range testResource.Metadata { + newMetadata[k] = v + } + for k, v := range dataprocMetadata { + newMetadata[k] = v + } + dataprocResource.Metadata = newMetadata + pc.platform.TestGCEResourceOverride = dataprocResource + t.Run(pc.name, func(t *testing.T) { + testDir := filepath.Join(goldensDir, testName) + got, err := generateConfigs(pc, testDir) + assert.NilError(t, err, "Failed to generate configs: %v", err) + if err := testGeneratedFiles(t, got, filepath.Join(testDir, goldenDir, "linux-dataproc")); err != nil { + t.Errorf("Failed to check generated configs: %v", err) + } + }) + }) +} + func getTestsInDir(t *testing.T, testDir string) []string { t.Helper() @@ -310,25 +367,6 @@ func TestMain(m *testing.M) { } func init() { - testResource := resourcedetector.GCEResource{ - Project: "test-project", - Zone: "test-zone", - Network: "test-network", - Subnetwork: "test-subnetwork", - PublicIP: "test-public-ip", - PrivateIP: "test-private-ip", - InstanceID: "test-instance-id", - InstanceName: "test-instance-name", - Tags: "test-tag", - MachineType: "test-machine-type", - Metadata: map[string]string{"test-key": "test-value", "test-escape": "${foo:bar}"}, - Label: map[string]string{"test-label-key": "test-label-value"}, - InterfaceIPv4: map[string]string{"test-interface": "test-interface-ipv4"}, - } - - // Set up the test environment with mocked data. - confgenerator.MetadataResource = testResource - // Enable experimental features here by calling: // os.Setenv("EXPERIMENTAL_FEATURES", "...(comma-separated feature list)...") } diff --git a/confgenerator/files.go b/confgenerator/files.go index 9109434aa8..c31958b974 100644 --- a/confgenerator/files.go +++ b/confgenerator/files.go @@ -20,8 +20,6 @@ import ( "io/ioutil" "os" "path/filepath" - - "github.com/GoogleCloudPlatform/ops-agent/confgenerator/resourcedetector" ) // ReadUnifiedConfigFromFile reads the user config file and returns a UnifiedConfig. @@ -62,13 +60,6 @@ func (uc *UnifiedConfig) GenerateFilesFromConfig(ctx context.Context, service, l } } case "otel": - // Fetch resource information from the metadata server. - var err error - MetadataResource, err = resourcedetector.GetResource() - if err != nil { - return fmt.Errorf("can't get resource metadata: %w", err) - } - otelConfig, err := uc.GenerateOtelConfig(ctx) if err != nil { return fmt.Errorf("can't parse configuration: %w", err) diff --git a/confgenerator/prometheus.go b/confgenerator/prometheus.go index cc49b5f0ad..737d8e47c4 100644 --- a/confgenerator/prometheus.go +++ b/confgenerator/prometheus.go @@ -18,6 +18,7 @@ import ( "context" "errors" "fmt" + "log" "os" "reflect" "sort" @@ -25,7 +26,6 @@ import ( "strings" "github.com/GoogleCloudPlatform/ops-agent/confgenerator/otel" - "github.com/GoogleCloudPlatform/ops-agent/confgenerator/resourcedetector" "github.com/GoogleCloudPlatform/ops-agent/internal/platform" "github.com/go-playground/validator/v10" yaml "github.com/goccy/go-yaml" @@ -36,12 +36,6 @@ import ( _ "github.com/prometheus/prometheus/discovery/install" // init() of this package registers service discovery impl. ) -var ( - // MetadataResource is the resource metadata for the instance we're running on. - // Note: This is a global variable so that it can be set in tests. - MetadataResource resourcedetector.Resource -) - type PrometheusMetrics struct { ConfigComponent `yaml:",inline"` @@ -61,9 +55,10 @@ func (r PrometheusMetrics) Type() string { } func (r PrometheusMetrics) Pipelines(ctx context.Context) []otel.ReceiverPipeline { - resource := MetadataResource - if p := platform.FromContext(ctx).ResourceOverride; p != nil { - resource = p + resource, err := platform.FromContext(ctx).GetResource() + if err != nil { + log.Printf("can't get resource metadata: %v", err) + return nil } if resource != nil { // Get the resource metadata for the instance we're running on. diff --git a/confgenerator/resourcedetector/detector.go b/confgenerator/resourcedetector/detector.go index 0570dd5ce5..6fa0dbfe16 100644 --- a/confgenerator/resourcedetector/detector.go +++ b/confgenerator/resourcedetector/detector.go @@ -22,17 +22,36 @@ import ( // An implementation of the Resource interface will have fields represent // available attributes about the current monitoring resource. type Resource interface { + ProjectName() string MonitoredResource() *monitoredres.MonitoredResource OTelResourceAttributes() map[string]string - ProjectName() string PrometheusStyleMetadata() map[string]string } +type resourceCache struct { + Resource Resource + Error error +} + +var cachedResourceAndError *resourceCache + // Get a resource instance for the current environment; // In order to access the attributes of a specific type of resource, // needs to cast the returned Resource instance to its underlying type: // actual, ok := resource.(GCEResource) func GetResource() (Resource, error) { + if cachedResourceAndError != nil { + return cachedResourceAndError.Resource, cachedResourceAndError.Error + } + r, err := getUncachedResource() + cachedResourceAndError = &resourceCache{ + Resource: r, + Error: err, + } + return r, err +} + +func getUncachedResource() (Resource, error) { switch { case gcp_metadata.OnGCE(): return GetGCEResource() @@ -46,10 +65,6 @@ func GetResource() (Resource, error) { type UnrecognizedPlatformResource struct { } -func (UnrecognizedPlatformResource) OTelResourceAttributes() map[string]string { - return nil -} - func (UnrecognizedPlatformResource) ProjectName() string { return "" } @@ -58,6 +73,10 @@ func (UnrecognizedPlatformResource) MonitoredResource() *monitoredres.MonitoredR return nil } +func (UnrecognizedPlatformResource) OTelResourceAttributes() map[string]string { + return nil +} + func (UnrecognizedPlatformResource) PrometheusStyleMetadata() map[string]string { return nil } diff --git a/confgenerator/testdata/goldens/builtin/golden/linux-dataproc/3c47867f5b9bfc4409551ecdaaac4562.lua b/confgenerator/testdata/goldens/builtin/golden/linux-dataproc/3c47867f5b9bfc4409551ecdaaac4562.lua new file mode 100644 index 0000000000..c4acb5b5fa --- /dev/null +++ b/confgenerator/testdata/goldens/builtin/golden/linux-dataproc/3c47867f5b9bfc4409551ecdaaac4562.lua @@ -0,0 +1,28 @@ + +function process(tag, timestamp, record) +local v = "test-cluster"; +(function(value) +if record["logging.googleapis.com/labels"] == nil +then +record["logging.googleapis.com/labels"] = {} +end +record["logging.googleapis.com/labels"]["compute.googleapis.com/attributes/dataproc-cluster-name"] = value +end)(v) +local v = "test-uuid"; +(function(value) +if record["logging.googleapis.com/labels"] == nil +then +record["logging.googleapis.com/labels"] = {} +end +record["logging.googleapis.com/labels"]["compute.googleapis.com/attributes/dataproc-cluster-uuid"] = value +end)(v) +local v = "test-region"; +(function(value) +if record["logging.googleapis.com/labels"] == nil +then +record["logging.googleapis.com/labels"] = {} +end +record["logging.googleapis.com/labels"]["compute.googleapis.com/attributes/dataproc-region"] = value +end)(v) +return 2, timestamp, record +end diff --git a/confgenerator/testdata/goldens/builtin/golden/linux-dataproc/4d6012ff003886818fb9b9285b4af962.lua b/confgenerator/testdata/goldens/builtin/golden/linux-dataproc/4d6012ff003886818fb9b9285b4af962.lua new file mode 100644 index 0000000000..c225be24d5 --- /dev/null +++ b/confgenerator/testdata/goldens/builtin/golden/linux-dataproc/4d6012ff003886818fb9b9285b4af962.lua @@ -0,0 +1,19 @@ + +function process(tag, timestamp, record) +local __field_0 = (function() +return record["severity"] +end)(); +(function(value) +record["severity"] = value +end)(nil); +local v = __field_0; +if v == "debug" then v = "DEBUG" +elseif v == "error" then v = "ERROR" +elseif v == "info" then v = "INFO" +elseif v == "warn" then v = "WARNING" +end +(function(value) +record["logging.googleapis.com/severity"] = value +end)(v) +return 2, timestamp, record +end diff --git a/confgenerator/testdata/goldens/builtin/golden/linux-dataproc/5fc5f42c16c9e1ab8292e3d42f74f3be.lua b/confgenerator/testdata/goldens/builtin/golden/linux-dataproc/5fc5f42c16c9e1ab8292e3d42f74f3be.lua new file mode 100644 index 0000000000..c5465182b5 --- /dev/null +++ b/confgenerator/testdata/goldens/builtin/golden/linux-dataproc/5fc5f42c16c9e1ab8292e3d42f74f3be.lua @@ -0,0 +1,49 @@ + + function shallow_merge(record, parsedRecord) + -- If no exiting record exists + if (record == nil) then + return parsedRecord + end + + for k, v in pairs(parsedRecord) do + record[k] = v + end + + return record +end + +function merge(record, parsedRecord) + -- If no exiting record exists + if record == nil then + return parsedRecord + end + + -- Potentially overwrite or merge the original records. + for k, v in pairs(parsedRecord) do + -- If there is no conflict + if k == "logging.googleapis.com/logName" then + -- Ignore the parsed payload since the logName is controlled + -- by the OpsAgent. + elseif k == "logging.googleapis.com/labels" then + -- LogEntry.labels are basically a map[string]string and so only require a + -- shallow merge (one level deep merge). + record[k] = shallow_merge(record[k], v) + else + record[k] = v + end + end + + return record +end + +function parser_merge_record(tag, timestamp, record) + originalPayload = record["logging.googleapis.com/__tmp"] + if originalPayload == nil then + return 0, timestamp, record + end + + -- Remove original payload + record["logging.googleapis.com/__tmp"] = nil + record = merge(originalPayload, record) + return 2, timestamp, record +end diff --git a/confgenerator/testdata/goldens/builtin/golden/linux-dataproc/68d6a3a0e8edf37868e1bf94adc737f4.lua b/confgenerator/testdata/goldens/builtin/golden/linux-dataproc/68d6a3a0e8edf37868e1bf94adc737f4.lua new file mode 100644 index 0000000000..16b933cb7d --- /dev/null +++ b/confgenerator/testdata/goldens/builtin/golden/linux-dataproc/68d6a3a0e8edf37868e1bf94adc737f4.lua @@ -0,0 +1,38 @@ + +function process(tag, timestamp, record) +local __field_0 = (function() +return record["message"] +end)(); +local v = __field_0; +if v == "LogParseErr" then v = "Ops Agent failed to parse logs, Code: LogParseErr, Documentation: https://cloud.google.com/stackdriver/docs/solutions/agents/ops-agent/troubleshoot-find-info" +elseif v == "LogPipelineErr" then v = "Ops Agent logging pipeline failed, Code: LogPipelineErr, Documentation: https://cloud.google.com/stackdriver/docs/solutions/agents/ops-agent/troubleshoot-find-info" +end +(function(value) +record["message"] = value +end)(v) +local v = "ops-agent"; +(function(value) +if record["logging.googleapis.com/labels"] == nil +then +record["logging.googleapis.com/labels"] = {} +end +record["logging.googleapis.com/labels"]["agent.googleapis.com/health/agentKind"] = value +end)(v) +local v = "latest"; +(function(value) +if record["logging.googleapis.com/labels"] == nil +then +record["logging.googleapis.com/labels"] = {} +end +record["logging.googleapis.com/labels"]["agent.googleapis.com/health/agentVersion"] = value +end)(v) +local v = "v1"; +(function(value) +if record["logging.googleapis.com/labels"] == nil +then +record["logging.googleapis.com/labels"] = {} +end +record["logging.googleapis.com/labels"]["agent.googleapis.com/health/schemaVersion"] = value +end)(v) +return 2, timestamp, record +end diff --git a/confgenerator/testdata/goldens/builtin/golden/linux-dataproc/b4a0dead382dce7b4fe011d3f59fdb6d.lua b/confgenerator/testdata/goldens/builtin/golden/linux-dataproc/b4a0dead382dce7b4fe011d3f59fdb6d.lua new file mode 100644 index 0000000000..6263563b66 --- /dev/null +++ b/confgenerator/testdata/goldens/builtin/golden/linux-dataproc/b4a0dead382dce7b4fe011d3f59fdb6d.lua @@ -0,0 +1,17 @@ + +function parser_nest(tag, timestamp, record) + local nestedRecord = {} + local parseKey = "message" + for k, v in pairs(record) do + if k ~= parseKey then + nestedRecord[k] = v + end + end + + local result = {} + result[parseKey] = record[parseKey] + result["logging.googleapis.com/__tmp"] = nestedRecord + + return 2, timestamp, result +end + diff --git a/confgenerator/testdata/goldens/builtin/golden/linux-dataproc/builtin_conf.yaml b/confgenerator/testdata/goldens/builtin/golden/linux-dataproc/builtin_conf.yaml new file mode 100644 index 0000000000..cd84fc39b6 --- /dev/null +++ b/confgenerator/testdata/goldens/builtin/golden/linux-dataproc/builtin_conf.yaml @@ -0,0 +1,25 @@ +logging: + receivers: + syslog: + type: files + include_paths: + - /var/log/messages + - /var/log/syslog + service: + pipelines: + default_pipeline: + receivers: [syslog] +metrics: + receivers: + hostmetrics: + type: hostmetrics + collection_interval: 60s + processors: + metrics_filter: + type: exclude_metrics + metrics_pattern: [] + service: + pipelines: + default_pipeline: + receivers: [hostmetrics] + processors: [metrics_filter] diff --git a/confgenerator/testdata/goldens/builtin/golden/linux-dataproc/f120d4527bd717cab023dbbe5fbdc332.lua b/confgenerator/testdata/goldens/builtin/golden/linux-dataproc/f120d4527bd717cab023dbbe5fbdc332.lua new file mode 100644 index 0000000000..c09b885e11 --- /dev/null +++ b/confgenerator/testdata/goldens/builtin/golden/linux-dataproc/f120d4527bd717cab023dbbe5fbdc332.lua @@ -0,0 +1,42 @@ + +function process(tag, timestamp, record) +local __field_0 = (function() +return record["agent.googleapis.com/log_file_path"] +end)(); +local __field_1 = (function() +if record["logging.googleapis.com/labels"] == nil +then +return nil +end +return record["logging.googleapis.com/labels"]["compute.googleapis.com/resource_name"] +end)(); +local __field_2 = (function() +return record["logging.googleapis.com/logName"] +end)(); +(function(value) +record["agent.googleapis.com/log_file_path"] = value +end)(nil); +local v = __field_0; +(function(value) +if record["logging.googleapis.com/labels"] == nil +then +record["logging.googleapis.com/labels"] = {} +end +record["logging.googleapis.com/labels"]["agent.googleapis.com/log_file_path"] = value +end)(v) +local v = __field_1; +if v == nil then v = "" end; +(function(value) +if record["logging.googleapis.com/labels"] == nil +then +record["logging.googleapis.com/labels"] = {} +end +record["logging.googleapis.com/labels"]["compute.googleapis.com/resource_name"] = value +end)(v) +local v = __field_2; +if v == nil then v = "syslog" end; +(function(value) +record["logging.googleapis.com/logName"] = value +end)(v) +return 2, timestamp, record +end diff --git a/confgenerator/testdata/goldens/builtin/golden/linux-dataproc/features.yaml b/confgenerator/testdata/goldens/builtin/golden/linux-dataproc/features.yaml new file mode 100644 index 0000000000..6d4de0791f --- /dev/null +++ b/confgenerator/testdata/goldens/builtin/golden/linux-dataproc/features.yaml @@ -0,0 +1,12 @@ +- module: logging + feature: service:pipelines + key: default_pipeline_overridden + value: "false" +- module: metrics + feature: service:pipelines + key: default_pipeline_overridden + value: "false" +- module: global + feature: default:self_log + key: default_self_log_file_collection + value: "true" diff --git a/confgenerator/testdata/goldens/builtin/golden/linux-dataproc/fluent_bit_main.conf b/confgenerator/testdata/goldens/builtin/golden/linux-dataproc/fluent_bit_main.conf new file mode 100644 index 0000000000..b803fa3274 --- /dev/null +++ b/confgenerator/testdata/goldens/builtin/golden/linux-dataproc/fluent_bit_main.conf @@ -0,0 +1,186 @@ +@SET buffers_dir=/var/lib/google-cloud-ops-agent/fluent-bit/buffers +@SET logs_dir=/var/log/google-cloud-ops-agent + +[SERVICE] + Daemon off + Flush 1 + Log_Level info + dns.resolver legacy + storage.backlog.mem_limit 50M + storage.checksum off + storage.max_chunks_up 128 + storage.metrics on + storage.sync normal + +[INPUT] + Name fluentbit_metrics + Scrape_Interval 60 + Scrape_On_Start True + +[INPUT] + Buffer_Chunk_Size 512k + Buffer_Max_Size 2M + DB ${buffers_dir}/default_pipeline_syslog + DB.locking true + Key message + Mem_Buf_Limit 10M + Name tail + Path /var/log/messages,/var/log/syslog + Read_from_Head True + Rotate_Wait 30 + Skip_Long_Lines On + Tag default_pipeline.syslog + storage.type filesystem + +[INPUT] + Buffer_Chunk_Size 512k + Buffer_Max_Size 2M + DB ${buffers_dir}/ops-agent-fluent-bit + DB.locking true + Key message + Mem_Buf_Limit 10M + Name tail + Path ${logs_dir}/subagents/logging-module.log + Read_from_Head True + Rotate_Wait 30 + Skip_Long_Lines On + Tag ops-agent-fluent-bit + storage.type memory + +[INPUT] + Buffer_Chunk_Size 512k + Buffer_Max_Size 2M + DB ${buffers_dir}/ops-agent-health + DB.locking true + Key message + Mem_Buf_Limit 10M + Name tail + Path ${logs_dir}/health-checks.log + Read_from_Head True + Rotate_Wait 30 + Skip_Long_Lines On + Tag ops-agent-health + storage.type memory + +[FILTER] + Match default_pipeline.syslog + Name lua + call process + script f120d4527bd717cab023dbbe5fbdc332.lua + +[FILTER] + Match ops-agent-fluent-bit + Name lua + call parser_nest + script b4a0dead382dce7b4fe011d3f59fdb6d.lua + +[FILTER] + Key_Name message + Match ops-agent-fluent-bit + Name parser + Preserve_Key True + Reserve_Data True + Parser ops-agent-fluent-bit.fluent-bit-self-log-regex-parsing + +[FILTER] + Match ops-agent-fluent-bit + Name lua + call parser_merge_record + script 5fc5f42c16c9e1ab8292e3d42f74f3be.lua + +[FILTER] + Match ops-agent-health + Name lua + call parser_nest + script b4a0dead382dce7b4fe011d3f59fdb6d.lua + +[FILTER] + Key_Name message + Match ops-agent-health + Name parser + Reserve_Data True + Parser ops-agent-health.health-checks-json + +[FILTER] + Match ops-agent-health + Name lua + call parser_merge_record + script 5fc5f42c16c9e1ab8292e3d42f74f3be.lua + +[FILTER] + Match ops-agent-health + Name grep + Regex severity INFO|ERROR|WARNING|DEBUG|info|error|warning|debug + +[FILTER] + Match ops-agent-fluent-bit + Name rewrite_tag + Rule message \[error\]\s\[lib\]\sbackend\sfailed ops-agent-health true + +[FILTER] + Name modify + Match ops-agent-health + Condition Key_value_matches message \[error\]\s\[lib\]\sbackend\sfailed + Set message LogPipelineErr + Set code LogPipelineErr + +[FILTER] + Match ops-agent-fluent-bit + Name rewrite_tag + Rule message \[error\]\s\[parser\]\scannot\sparse ops-agent-health true + +[FILTER] + Name modify + Match ops-agent-health + Condition Key_value_matches message \[error\]\s\[parser\]\scannot\sparse + Set message LogParseErr + Set code LogParseErr + +[FILTER] + Match ops-agent-health + Name lua + call process + script 68d6a3a0e8edf37868e1bf94adc737f4.lua + +[FILTER] + Match ops-agent-* + Name lua + call process + script 4d6012ff003886818fb9b9285b4af962.lua + +[FILTER] + Match * + Name lua + call process + script 3c47867f5b9bfc4409551ecdaaac4562.lua + +[OUTPUT] + Match_Regex ^(default_pipeline\.syslog)$ + Name stackdriver + Retry_Limit 3 + http_request_key logging.googleapis.com/httpRequest + net.connect_timeout_log_error False + resource gce_instance + stackdriver_agent Google-Cloud-Ops-Agent-Logging/latest (BuildDistro=build_distro;Platform=linux;ShortName=linux_platform;ShortVersion=linux_platform_version) + storage.total_limit_size 2G + tls On + tls.verify Off + workers 8 + +[OUTPUT] + Match_Regex ^(ops-agent-health|ops-agent-fluent-bit)$ + Name stackdriver + Retry_Limit 3 + http_request_key logging.googleapis.com/httpRequest + net.connect_timeout_log_error False + resource gce_instance + stackdriver_agent Google-Cloud-Ops-Agent-Logging/latest (BuildDistro=build_distro;Platform=linux;ShortName=linux_platform;ShortVersion=linux_platform_version) + tls On + tls.verify Off + workers 8 + +[OUTPUT] + Match * + Name prometheus_exporter + host 0.0.0.0 + port 20202 diff --git a/confgenerator/testdata/goldens/builtin/golden/linux-dataproc/fluent_bit_parser.conf b/confgenerator/testdata/goldens/builtin/golden/linux-dataproc/fluent_bit_parser.conf new file mode 100644 index 0000000000..e6c4f531ea --- /dev/null +++ b/confgenerator/testdata/goldens/builtin/golden/linux-dataproc/fluent_bit_parser.conf @@ -0,0 +1,13 @@ +[PARSER] + Format regex + Name ops-agent-fluent-bit.fluent-bit-self-log-regex-parsing + Regex (?\[[ ]*(?