add: JSONIncludeKeys, JSONExcludeKeys

.github/workflows/go.yml

@@ -19,7 +19,7 @@ jobs:
     - name: Set up Go
       uses: actions/setup-go@v4
       with:
-        go-version: '1.21'
+        go-version: '1.23'
     - name: Test
       run: |
         go install github.com/onsi/ginkgo/v2/ginkgo

Dockerfile

@@ -1,4 +1,4 @@
-FROM golang:1.21-alpine as base
+FROM golang:1.23-alpine as base
 WORKDIR /build
 RUN apk --update --no-cache add git gcc musl-dev libpcap-dev
 COPY go.mod .

go.mod

@@ -1,6 +1,6 @@
 module github.com/mimuret/dtap/v2
 
-go 1.21
+go 1.23
 
 require go.opentelemetry.io/contrib/instrumentation/net/http/otelhttp v0.44.0 // indirect
 
@@ -20,8 +20,8 @@ require (
 	github.com/miekg/dns v1.1.56
 	github.com/mimuret/dnsutils v1.11.0
 	github.com/nats-io/nats.go v1.28.0
-	github.com/onsi/ginkgo/v2 v2.13.0
-	github.com/onsi/gomega v1.27.10
+	github.com/onsi/ginkgo/v2 v2.22.1
+	github.com/onsi/gomega v1.36.2
 	github.com/pkg/errors v0.9.1
 	github.com/prometheus/client_golang v1.16.0
 	github.com/prometheus/client_model v0.4.0
@@ -31,9 +31,9 @@ require (
 	github.com/spf13/cobra v1.7.0
 	go.uber.org/zap v1.25.0
 	golang.org/x/exp v0.0.0-20230811145659-89c5cff77bcb
-	golang.org/x/net v0.17.0
-	golang.org/x/sync v0.3.0
-	google.golang.org/protobuf v1.31.0
+	golang.org/x/net v0.33.0
+	golang.org/x/sync v0.10.0
+	google.golang.org/protobuf v1.36.1
 	gopkg.in/natefinch/lumberjack.v2 v2.2.1
 	sigs.k8s.io/yaml v1.3.0
 )
@@ -61,7 +61,7 @@ require (
 	github.com/fatih/color v1.13.0 // indirect
 	github.com/felixge/httpsnoop v1.0.3 // indirect
 	github.com/go-logfmt/logfmt v0.5.1 // indirect
-	github.com/go-logr/logr v1.2.4 // indirect
+	github.com/go-logr/logr v1.4.2 // indirect
 	github.com/go-logr/stdr v1.2.2 // indirect
 	github.com/go-openapi/analysis v0.21.4 // indirect
 	github.com/go-openapi/errors v0.20.3 // indirect
@@ -72,15 +72,15 @@ require (
 	github.com/go-openapi/strfmt v0.21.3 // indirect
 	github.com/go-openapi/swag v0.22.3 // indirect
 	github.com/go-openapi/validate v0.22.0 // indirect
-	github.com/go-task/slim-sprig v0.0.0-20230315185526-52ccab3ef572 // indirect
+	github.com/go-task/slim-sprig/v3 v3.0.0 // indirect
 	github.com/gogo/googleapis v1.4.0 // indirect
 	github.com/gogo/protobuf v1.3.2 // indirect
 	github.com/gogo/status v1.1.1 // indirect
 	github.com/golang/protobuf v1.5.3 // indirect
 	github.com/golang/snappy v0.0.4 // indirect
 	github.com/google/btree v1.1.2 // indirect
-	github.com/google/go-cmp v0.5.9 // indirect
-	github.com/google/pprof v0.0.0-20221102093814-76f304f74e5e // indirect
+	github.com/google/go-cmp v0.6.0 // indirect
+	github.com/google/pprof v0.0.0-20241210010833-40e02aabc2ad // indirect
 	github.com/gorilla/mux v1.8.0 // indirect
 	github.com/grafana/dskit v0.0.0-20230109170026-7242706251b9 // indirect
 	github.com/grafana/regexp v0.0.0-20221005093135-b4c2bcb0a4b6 // indirect
@@ -153,20 +153,20 @@ require (
 	go.etcd.io/etcd/api/v3 v3.5.4 // indirect
 	go.etcd.io/etcd/client/pkg/v3 v3.5.4 // indirect
 	go.etcd.io/etcd/client/v3 v3.5.4 // indirect
-	go.mongodb.org/mongo-driver v1.11.0 // indirect
+	go.mongodb.org/mongo-driver v1.17.2 // indirect
 	go.opentelemetry.io/otel v1.18.0 // indirect
 	go.opentelemetry.io/otel/metric v1.18.0 // indirect
 	go.opentelemetry.io/otel/trace v1.18.0 // indirect
 	go.uber.org/atomic v1.10.0 // indirect
 	go.uber.org/goleak v1.2.0 // indirect
 	go.uber.org/multierr v1.10.0 // indirect
-	golang.org/x/crypto v0.14.0 // indirect
-	golang.org/x/mod v0.12.0 // indirect
+	golang.org/x/crypto v0.31.0 // indirect
+	golang.org/x/mod v0.22.0 // indirect
 	golang.org/x/oauth2 v0.8.0 // indirect
-	golang.org/x/sys v0.13.0 // indirect
-	golang.org/x/text v0.13.0 // indirect
+	golang.org/x/sys v0.28.0 // indirect
+	golang.org/x/text v0.21.0 // indirect
 	golang.org/x/time v0.3.0 // indirect
-	golang.org/x/tools v0.13.0 // indirect
+	golang.org/x/tools v0.28.0 // indirect
 	google.golang.org/appengine v1.6.7 // indirect
 	google.golang.org/genproto v0.0.0-20221027153422-115e99e71e1c // indirect
 	google.golang.org/grpc v1.50.1 // indirect

pkg/plugin/input/nats/nats_test.go

@@ -18,7 +18,6 @@ package nats_test
 import (
 	"context"
 	_ "embed"
-	"fmt"
 	"time"
 
 	"github.com/goccy/go-json"
@@ -238,7 +237,6 @@ var _ = Describe("input/nats", func() {
 							time.Sleep(time.Second)
 							cl, err := p.Open()
 							Expect(err).To(Succeed())
-							fmt.Println("publish")
 							err = cl.Publish("dnstap", dnstapRaw)
 							Expect(err).To(Succeed())
 						})

pkg/plugin/input/pcap/pcap_test.go

@@ -99,7 +99,6 @@ var _ = Describe("input/pcap", func() {
 			cancelFunc context.CancelFunc
 		)
 		BeforeEach(func() {
-			fmt.Println(`{"Name":"pcap","ID":"id6","Device":"` + ifName + `"}`)
 			ip, err = pcap.Setup(json.RawMessage(`{"Name":"pcap","ID":"id6","Device":"` + ifName + `"}`))
 			p = ip.(*pcap.PCAP)
 		})

pkg/plugin/output/file/config-schema.json

@@ -25,6 +25,20 @@
         "go-template"
       ]
     },
+    "JSONIncludeKeys": {
+      "description": "JSONIncludeKeys is a key filter when using the json/v1 format.",
+      "type": "array",
+      "items": {
+        "type": "string"
+      }
+    },
+    "JSONExcludeKeys": {
+      "description": "JSONExcludeKeys is a key filter when using the json/v1 format.",
+      "type": "array",
+      "items": {
+        "type": "string"
+      }
+    },
     "Template": {
       "description": "Template is the text/template template string.",
       "type": "string"

pkg/plugin/output/file/file.go

@@ -78,6 +78,10 @@ type Output struct {
 	// File format
 	Format OutputFormat
 
+	// JSON Key Filter
+	JSONIncludeKeys []string
+	JSONExcludeKeys []string
+
 	// Line go template for format type 'go-template"
 	Template string
 
@@ -96,7 +100,7 @@ func (o *Output) Open() error {
 func (o *Output) Write(dm *types.DnstapMessage) error {
 	switch o.Format {
 	case OutputFormatJsonV1:
-		buf, err := dm.ConvertV1JSON()
+		buf, err := dm.ConvertV1JSONWithFilter(o.JSONIncludeKeys, o.JSONExcludeKeys)
 		if err != nil {
 			return err
 		}

pkg/plugin/output/kafka/config-schema.json

@@ -52,8 +52,31 @@
         "Key": {
           "description": "Topic is the kafka message key.",
           "type": "string"
+        },
+        "OutputType": {
+          "description": "OutputType is output format.",
+          "type": "string",
+          "enum": [
+            "json",
+            "protobuf",
+            "avero"
+          ]
+        },
+        "JSONIncludeKeys": {
+          "description": "JSONIncludeKeys is a key filter when using the json OutputType.",
+          "type": "array",
+          "items": {
+            "type": "string"
+          }
+        },
+        "JSONExcludeKeys": {
+          "description": "JSONExcludeKeys is a key filter when using the json OutputType.",
+          "type": "array",
+          "items": {
+            "type": "string"
+          }
         }
-      }
+       }
     }
   }
 }

pkg/plugin/output/kafka/kafka.go

@@ -112,6 +112,8 @@ type KafkaConfig struct {
 	Topic            string
 	Key              string
 	OutputType       OutputType
+	JSONIncludeKeys  []string
+	JSONExcludeKeys  []string
 }
 
 func (o *Kafka) Open() error {
@@ -193,7 +195,7 @@ func (o *Kafka) Write(dm *types.DnstapMessage) error {
 			return err
 		}
 	case OutputTypeJSON:
-		buf, err := dm.ConvertV1JSON()
+		buf, err := dm.ConvertV1JSONWithFilter(o.KafkaConfig.JSONIncludeKeys, o.KafkaConfig.JSONExcludeKeys)
 		if err != nil {
 			return err
 		}

pkg/plugin/output/loki/config-schema.json

@@ -22,6 +22,20 @@
     "MaxStream": {
       "description": "The `MaxStream` is the number of promtail stream.",
       "type": "integer"
+    },
+    "JSONIncludeKeys": {
+      "description": "JSONIncludeKeys is a key filter.",
+      "type": "array",
+      "items": {
+        "type": "string"
+      }
+    },
+    "JSONExcludeKeys": {
+      "description": "JSONExcludeKeys is a key filter.",
+      "type": "array",
+      "items": {
+        "type": "string"
+      }
     }
-  }
+        }
 }

pkg/plugin/output/loki/loki.go

@@ -79,6 +79,10 @@ type Loki struct {
 
 	MaxStream   int
 	MaxLineSize int
+
+	JSONIncludeKeys []string
+	JSONExcludeKeys []string
+
 	client.Config
 	client.Client
 
@@ -104,7 +108,7 @@ func (f *Loki) Open() error {
 }
 
 func (f *Loki) Write(dm *types.DnstapMessage) error {
-	jsonRaw, err := dm.ConvertV1JSON()
+	jsonRaw, err := dm.ConvertV1JSONWithFilter(f.JSONIncludeKeys, f.JSONExcludeKeys)
 	if err != nil {
 		return err
 	}

pkg/plugin/output/nats/nats.go

@@ -17,7 +17,6 @@
 package nats
 
 import (
-	"fmt"
 	"sync"
 
 	json "github.com/goccy/go-json"
@@ -72,7 +71,6 @@ func Setup(bs json.RawMessage) (types.OutputPlugin, error) {
 	}
 	s.DnstapOutput = output.NewDnstapOutput(s, s.MaxRetry)
 
-	fmt.Println(s.GetID())
 	s.openErr = promauto.NewCounter(prometheus.CounterOpts{
 		Namespace:   "dtap",
 		Subsystem:   "output_nats",

pkg/plugin/output/stdout/config-schema.json

@@ -22,6 +22,20 @@
         "go-template"
       ]
     },
+    "JSONIncludeKeys": {
+      "description": "JSONIncludeKeys is a key filter when using the json/v1 format.",
+      "type": "array",
+      "items": {
+        "type": "string"
+      }
+    },
+    "JSONExcludeKeys": {
+      "description": "JSONExcludeKeys is a key filter when using the json/v1 format.",
+      "type": "array",
+      "items": {
+        "type": "string"
+      }
+    },
     "Template": {
       "description": "The `Template` is the `text/template` style template string.",
       "type": "string"

pkg/plugin/output/stdout/stdout.go

@@ -73,6 +73,10 @@ type Stdout struct {
 	// output format type
 	Type OutputFormat
 
+	// JSON Key Filter
+	JSONIncludeKeys []string
+	JSONExcludeKeys []string
+
 	// Line go template for format type 'go-template"
 	Template string
 
@@ -91,7 +95,7 @@ func (o *Stdout) Open() error {
 func (o *Stdout) Write(dm *types.DnstapMessage) error {
 	switch o.Type {
 	case OutputFormatJsonV1:
-		buf, err := dm.ConvertV1JSON()
+		buf, err := dm.ConvertV1JSONWithFilter(o.JSONIncludeKeys, o.JSONExcludeKeys)
 		if err != nil {
 			return err
 		}

pkg/types/dnstap_message_v1_convert.go

@@ -204,6 +204,27 @@ func (d *DnstapMessage) ConvertV1JSON() ([]byte, error) {
 	return json.Marshal(flat)
 }
 
+func (d *DnstapMessage) ConvertV1JSONWithFilter(includeKeys, excludeKeys []string) ([]byte, error) {
+	mapString, err := d.ConvertV1MapString()
+	if err != nil {
+		return nil, err
+	}
+	res := map[string]interface{}{}
+	if len(includeKeys) > 0 {
+		for _, k := range includeKeys {
+			if v, ok := mapString[k]; ok {
+				res[k] = v
+			}
+		}
+	} else {
+		res = mapString
+	}
+	for _, k := range excludeKeys {
+		delete(res, k)
+	}
+	return json.Marshal(res)
+}
+
 func getName(labels []string, i int) string {
 	var res string
 	labelsLen := len(labels)