fix prometheus

Author: mimuret

config.go

@@ -55,18 +55,46 @@ var (
 		OutputPrometheusMetrics{
 			Name:   "dtap_query_qtype_total",
 			Help:   "Total number of queries with a given query type.",
-			Labels: []string{"qtype"},
+			Labels: []string{"Qtype"},
 		},
 		OutputPrometheusMetrics{
-			Name:   "dtap_query_protocol_total",
-			Help:   "Total number of queries with a given query rotocol.",
-			Labels: []string{"socket_protocol"},
+			Name:   "dtap_query_rcode_total",
+			Help:   "Total number of queries with a given query type.",
+			Labels: []string{"Rcode"},
+		},
+		OutputPrometheusMetrics{
+			Name:   "dtap_query_tc_bit_total",
+			Help:   "Total number of queries with a given query tc bit.",
+			Labels: []string{"TC"},
+		},
+		OutputPrometheusMetrics{
+			Name:   "dtap_query_ad_bit_total",
+			Help:   "Total number of queries with a given query ad bit.",
+			Labels: []string{"AD"},
+		},
+		OutputPrometheusMetrics{
+			Name:   "dtap_query_socket_protocol_total",
+			Help:   "Total number of queries with a given query transport rotocol.",
+			Labels: []string{"SocketProtocol"},
+		},
+		OutputPrometheusMetrics{
+			Name:   "dtap_query_socket_family_total",
+			Help:   "Total number of queries with a given query IP Protocol.",
+			Labels: []string{"SocketFamily"},
 		},
 		OutputPrometheusMetrics{
-			Name:   "dtap_query_tld_total",
-			Help:   "Total number of queries with a given query tld.",
-			Labels: []string{"tld"},
-			Limit:  100,
+			Name:           "dtap_query_tld_total",
+			Help:           "Total number of queries with a given query tld.",
+			Labels:         []string{"TopLevelDomainName"},
+			ExpireInterval: 5,
+			ExpireSec:      60,
+		},
+		OutputPrometheusMetrics{
+			Name:           "dtap_query_sld_total",
+			Help:           "Total number of queries with a given query tld.",
+			Labels:         []string{"TopLevelDomainName"},
+			ExpireInterval: 5,
+			ExpireSec:      60,
 		},
 	}
 )
@@ -466,7 +494,6 @@ func (o *OutputNatsConfig) GetToken() string {
 
 type OutputPrometheus struct {
 	Counters []OutputPrometheusMetrics
-	Interval int
 	Flat     OutputCommonConfig
 	Buffer   OutputBufferConfig
 }
@@ -478,19 +505,17 @@ func (o *OutputPrometheus) GetCounters() []OutputPrometheusMetrics {
 	return o.Counters
 }
 
-func (o *OutputPrometheus) GetInternal() int {
-	return o.Interval
-}
-
 func (o *OutputPrometheus) Validate() *ValidationError {
 	return nil
 }
 
 type OutputPrometheusMetrics struct {
-	Name   string
-	Help   string
-	Labels []string
-	Limit  int
+	Name           string
+	Help           string
+	Labels         []string
+	Limit          int
+	ExpireInterval int
+	ExpireSec      int
 }
 
 func (o *OutputPrometheusMetrics) GetName() string {
@@ -509,6 +534,14 @@ func (o *OutputPrometheusMetrics) GetLimit() int {
 	return o.Limit
 }
 
+func (o *OutputPrometheusMetrics) GetExpireInterval() int {
+	return o.ExpireInterval
+}
+
+func (o *OutputPrometheusMetrics) GetExpireSec() int {
+	return o.ExpireSec
+}
+
 type OutputBufferConfig struct {
 	BufferSize uint
 }

dnstap_prometheus_output.go

@@ -17,6 +17,12 @@
 package dtap
 
 import (
+	"context"
+	"strings"
+	"time"
+
+	log "github.com/sirupsen/logrus"
+
 	"fmt"
 	"reflect"
 	"strconv"
@@ -33,21 +39,71 @@ type DnstapPrometheusOutput struct {
 }
 
 type DnstapPrometheusOutputMetrics struct {
-	Vec    *prometheus.CounterVec
-	Labels []string
+	Name        string
+	Vec         *prometheus.CounterVec
+	LabelKeys   []string
+	LabelValues map[string]*DnstapPrometheusOutputMetricsValues
+	Interval    int
+	Expire      int
+	CancelFunc  context.CancelFunc
+}
+
+func (d *DnstapPrometheusOutputMetrics) GetInterval() int {
+	if d.Interval <= 0 {
+		return 0
+	}
+	return d.Interval
+}
+
+func (d *DnstapPrometheusOutputMetrics) GetExpire() int {
+	if d.Expire <= 0 {
+		return 0
+	}
+	return d.Expire
+}
+
+type DnstapPrometheusOutputMetricsValues struct {
+	Values     []string
+	LastUpdate time.Time
+}
+
+func NewDnstapPrometheusOutputMetrics(counterConfig OutputPrometheusMetrics) *DnstapPrometheusOutputMetrics {
+	return &DnstapPrometheusOutputMetrics{
+		Name: counterConfig.GetName(),
+		Vec: promauto.NewCounterVec(prometheus.CounterOpts{
+			Name: counterConfig.GetName(),
+			Help: counterConfig.GetHelp(),
+		}, counterConfig.GetLabels()),
+		LabelKeys:   counterConfig.GetLabels(),
+		LabelValues: map[string]*DnstapPrometheusOutputMetricsValues{},
+		Expire:      counterConfig.GetExpireSec(),
+		Interval:    counterConfig.GetExpireInterval(),
+	}
 }
 
-func NewDtapCounterVec(opts prometheus.CounterOpts) *prometheus.CounterVec {
-	s := DnstapFlatT{}
-	t := reflect.TypeOf(s)
-	var labels []string
+func (d *DnstapPrometheusOutputMetrics) Inc(values []string) {
+	d.Vec.WithLabelValues(values...).Inc()
+	d.LabelValues[strings.Join(values, ",")] = &DnstapPrometheusOutputMetricsValues{
+		Values:     values,
+		LastUpdate: time.Now(),
+	}
+}
 
-	for i := 0; i < t.NumField(); i++ {
-		field := t.Field(i)
-		j := field.Tag.Get("json")
-		labels = append(labels, j)
+func (d *DnstapPrometheusOutputMetrics) Flush(ctx context.Context) {
+	ticker := time.NewTicker(time.Second * time.Duration(d.Interval))
+	for {
+		select {
+		case <-ctx.Done():
+			return
+		case <-ticker.C:
+			for k, value := range d.LabelValues {
+				if time.Now().Sub(value.LastUpdate) > time.Second*time.Duration(d.Expire) {
+					d.Vec.DeleteLabelValues(value.Values...)
+					delete(d.LabelValues, k)
+				}
+			}
+		}
 	}
-	return promauto.NewCounterVec(opts, labels)
 }
 
 func NewDnstapPrometheusOutput(config *OutputPrometheus, params *DnstapOutputParams) *DnstapOutput {
@@ -56,20 +112,20 @@ func NewDnstapPrometheusOutput(config *OutputPrometheus, params *DnstapOutputPar
 		Metrics: []*DnstapPrometheusOutputMetrics{},
 	}
 	for _, counterConfig := range config.GetCounters() {
-		counter := &DnstapPrometheusOutputMetrics{
-			Vec: NewDtapCounterVec(prometheus.CounterOpts{
-				Name: counterConfig.GetName(),
-				Help: counterConfig.GetHelp(),
-			}),
-			Labels: counterConfig.GetLabels(),
-		}
-		p.Metrics = append(p.Metrics, counter)
+		p.Metrics = append(p.Metrics, NewDnstapPrometheusOutputMetrics(counterConfig))
 	}
 	params.Handler = p
 	return NewDnstapOutput(params)
 }
 
 func (o *DnstapPrometheusOutput) open() error {
+	for _, metrics := range o.Metrics {
+		if metrics.GetInterval() > 0 && metrics.GetExpire() > 0 {
+			ctx, cancelFunc := context.WithCancel(context.Background())
+			metrics.CancelFunc = cancelFunc
+			go metrics.Flush(ctx)
+		}
+	}
 	return nil
 }
 
@@ -85,31 +141,47 @@ func (o *DnstapPrometheusOutput) write(frame []byte) error {
 	e := reflect.ValueOf(data).Elem()
 	m := make(map[string]string)
 	for i := 0; i < e.NumField(); i++ {
-		field := e.Type().Field(i).Tag.Get("json")
+		field := e.Type().Field(i).Name
 		value := e.Field(i).Interface()
+		if value == nil {
+			continue
+		}
 		switch v := value.(type) {
 		case string:
 			m[field] = v
 		case uint32:
 			m[field] = strconv.Itoa(int(v))
 		case uint16:
 			m[field] = strconv.Itoa(int(v))
+		case bool:
+			if v {
+				m[field] = "1"
+			} else {
+				m[field] = "0"
+			}
 		case fmt.Stringer:
 			m[field] = v.String()
 		}
 	}
 
 	for _, counter := range o.Metrics {
-		labels := map[string]string{}
-		for _, l := range counter.Labels {
+		labelValues := make([]string, 0, len(counter.LabelKeys))
+		for _, l := range counter.LabelKeys {
 			if v, ok := m[l]; ok {
-				labels[l] = v
+				labelValues = append(labelValues, v)
+			} else {
+				log.Warnf("can't get metrics: %v, %v", l, counter.Name)
 			}
 		}
-		counter.Vec.With(labels).Inc()
+		counter.Inc(labelValues)
 	}
 	return nil
 }
 
 func (o *DnstapPrometheusOutput) close() {
+	for _, metrics := range o.Metrics {
+		if metrics.CancelFunc != nil {
+			metrics.CancelFunc()
+		}
+	}
 }

net.go

@@ -29,6 +29,9 @@ type Net struct {
 }
 
 func (n *Net) String() string {
+	if n == nil || n.IP == nil {
+		return "<nil>"
+	}
 	return n.IP.String() + "/" + strconv.Itoa(n.PrefixLength)
 }