From 468cf908a806fa57244c0fc2edfac078c7d463e1 Mon Sep 17 00:00:00 2001 From: Shubham Aggarwal Date: Wed, 13 Apr 2022 13:02:40 +0530 Subject: [PATCH] Prometheus Metrics (#123) - Expose Prometheus metrics over http-listen-addr - Prometheus cluster metrics aggregator - Better replication lag metrics - extras/prometheus-dashboard Co-authored-by: Kinshuk Co-authored-by: Sangam Kumar Jain Co-authored-by: Shubham Aggarwal --- .../dkv-client/src/test/resources/Procfile | 42 +- cmd/dkvsrv/main.go | 162 +++- dkvsrv.yaml | 15 +- extras/prometheus-dashboard/.env | 1 + .../prometheus-dashboard/docker-compose.yml | 28 + .../provisioning/dashboards/dashboard.yml | 12 + .../provisioning/dashboards/dkv-master.json | 581 +++++++++++++ .../provisioning/dashboards/dkv-slave.json | 772 ++++++++++++++++++ .../provisioning/datasources/datasource.yml | 11 + .../prometheus/prometheus.yml | 11 + go.mod | 4 +- go.sum | 2 + internal/discovery/client.go | 13 +- internal/discovery/client_test.go | 11 +- internal/discovery/service.go | 1 + internal/discovery/service_test.go | 11 +- internal/master/service.go | 53 +- internal/master/ss_service_test.go | 1 + internal/opts/config.go | 7 +- internal/opts/serverOpts.go | 2 + internal/slave/service.go | 75 +- internal/slave/service_test.go | 2 + internal/stats/aggregate/aggregator.go | 178 ++++ internal/stats/aggregate/collector.go | 182 +++++ internal/stats/models.go | 113 +++ internal/stats/prometheus.go | 84 ++ internal/stats/streamer.go | 61 ++ internal/storage/badger/store.go | 38 +- internal/storage/rocksdb/store.go | 34 + internal/storage/store.go | 25 + pkg/serverpb/admin.pb.go | 150 ++-- pkg/serverpb/admin.proto | 3 + 32 files changed, 2545 insertions(+), 140 deletions(-) create mode 100644 extras/prometheus-dashboard/.env create mode 100644 extras/prometheus-dashboard/docker-compose.yml create mode 100644 extras/prometheus-dashboard/grafana/provisioning/dashboards/dashboard.yml create mode 100644 extras/prometheus-dashboard/grafana/provisioning/dashboards/dkv-master.json create mode 100644 extras/prometheus-dashboard/grafana/provisioning/dashboards/dkv-slave.json create mode 100644 extras/prometheus-dashboard/grafana/provisioning/datasources/datasource.yml create mode 100644 extras/prometheus-dashboard/prometheus/prometheus.yml create mode 100644 internal/stats/aggregate/aggregator.go create mode 100644 internal/stats/aggregate/collector.go create mode 100644 internal/stats/models.go create mode 100644 internal/stats/prometheus.go create mode 100644 internal/stats/streamer.go diff --git a/clients/java/dkv-client/src/test/resources/Procfile b/clients/java/dkv-client/src/test/resources/Procfile index 214c7ad8..633d2be4 100644 --- a/clients/java/dkv-client/src/test/resources/Procfile +++ b/clients/java/dkv-client/src/test/resources/Procfile @@ -1,28 +1,28 @@ # run using `goreman -exit-on-error start` -dkv_discovery1:./bin/dkvsrv --role discovery --listen-addr 127.0.0.1:8001 --node-name d1 --nexus-node-url "http://127.0.0.1:7021" --nexus-cluster-url "http://127.0.0.1:7021,http://127.0.0.1:7022,http://127.0.0.1:7023" --config clients/java/dkv-client/src/test/resources/dkv_config.yaml -dkv_discovery2:./bin/dkvsrv --role discovery --listen-addr 127.0.0.1:8002 --node-name d2 --nexus-node-url "http://127.0.0.1:7022" --nexus-cluster-url "http://127.0.0.1:7021,http://127.0.0.1:7022,http://127.0.0.1:7023" --config clients/java/dkv-client/src/test/resources/dkv_config.yaml -dkv_discovery3:./bin/dkvsrv --role discovery --listen-addr 127.0.0.1:8003 --node-name d3 --nexus-node-url "http://127.0.0.1:7023" --nexus-cluster-url "http://127.0.0.1:7021,http://127.0.0.1:7022,http://127.0.0.1:7023" --config clients/java/dkv-client/src/test/resources/dkv_config.yaml +dkv_discovery1:./bin/dkvsrv --role discovery --listen-addr 127.0.0.1:8001 --http-listen-addr 127.0.0.1:8005 --node-name d1 --nexus-node-url "http://127.0.0.1:7021" --nexus-cluster-url "http://127.0.0.1:7021,http://127.0.0.1:7022,http://127.0.0.1:7023" --config clients/java/dkv-client/src/test/resources/dkv_config.yaml +dkv_discovery2:./bin/dkvsrv --role discovery --listen-addr 127.0.0.1:8002 --http-listen-addr 127.0.0.1:8006 --node-name d2 --nexus-node-url "http://127.0.0.1:7022" --nexus-cluster-url "http://127.0.0.1:7021,http://127.0.0.1:7022,http://127.0.0.1:7023" --config clients/java/dkv-client/src/test/resources/dkv_config.yaml +dkv_discovery3:./bin/dkvsrv --role discovery --listen-addr 127.0.0.1:8003 --http-listen-addr 127.0.0.1:8007 --node-name d3 --nexus-node-url "http://127.0.0.1:7023" --nexus-cluster-url "http://127.0.0.1:7021,http://127.0.0.1:7022,http://127.0.0.1:7023" --config clients/java/dkv-client/src/test/resources/dkv_config.yaml -dkv_smaster1:./bin/dkvsrv --role master --listen-addr 127.0.0.1:7080 --node-name m1 --dc-id dc1 --database s0 --config clients/java/dkv-client/src/test/resources/standalone_config.yaml -dkv_smaster2:./bin/dkvsrv --role master --listen-addr 127.0.0.1:8080 --node-name m2 --dc-id dc1 --database s1 --config clients/java/dkv-client/src/test/resources/standalone_config.yaml -dkv_smaster3:./bin/dkvsrv --role master --listen-addr 127.0.0.1:9080 --node-name m3 --dc-id dc1 --database s2 --config clients/java/dkv-client/src/test/resources/standalone_config.yaml +dkv_smaster1:./bin/dkvsrv --role master --listen-addr 127.0.0.1:7080 --http-listen-addr 127.0.0.1:7081 --node-name m1 --dc-id dc1 --database s0 --config clients/java/dkv-client/src/test/resources/standalone_config.yaml +dkv_smaster2:./bin/dkvsrv --role master --listen-addr 127.0.0.1:8080 --http-listen-addr 127.0.0.1:8081 --node-name m2 --dc-id dc1 --database s1 --config clients/java/dkv-client/src/test/resources/standalone_config.yaml +dkv_smaster3:./bin/dkvsrv --role master --listen-addr 127.0.0.1:9080 --http-listen-addr 127.0.0.1:9081 --node-name m3 --dc-id dc1 --database s2 --config clients/java/dkv-client/src/test/resources/standalone_config.yaml -dkv_slave1a:./bin/dkvsrv --diskless --db-engine badger --role slave --listen-addr 127.0.0.1:7091 --node-name s1a --database s0 --config clients/java/dkv-client/src/test/resources/standalone_config.yaml -dkv_slave1b:./bin/dkvsrv --diskless --db-engine badger --role slave --listen-addr 127.0.0.1:7092 --node-name s1b --database s0 --config clients/java/dkv-client/src/test/resources/standalone_config.yaml -dkv_slave1c:./bin/dkvsrv --diskless --db-engine badger --role slave --listen-addr 127.0.0.1:7093 --node-name s1c --database s0 --config clients/java/dkv-client/src/test/resources/standalone_config.yaml -dkv_slave1d:./bin/dkvsrv --diskless --db-engine badger --role slave --listen-addr 127.0.0.1:7094 --node-name s1d --database s0 --config clients/java/dkv-client/src/test/resources/standalone_config.yaml +dkv_slave1a:./bin/dkvsrv --diskless --db-engine badger --role slave --listen-addr 127.0.0.1:7091 --http-listen-addr 127.0.0.1:7095 --node-name s1a --database s0 --config clients/java/dkv-client/src/test/resources/standalone_config.yaml +dkv_slave1b:./bin/dkvsrv --diskless --db-engine badger --role slave --listen-addr 127.0.0.1:7092 --http-listen-addr 127.0.0.1:7096 --node-name s1b --database s0 --config clients/java/dkv-client/src/test/resources/standalone_config.yaml +dkv_slave1c:./bin/dkvsrv --diskless --db-engine badger --role slave --listen-addr 127.0.0.1:7093 --http-listen-addr 127.0.0.1:7097 --node-name s1c --database s0 --config clients/java/dkv-client/src/test/resources/standalone_config.yaml +dkv_slave1d:./bin/dkvsrv --diskless --db-engine badger --role slave --listen-addr 127.0.0.1:7094 --http-listen-addr 127.0.0.1:7098 --node-name s1d --database s0 --config clients/java/dkv-client/src/test/resources/standalone_config.yaml -dkv_slave2a:./bin/dkvsrv --diskless --db-engine badger --role slave --listen-addr 127.0.0.1:8091 --node-name s2a --database s1 --config clients/java/dkv-client/src/test/resources/standalone_config.yaml -dkv_slave2b:./bin/dkvsrv --diskless --db-engine badger --role slave --listen-addr 127.0.0.1:8092 --node-name s2b --database s1 --config clients/java/dkv-client/src/test/resources/standalone_config.yaml -dkv_slave2c:./bin/dkvsrv --diskless --db-engine badger --role slave --listen-addr 127.0.0.1:8093 --node-name s2c --database s1 --config clients/java/dkv-client/src/test/resources/standalone_config.yaml -dkv_slave2d:./bin/dkvsrv --diskless --db-engine badger --role slave --listen-addr 127.0.0.1:8094 --node-name s2d --database s1 --config clients/java/dkv-client/src/test/resources/standalone_config.yaml +dkv_slave2a:./bin/dkvsrv --diskless --db-engine badger --role slave --listen-addr 127.0.0.1:8091 --http-listen-addr 127.0.0.1:8095 --node-name s2a --database s1 --config clients/java/dkv-client/src/test/resources/standalone_config.yaml +dkv_slave2b:./bin/dkvsrv --diskless --db-engine badger --role slave --listen-addr 127.0.0.1:8092 --http-listen-addr 127.0.0.1:8096 --node-name s2b --database s1 --config clients/java/dkv-client/src/test/resources/standalone_config.yaml +dkv_slave2c:./bin/dkvsrv --diskless --db-engine badger --role slave --listen-addr 127.0.0.1:8093 --http-listen-addr 127.0.0.1:8097 --node-name s2c --database s1 --config clients/java/dkv-client/src/test/resources/standalone_config.yaml +dkv_slave2d:./bin/dkvsrv --diskless --db-engine badger --role slave --listen-addr 127.0.0.1:8094 --http-listen-addr 127.0.0.1:8098 --node-name s2d --database s1 --config clients/java/dkv-client/src/test/resources/standalone_config.yaml -dkv_slave3a:./bin/dkvsrv --diskless --db-engine badger --role slave --listen-addr 127.0.0.1:9091 --node-name s3a --database s2 --config clients/java/dkv-client/src/test/resources/standalone_config.yaml -dkv_slave3b:./bin/dkvsrv --diskless --db-engine badger --role slave --listen-addr 127.0.0.1:9092 --node-name s3b --database s2 --config clients/java/dkv-client/src/test/resources/standalone_config.yaml -dkv_slave3c:./bin/dkvsrv --diskless --db-engine badger --role slave --listen-addr 127.0.0.1:9093 --node-name s3c --database s2 --config clients/java/dkv-client/src/test/resources/standalone_config.yaml -dkv_slave3d:./bin/dkvsrv --diskless --db-engine badger --role slave --listen-addr 127.0.0.1:9094 --node-name s3d --database s2 --config clients/java/dkv-client/src/test/resources/standalone_config.yaml +dkv_slave3a:./bin/dkvsrv --diskless --db-engine badger --role slave --listen-addr 127.0.0.1:9091 --http-listen-addr 127.0.0.1:9095 --node-name s3a --database s2 --config clients/java/dkv-client/src/test/resources/standalone_config.yaml +dkv_slave3b:./bin/dkvsrv --diskless --db-engine badger --role slave --listen-addr 127.0.0.1:9092 --http-listen-addr 127.0.0.1:9096 --node-name s3b --database s2 --config clients/java/dkv-client/src/test/resources/standalone_config.yaml +dkv_slave3c:./bin/dkvsrv --diskless --db-engine badger --role slave --listen-addr 127.0.0.1:9093 --http-listen-addr 127.0.0.1:9097 --node-name s3c --database s2 --config clients/java/dkv-client/src/test/resources/standalone_config.yaml +dkv_slave3d:./bin/dkvsrv --diskless --db-engine badger --role slave --listen-addr 127.0.0.1:9094 --http-listen-addr 127.0.0.1:9098 --node-name s3d --database s2 --config clients/java/dkv-client/src/test/resources/standalone_config.yaml -dkv_master1:./bin/dkvsrv --role master --listen-addr 127.0.0.1:6080 --node-name s0 --nexus-node-url "http://127.0.0.1:8021" --nexus-cluster-url "http://127.0.0.1:8021,http://127.0.0.1:8022,http://127.0.0.1:8023" --config clients/java/dkv-client/src/test/resources/dkv_config.yaml -dkv_master2:./bin/dkvsrv --role master --listen-addr 127.0.0.1:6081 --node-name s1 --nexus-node-url "http://127.0.0.1:8022" --nexus-cluster-url "http://127.0.0.1:8021,http://127.0.0.1:8022,http://127.0.0.1:8023" --config clients/java/dkv-client/src/test/resources/dkv_config.yaml -dkv_master3:./bin/dkvsrv --role master --listen-addr 127.0.0.1:6082 --node-name s2 --nexus-node-url "http://127.0.0.1:8023" --nexus-cluster-url "http://127.0.0.1:8021,http://127.0.0.1:8022,http://127.0.0.1:8023" --config clients/java/dkv-client/src/test/resources/dkv_config.yaml +dkv_master1:./bin/dkvsrv --role master --listen-addr 127.0.0.1:6080 --http-listen-addr 127.0.0.1:6085 --node-name s0 --nexus-node-url "http://127.0.0.1:8021" --nexus-cluster-url "http://127.0.0.1:8021,http://127.0.0.1:8022,http://127.0.0.1:8023" --config clients/java/dkv-client/src/test/resources/dkv_config.yaml +dkv_master2:./bin/dkvsrv --role master --listen-addr 127.0.0.1:6081 --http-listen-addr 127.0.0.1:6086 --node-name s1 --nexus-node-url "http://127.0.0.1:8022" --nexus-cluster-url "http://127.0.0.1:8021,http://127.0.0.1:8022,http://127.0.0.1:8023" --config clients/java/dkv-client/src/test/resources/dkv_config.yaml +dkv_master3:./bin/dkvsrv --role master --listen-addr 127.0.0.1:6082 --http-listen-addr 127.0.0.1:6087 --node-name s2 --nexus-node-url "http://127.0.0.1:8023" --nexus-cluster-url "http://127.0.0.1:8021,http://127.0.0.1:8022,http://127.0.0.1:8023" --config clients/java/dkv-client/src/test/resources/dkv_config.yaml diff --git a/cmd/dkvsrv/main.go b/cmd/dkvsrv/main.go index 9f9334aa..e563f8f3 100644 --- a/cmd/dkvsrv/main.go +++ b/cmd/dkvsrv/main.go @@ -1,7 +1,10 @@ package main import ( + "encoding/json" "fmt" + "io" + "io/ioutil" "log" "net" "net/http" @@ -12,11 +15,17 @@ import ( "strings" "syscall" + "github.com/gorilla/mux" + "github.com/prometheus/client_golang/prometheus" + "github.com/prometheus/client_golang/prometheus/promhttp" + "gopkg.in/ini.v1" + "github.com/flipkart-incubator/dkv/internal/discovery" "github.com/flipkart-incubator/dkv/internal/master" "github.com/flipkart-incubator/dkv/internal/opts" "github.com/flipkart-incubator/dkv/internal/slave" "github.com/flipkart-incubator/dkv/internal/stats" + "github.com/flipkart-incubator/dkv/internal/stats/aggregate" "github.com/flipkart-incubator/dkv/internal/storage" "github.com/flipkart-incubator/dkv/internal/storage/badger" "github.com/flipkart-incubator/dkv/internal/storage/rocksdb" @@ -32,9 +41,8 @@ import ( "go.uber.org/zap/zapcore" "google.golang.org/grpc" "google.golang.org/grpc/reflection" - "gopkg.in/ini.v1" - _ "net/http/pprof" + "net/http/pprof" ) type dkvSrvrRole string @@ -64,10 +72,15 @@ var ( verboseLogging bool accessLogger *zap.Logger dkvLogger *zap.Logger - pprofEnable bool // Other vars - statsCli stats.Client + pprofEnable bool + statsCli stats.Client + promRegistry prometheus.Registerer + statsStreamer *stats.StatStreamer + + discoveryClient discovery.Client + statAggregatorRegistry *aggregate.StatAggregatorRegistry ) func init() { @@ -92,13 +105,7 @@ func main() { setupAccessLogger() setFlagsForNexusDirs() setupStats() - - if pprofEnable { - go func() { - log.Printf("[INFO] Starting pprof on port 6060\n") - log.Println(http.ListenAndServe("0.0.0.0:6060", nil)) - }() - } + go setupHttpServer() kvs, cp, ca, br := newKVStore() grpcSrvr, lstnr := newGrpcServerListener() @@ -107,13 +114,21 @@ func main() { //srvrRole.printFlags() // Create the region info which is passed to DKVServer - nodeAddr, err := nodeAddress() + nodeAddr, err := nodeAddress(config.ListenAddr) + if err != nil { + log.Panicf("Failed to parse GRPC Listen Address %v.", err) + } + + //HTTP listen Address + nodeHTTPAddr, err := nodeAddress(config.HttpListenAddr) if err != nil { - log.Panicf("Failed to detect IP Address %v.", err) + log.Panicf("Failed to parse HTTP Listen Address %v.", err) } + regionInfo := &serverpb.RegionInfo{ DcID: config.DcID, NodeAddress: nodeAddr.Host, + HttpAddress: nodeHTTPAddr.Host, Database: config.Database, VBucket: config.VBucket, Status: serverpb.RegionStatus_INACTIVE, @@ -125,9 +140,9 @@ func main() { Logger: dkvLogger, HealthCheckTickerInterval: opts.DefaultHealthCheckTickterInterval, //to be exposed later via app.conf StatsCli: statsCli, + PrometheusRegistry: promRegistry, } - var discoveryClient discovery.Client if srvrRole != noRole && srvrRole != discoveryRole { var err error discoveryClient, err = newDiscoveryClient() @@ -327,6 +342,10 @@ func setupStats() { } else { statsCli = stats.NewNoOpClient() } + promRegistry = stats.NewPromethousRegistry() + statsStreamer = stats.NewStatStreamer() + statAggregatorRegistry = aggregate.NewStatAggregatorRegistry() + go statsStreamer.Run() } func newKVStore() (storage.KVStore, storage.ChangePropagator, storage.ChangeApplier, storage.Backupable) { @@ -353,7 +372,8 @@ func newKVStore() (storage.KVStore, storage.ChangePropagator, storage.ChangeAppl rocksdb.WithCacheSize(config.BlockCacheSize), rocksdb.WithRocksDBConfig(config.DbEngineIni), rocksdb.WithLogger(dkvLogger), - rocksdb.WithStats(statsCli)) + rocksdb.WithStats(statsCli), + rocksdb.WithPromStats(promRegistry)) if err != nil { dkvLogger.Panic("RocksDB engine init failed", zap.Error(err)) } @@ -368,6 +388,7 @@ func newKVStore() (storage.KVStore, storage.ChangePropagator, storage.ChangeAppl badger.WithBadgerConfig(config.DbEngineIni), badger.WithLogger(dkvLogger), badger.WithStats(statsCli), + badger.WithPromStats(promRegistry), } if config.DisklessMode { bdbOpts = append(bdbOpts, badger.WithInMemory()) @@ -452,8 +473,8 @@ func newDiscoveryClient() (discovery.Client, error) { } -func nodeAddress() (*url.URL, error) { - ip, port, err := net.SplitHostPort(config.ListenAddr) +func nodeAddress(listenAddress string) (*url.URL, error) { + ip, port, err := net.SplitHostPort(listenAddress) if err != nil { return nil, err } @@ -477,3 +498,110 @@ func nodeAddress() (*url.URL, error) { ep := url.URL{Host: fmt.Sprintf("%s:%s", ip, port)} return &ep, nil } + +func setupHttpServer() { + router := mux.NewRouter() + router.Handle("/metrics", promhttp.Handler()) + router.HandleFunc("/metrics/json", jsonMetricHandler) + + router.HandleFunc("/metrics/stream", statsStreamHandler) + if toDKVSrvrRole(config.DbRole) == masterRole { + // Should be enabled only for discovery server ? + router.HandleFunc("/metrics/cluster", clusterMetricsHandler) + } + + //Pprof + if pprofEnable { + log.Printf("[INFO] Enabling pprof...\n") + router.HandleFunc("/debug/pprof/", pprof.Index) + router.HandleFunc("/debug/pprof/cmdline", pprof.Cmdline) + router.HandleFunc("/debug/pprof/profile", pprof.Profile) + router.HandleFunc("/debug/pprof/symbol", pprof.Symbol) + router.HandleFunc("/debug/pprof/trace", pprof.Trace) + } + + http.Handle("/", router) + http.ListenAndServe(config.HttpListenAddr, nil) +} + +func jsonMetricHandler(w http.ResponseWriter, r *http.Request) { + w.Header().Set("Content-Type", "application/json") + metrics, _ := stats.GetMetrics() + json.NewEncoder(w).Encode(metrics) +} + +func statsStreamHandler(w http.ResponseWriter, r *http.Request) { + if f, ok := w.(http.Flusher); !ok { + http.Error(w, "Streaming unsupported!", http.StatusInternalServerError) + return + } else { + // Set the headers related to event streaming. + w.Header().Set("Content-Type", "text/event-stream") + w.Header().Set("Cache-Control", "no-cache") + w.Header().Set("Connection", "keep-alive") + w.Header().Set("Access-Control-Allow-Origin", r.Header.Get("Origin")) + w.Header().Set("Access-Control-Allow-Methods", "POST, GET, OPTIONS, PUT, DELETE") + + statChannel := make(chan stats.DKVMetrics, 5) + channelId := statsStreamer.Register(statChannel) + defer func() { + io.Copy(ioutil.Discard, r.Body) + r.Body.Close() + }() + // Listen to the closing of the http connection via the CloseNotifier + log.Printf("[INFO] Starting Metrics Stream %v\n", channelId) + for { + select { + case stat := <-statChannel: + statJson, _ := json.Marshal(stat) + fmt.Fprintf(w, "data: %s\n\n", statJson) + f.Flush() + case <-r.Context().Done(): + statsStreamer.DeRegister(channelId) + log.Printf("[INFO] Closing Metics Stream %v\n", channelId) + return + } + } + } +} + +func clusterMetricsHandler(w http.ResponseWriter, r *http.Request) { + regions, err := discoveryClient.GetClusterStatus("", "") + if err != nil { + http.Error(w, "Unable to discover peers!", http.StatusInternalServerError) + return + } + if f, ok := w.(http.Flusher); !ok { + http.Error(w, "Streaming unsupported!", http.StatusInternalServerError) + return + } else { + // Set the headers related to event streaming. + w.Header().Set("Content-Type", "text/event-stream") + w.Header().Set("Cache-Control", "no-cache") + w.Header().Set("Connection", "keep-alive") + w.Header().Set("Access-Control-Allow-Origin", r.Header.Get("Origin")) + w.Header().Set("Access-Control-Allow-Methods", "POST, GET, OPTIONS, PUT, DELETE") + + statChannel := make(chan map[string]*stats.DKVMetrics, 5) + channelId := statAggregatorRegistry.Register(regions, func(region *serverpb.RegionInfo) string { return region.Database }, statChannel) + defer func() { + io.Copy(ioutil.Discard, r.Body) + r.Body.Close() + }() + + // Listen to the closing of the http connection via the CloseNotifier + log.Printf("[INFO] Starting ClusterMetics Stream %v\n", channelId) + for { + select { + case stat := <-statChannel: + statJson, _ := json.Marshal(stat) + fmt.Fprintf(w, "data: %s\n\n", statJson) + f.Flush() + case <-r.Context().Done(): + log.Printf("[INFO] Closing ClusterMetics Stream %v\n", channelId) + statAggregatorRegistry.DeRegister(channelId) + return + } + } + } +} diff --git a/dkvsrv.yaml b/dkvsrv.yaml index 852ff56d..4dbbb353 100644 --- a/dkvsrv.yaml +++ b/dkvsrv.yaml @@ -1,11 +1,12 @@ -node-name : "" #Name of the current Node Name -listen-addr : "0.0.0.0:8080" #listen address -role : "none" #Role of the node - master|slave|standalone -pprof : false #Enable profiling -statsd-addr : "" #StatsdD Address -verbose : false # Enable verbose logging. By default, only warnings and errors are logged. -log-level : "warn" # Use exact log level. +node-name : "" #Name of the current Node Name +listen-addr : "0.0.0.0:8080" #listen address +http-listen-addr : "0.0.0.0:8081" # http listen address +role : "none" #Role of the node - master|slave|standalone +pprof : false #Enable profiling +statsd-addr : "" #StatsdD Address +verbose : false # Enable verbose logging. By default, only warnings and errors are logged. +log-level : "warn" # Use exact log level. db-engine : "rocksdb" #Underlying DB engine for storing data - badger|rocksdb db-engine-ini : "rocksdb.ini" #An .ini file for configuring the underlying storage engine. Refer badger.ini or rocks.ini for more details. diff --git a/extras/prometheus-dashboard/.env b/extras/prometheus-dashboard/.env new file mode 100644 index 00000000..bd6e8ff3 --- /dev/null +++ b/extras/prometheus-dashboard/.env @@ -0,0 +1 @@ +PROM_DIR=./prometheus \ No newline at end of file diff --git a/extras/prometheus-dashboard/docker-compose.yml b/extras/prometheus-dashboard/docker-compose.yml new file mode 100644 index 00000000..aa314223 --- /dev/null +++ b/extras/prometheus-dashboard/docker-compose.yml @@ -0,0 +1,28 @@ +volumes: + prometheus_data: {} + grafana_data: {} + +services: + prometheus: + image: prom/prometheus + container_name: dkv-prometheus + environment: + - PROM_DIR=./prometheus + command: + --web.enable-lifecycle + --config.file=/etc/prometheus/prometheus.yml + volumes: + - ${PROM_DIR:?err}:/etc/prometheus + - prometheus_data:/var/lib/prometheus + ports: + - 9090:9090 + + grafana: + image: grafana/grafana-oss + container_name: dkv-grafana + volumes: + - grafana_data:/var/lib/grafana + - ./grafana/provisioning/dashboards:/etc/grafana/provisioning/dashboards + - ./grafana/provisioning/datasources:/etc/grafana/provisioning/datasources + ports: + - 3000:3000 \ No newline at end of file diff --git a/extras/prometheus-dashboard/grafana/provisioning/dashboards/dashboard.yml b/extras/prometheus-dashboard/grafana/provisioning/dashboards/dashboard.yml new file mode 100644 index 00000000..4ba07fdc --- /dev/null +++ b/extras/prometheus-dashboard/grafana/provisioning/dashboards/dashboard.yml @@ -0,0 +1,12 @@ +apiVersion: 1 + +providers: + - name: 'dkv' + orgId: 1 + folder: '' + type: file + disableDeletion: false + editable: true + allowUiUpdates: false + options: + path: /etc/grafana/provisioning/dashboards \ No newline at end of file diff --git a/extras/prometheus-dashboard/grafana/provisioning/dashboards/dkv-master.json b/extras/prometheus-dashboard/grafana/provisioning/dashboards/dkv-master.json new file mode 100644 index 00000000..cc942f83 --- /dev/null +++ b/extras/prometheus-dashboard/grafana/provisioning/dashboards/dkv-master.json @@ -0,0 +1,581 @@ +{ + "annotations": { + "list": [ + { + "builtIn": 1, + "datasource": "-- Grafana --", + "enable": true, + "hide": true, + "iconColor": "rgba(0, 211, 255, 1)", + "name": "Annotations & Alerts", + "target": { + "limit": 100, + "matchAny": false, + "tags": [], + "type": "dashboard" + }, + "type": "dashboard" + } + ] + }, + "editable": true, + "fiscalYearStartMonth": 0, + "graphTooltip": 0, + "links": [], + "liveNow": false, + "panels": [ + { + "gridPos": { + "h": 1, + "w": 24, + "x": 0, + "y": 0 + }, + "id": 4, + "title": "Latency Metrics", + "type": "row" + }, + { + "fieldConfig": { + "defaults": { + "color": { + "mode": "palette-classic" + }, + "custom": { + "axisLabel": "", + "axisPlacement": "auto", + "barAlignment": 0, + "drawStyle": "line", + "fillOpacity": 0, + "gradientMode": "none", + "hideFrom": { + "legend": false, + "tooltip": false, + "viz": false + }, + "lineInterpolation": "linear", + "lineWidth": 1, + "pointSize": 5, + "scaleDistribution": { + "type": "linear" + }, + "showPoints": "auto", + "spanNulls": false, + "stacking": { + "group": "A", + "mode": "none" + }, + "thresholdsStyle": { + "mode": "off" + } + }, + "mappings": [], + "thresholds": { + "mode": "absolute", + "steps": [ + { + "color": "green", + "value": null + }, + { + "color": "red", + "value": 80 + } + ] + }, + "unit": "s" + }, + "overrides": [] + }, + "gridPos": { + "h": 8, + "w": 12, + "x": 0, + "y": 1 + }, + "id": 10, + "options": { + "legend": { + "calcs": [], + "displayMode": "list", + "placement": "bottom" + }, + "tooltip": { + "mode": "single", + "sort": "none" + } + }, + "targets": [ + { + "datasource": { + "type": "prometheus", + "uid": "PBFA97CFB590B2093" + }, + "exemplar": true, + "expr": "dkv_latency{role=\"master\", Ops=\"put\",quantile=\"0.5\"}", + "interval": "", + "legendFormat": "50p", + "refId": "A" + }, + { + "datasource": { + "type": "prometheus", + "uid": "PBFA97CFB590B2093" + }, + "exemplar": true, + "expr": "dkv_latency{role=\"master\", Ops=\"put\",quantile=\"0.9\"}", + "hide": false, + "interval": "", + "legendFormat": "90p", + "refId": "B" + }, + { + "datasource": { + "type": "prometheus", + "uid": "PBFA97CFB590B2093" + }, + "exemplar": true, + "expr": "dkv_latency{role=\"master\", Ops=\"put\",quantile=\"0.99\"}", + "hide": false, + "interval": "", + "legendFormat": "99p", + "refId": "C" + } + ], + "title": "PUT Latency", + "type": "timeseries" + }, + { + "fieldConfig": { + "defaults": { + "color": { + "mode": "palette-classic" + }, + "custom": { + "axisLabel": "", + "axisPlacement": "auto", + "barAlignment": 0, + "drawStyle": "line", + "fillOpacity": 0, + "gradientMode": "none", + "hideFrom": { + "legend": false, + "tooltip": false, + "viz": false + }, + "lineInterpolation": "linear", + "lineWidth": 1, + "pointSize": 5, + "scaleDistribution": { + "type": "linear" + }, + "showPoints": "auto", + "spanNulls": false, + "stacking": { + "group": "A", + "mode": "none" + }, + "thresholdsStyle": { + "mode": "off" + } + }, + "mappings": [], + "thresholds": { + "mode": "absolute", + "steps": [ + { + "color": "green", + "value": null + }, + { + "color": "red", + "value": 80 + } + ] + }, + "unit": "s" + }, + "overrides": [] + }, + "gridPos": { + "h": 8, + "w": 12, + "x": 12, + "y": 1 + }, + "id": 12, + "options": { + "legend": { + "calcs": [], + "displayMode": "list", + "placement": "bottom" + }, + "tooltip": { + "mode": "single", + "sort": "none" + } + }, + "targets": [ + { + "datasource": { + "type": "prometheus", + "uid": "PBFA97CFB590B2093" + }, + "exemplar": true, + "expr": "dkv_latency{role=\"master\", Ops=\"iter\",quantile=\"0.5\"}", + "interval": "", + "legendFormat": "50p", + "refId": "A" + }, + { + "datasource": { + "type": "prometheus", + "uid": "PBFA97CFB590B2093" + }, + "exemplar": true, + "expr": "dkv_latency{role=\"master\", Ops=\"iter\",quantile=\"0.9\"}", + "hide": false, + "interval": "", + "legendFormat": "90p", + "refId": "B" + }, + { + "datasource": { + "type": "prometheus", + "uid": "PBFA97CFB590B2093" + }, + "exemplar": true, + "expr": "dkv_latency{role=\"master\", Ops=\"iter\",quantile=\"0.99\"}", + "hide": false, + "interval": "", + "legendFormat": "99p", + "refId": "C" + } + ], + "title": "ITER Latency", + "type": "timeseries" + }, + { + "collapsed": false, + "gridPos": { + "h": 1, + "w": 24, + "x": 0, + "y": 9 + }, + "id": 2, + "panels": [], + "title": "Count Metrics", + "type": "row" + }, + { + "fieldConfig": { + "defaults": { + "color": { + "mode": "thresholds" + }, + "mappings": [], + "thresholds": { + "mode": "absolute", + "steps": [ + { + "color": "green", + "value": null + } + ] + } + }, + "overrides": [] + }, + "gridPos": { + "h": 8, + "w": 12, + "x": 0, + "y": 10 + }, + "id": 16, + "options": { + "colorMode": "value", + "graphMode": "area", + "justifyMode": "auto", + "orientation": "auto", + "reduceOptions": { + "calcs": [ + "lastNotNull" + ], + "fields": "", + "values": false + }, + "textMode": "auto" + }, + "pluginVersion": "8.4.0", + "targets": [ + { + "datasource": { + "type": "prometheus", + "uid": "PBFA97CFB590B2093" + }, + "exemplar": true, + "expr": "dkv_latency_count{role=\"master\", Ops=\"put\"}", + "interval": "", + "legendFormat": "", + "refId": "A" + } + ], + "title": "PUT One Minute Rate", + "type": "stat" + }, + { + "fieldConfig": { + "defaults": { + "color": { + "mode": "thresholds" + }, + "mappings": [], + "thresholds": { + "mode": "absolute", + "steps": [ + { + "color": "green", + "value": null + } + ] + } + }, + "overrides": [] + }, + "gridPos": { + "h": 8, + "w": 12, + "x": 12, + "y": 10 + }, + "id": 15, + "options": { + "colorMode": "value", + "graphMode": "area", + "justifyMode": "auto", + "orientation": "auto", + "reduceOptions": { + "calcs": [ + "lastNotNull" + ], + "fields": "", + "values": false + }, + "textMode": "auto" + }, + "pluginVersion": "8.4.0", + "targets": [ + { + "datasource": { + "type": "prometheus", + "uid": "PBFA97CFB590B2093" + }, + "exemplar": true, + "expr": "dkv_latency_count{role=\"master\", Ops=\"iter\"}", + "interval": "", + "legendFormat": "", + "refId": "A" + } + ], + "title": "ITER Count", + "type": "stat" + }, + { + "fieldConfig": { + "defaults": { + "color": { + "mode": "palette-classic" + }, + "custom": { + "axisLabel": "", + "axisPlacement": "auto", + "barAlignment": 0, + "drawStyle": "line", + "fillOpacity": 0, + "gradientMode": "none", + "hideFrom": { + "legend": false, + "tooltip": false, + "viz": false + }, + "lineInterpolation": "linear", + "lineWidth": 1, + "pointSize": 5, + "scaleDistribution": { + "type": "linear" + }, + "showPoints": "auto", + "spanNulls": false, + "stacking": { + "group": "A", + "mode": "none" + }, + "thresholdsStyle": { + "mode": "off" + } + }, + "mappings": [], + "thresholds": { + "mode": "absolute", + "steps": [ + { + "color": "green", + "value": null + } + ] + } + }, + "overrides": [] + }, + "gridPos": { + "h": 8, + "w": 12, + "x": 0, + "y": 18 + }, + "id": 14, + "options": { + "legend": { + "calcs": [], + "displayMode": "list", + "placement": "bottom" + }, + "tooltip": { + "mode": "single", + "sort": "none" + } + }, + "pluginVersion": "8.4.0", + "targets": [ + { + "datasource": { + "type": "prometheus", + "uid": "PBFA97CFB590B2093" + }, + "exemplar": true, + "expr": "rate(dkv_latency_count{role=\"master\", Ops=\"put\"}[1m])", + "interval": "", + "legendFormat": "", + "refId": "A" + } + ], + "title": "PUT One Minute Rate", + "type": "timeseries" + }, + { + "fieldConfig": { + "defaults": { + "color": { + "mode": "palette-classic" + }, + "custom": { + "axisLabel": "", + "axisPlacement": "auto", + "barAlignment": 0, + "drawStyle": "line", + "fillOpacity": 0, + "gradientMode": "none", + "hideFrom": { + "legend": false, + "tooltip": false, + "viz": false + }, + "lineInterpolation": "linear", + "lineWidth": 1, + "pointSize": 5, + "scaleDistribution": { + "type": "linear" + }, + "showPoints": "auto", + "spanNulls": false, + "stacking": { + "group": "A", + "mode": "none" + }, + "thresholdsStyle": { + "mode": "off" + } + }, + "mappings": [], + "thresholds": { + "mode": "absolute", + "steps": [ + { + "color": "green", + "value": null + } + ] + } + }, + "overrides": [] + }, + "gridPos": { + "h": 8, + "w": 12, + "x": 12, + "y": 18 + }, + "id": 17, + "options": { + "legend": { + "calcs": [], + "displayMode": "list", + "placement": "bottom" + }, + "tooltip": { + "mode": "single", + "sort": "none" + } + }, + "pluginVersion": "8.4.0", + "targets": [ + { + "datasource": { + "type": "prometheus", + "uid": "PBFA97CFB590B2093" + }, + "exemplar": true, + "expr": "rate(dkv_latency_count{role=\"master\", Ops=\"iter\"}[1m])", + "interval": "", + "legendFormat": "", + "refId": "A" + } + ], + "title": "ITER One Minute Rate", + "type": "timeseries" + } + ], + "refresh": "5s", + "schemaVersion": 35, + "style": "dark", + "tags": [], + "templating": { + "list": [] + }, + "time": { + "from": "now-6h", + "to": "now" + }, + "timepicker": { + "refresh_intervals": [ + "1s", + "5s", + "10s", + "30s", + "1m", + "5m", + "15m", + "30m", + "1h", + "2h", + "1d" + ] + }, + "timezone": "", + "title": "DKV Master Dashboard", + "uid": "k1FFJEBnk", + "version": 1, + "weekStart": "" +} \ No newline at end of file diff --git a/extras/prometheus-dashboard/grafana/provisioning/dashboards/dkv-slave.json b/extras/prometheus-dashboard/grafana/provisioning/dashboards/dkv-slave.json new file mode 100644 index 00000000..c45fd346 --- /dev/null +++ b/extras/prometheus-dashboard/grafana/provisioning/dashboards/dkv-slave.json @@ -0,0 +1,772 @@ +{ + "annotations": { + "list": [ + { + "builtIn": 1, + "datasource": "-- Grafana --", + "enable": true, + "hide": true, + "iconColor": "rgba(0, 211, 255, 1)", + "name": "Annotations & Alerts", + "target": { + "limit": 100, + "matchAny": false, + "tags": [], + "type": "dashboard" + }, + "type": "dashboard" + } + ] + }, + "editable": true, + "fiscalYearStartMonth": 0, + "graphTooltip": 0, + "links": [], + "liveNow": false, + "panels": [ + { + "collapsed": false, + "gridPos": { + "h": 1, + "w": 24, + "x": 0, + "y": 0 + }, + "id": 24, + "panels": [], + "title": "Replication Details", + "type": "row" + }, + { + "aliasColors": {}, + "bars": false, + "dashLength": 10, + "dashes": false, + "description": "Replication Status of the Slave", + "fill": 1, + "fillGradient": 0, + "gridPos": { + "h": 8, + "w": 12, + "x": 0, + "y": 1 + }, + "hiddenSeries": false, + "id": 22, + "legend": { + "avg": false, + "current": false, + "hideEmpty": false, + "hideZero": false, + "max": false, + "min": false, + "show": true, + "total": false, + "values": false + }, + "lines": true, + "linewidth": 1, + "nullPointMode": "null", + "options": { + "alertThreshold": true + }, + "percentage": false, + "pluginVersion": "8.4.3", + "pointradius": 2, + "points": false, + "renderer": "flot", + "seriesOverrides": [], + "spaceLength": 10, + "stack": false, + "steppedLine": false, + "targets": [ + { + "datasource": { + "type": "prometheus", + "uid": "PBFA97CFB590B2093" + }, + "exemplar": false, + "expr": "rate(slave_replication_status_count{role=\"slave\",masterAddr!=\"no-master\"}[1m])", + "hide": false, + "instant": false, + "interval": "", + "intervalFactor": 10, + "legendFormat": "Master: {{masterAddr}}", + "refId": "A" + } + ], + "thresholds": [], + "timeRegions": [], + "title": "Replication Status", + "tooltip": { + "shared": true, + "sort": 0, + "value_type": "individual" + }, + "type": "graph", + "xaxis": { + "mode": "time", + "show": true, + "values": [] + }, + "yaxes": [ + { + "$$hashKey": "object:47", + "format": "short", + "logBase": 1, + "show": true + }, + { + "$$hashKey": "object:48", + "format": "short", + "logBase": 1, + "show": true + } + ], + "yaxis": { + "align": false + } + }, + { + "aliasColors": {}, + "bars": false, + "dashLength": 10, + "dashes": false, + "fieldConfig": { + "defaults": { + "unit": "s" + }, + "overrides": [] + }, + "fill": 1, + "fillGradient": 0, + "gridPos": { + "h": 8, + "w": 12, + "x": 12, + "y": 1 + }, + "hiddenSeries": false, + "id": 2, + "legend": { + "avg": false, + "current": false, + "max": false, + "min": false, + "show": true, + "total": false, + "values": false + }, + "lines": true, + "linewidth": 1, + "nullPointMode": "null", + "options": { + "alertThreshold": true + }, + "percentage": false, + "pluginVersion": "8.4.3", + "pointradius": 2, + "points": false, + "renderer": "flot", + "seriesOverrides": [], + "spaceLength": 10, + "stack": false, + "steppedLine": false, + "targets": [ + { + "datasource": { + "type": "prometheus", + "uid": "PBFA97CFB590B2093" + }, + "exemplar": true, + "expr": "slave_replication_delay", + "interval": "", + "legendFormat": "ip: {{instance}}", + "refId": "A" + } + ], + "thresholds": [], + "timeRegions": [], + "title": "Slave Replication Lag", + "tooltip": { + "shared": true, + "sort": 0, + "value_type": "individual" + }, + "type": "graph", + "xaxis": { + "mode": "time", + "show": true, + "values": [] + }, + "yaxes": [ + { + "$$hashKey": "object:96", + "format": "s", + "logBase": 1, + "show": true + }, + { + "$$hashKey": "object:97", + "format": "short", + "logBase": 1, + "show": true + } + ], + "yaxis": { + "align": false + } + }, + { + "collapsed": false, + "gridPos": { + "h": 1, + "w": 24, + "x": 0, + "y": 9 + }, + "id": 14, + "panels": [], + "title": "Count Metrics", + "type": "row" + }, + { + "description": "", + "fieldConfig": { + "defaults": { + "color": { + "mode": "thresholds" + }, + "mappings": [], + "thresholds": { + "mode": "absolute", + "steps": [ + { + "color": "green", + "value": null + } + ] + } + }, + "overrides": [] + }, + "gridPos": { + "h": 8, + "w": 9, + "x": 0, + "y": 10 + }, + "id": 20, + "options": { + "colorMode": "value", + "graphMode": "area", + "justifyMode": "auto", + "orientation": "auto", + "reduceOptions": { + "calcs": [ + "lastNotNull" + ], + "fields": "", + "values": false + }, + "textMode": "auto" + }, + "pluginVersion": "8.4.3", + "targets": [ + { + "datasource": { + "type": "prometheus", + "uid": "PBFA97CFB590B2093" + }, + "exemplar": true, + "expr": "storage_latency_count{role=\"slave\", ops=\"saveChange\"}", + "interval": "", + "legendFormat": "", + "refId": "A" + } + ], + "title": "Save Changes Count", + "type": "stat" + }, + { + "fieldConfig": { + "defaults": { + "color": { + "mode": "thresholds" + }, + "mappings": [], + "thresholds": { + "mode": "absolute", + "steps": [ + { + "color": "green", + "value": null + }, + { + "color": "red", + "value": 80 + } + ] + } + }, + "overrides": [] + }, + "gridPos": { + "h": 8, + "w": 7, + "x": 9, + "y": 10 + }, + "id": 12, + "options": { + "colorMode": "value", + "graphMode": "area", + "justifyMode": "auto", + "orientation": "auto", + "reduceOptions": { + "calcs": [ + "lastNotNull" + ], + "fields": "", + "values": false + }, + "textMode": "auto" + }, + "pluginVersion": "8.4.3", + "targets": [ + { + "datasource": { + "type": "prometheus", + "uid": "PBFA97CFB590B2093" + }, + "exemplar": true, + "expr": "storage_latency_count{role=\"slave\", ops=\"get\"}", + "interval": "", + "legendFormat": "", + "refId": "A" + } + ], + "title": "GET Count", + "type": "stat" + }, + { + "aliasColors": {}, + "bars": false, + "dashLength": 10, + "dashes": false, + "fieldConfig": { + "defaults": { + "unit": "short" + }, + "overrides": [] + }, + "fill": 1, + "fillGradient": 0, + "gridPos": { + "h": 8, + "w": 8, + "x": 16, + "y": 10 + }, + "hiddenSeries": false, + "id": 8, + "legend": { + "avg": false, + "current": false, + "max": false, + "min": false, + "show": true, + "total": false, + "values": false + }, + "lines": true, + "linewidth": 1, + "nullPointMode": "null", + "options": { + "alertThreshold": true + }, + "percentage": false, + "pluginVersion": "8.4.3", + "pointradius": 2, + "points": false, + "renderer": "flot", + "seriesOverrides": [], + "spaceLength": 10, + "stack": false, + "steppedLine": false, + "targets": [ + { + "datasource": { + "type": "prometheus", + "uid": "PBFA97CFB590B2093" + }, + "exemplar": true, + "expr": "rate(storage_latency_count{role=\"slave\", ops=\"get\"}[1m])", + "interval": "", + "legendFormat": "GET", + "refId": "A" + }, + { + "datasource": { + "type": "prometheus", + "uid": "PBFA97CFB590B2093" + }, + "exemplar": true, + "expr": "rate(storage_latency_count{role=\"slave\", ops=\"saveChange\"}[1m])", + "hide": false, + "interval": "", + "legendFormat": "Save Change", + "refId": "B" + } + ], + "thresholds": [], + "timeRegions": [], + "title": "Operations One Minute Rate", + "tooltip": { + "shared": true, + "sort": 0, + "value_type": "individual" + }, + "type": "graph", + "xaxis": { + "mode": "time", + "show": true, + "values": [] + }, + "yaxes": [ + { + "$$hashKey": "object:37", + "format": "short", + "logBase": 1, + "show": true + }, + { + "$$hashKey": "object:38", + "format": "short", + "logBase": 1, + "show": true + } + ], + "yaxis": { + "align": false + } + }, + { + "collapsed": false, + "gridPos": { + "h": 1, + "w": 24, + "x": 0, + "y": 18 + }, + "id": 18, + "panels": [], + "title": "Error Metrics", + "type": "row" + }, + { + "fieldConfig": { + "defaults": { + "color": { + "mode": "palette-classic" + }, + "custom": { + "axisLabel": "", + "axisPlacement": "auto", + "barAlignment": 0, + "drawStyle": "line", + "fillOpacity": 0, + "gradientMode": "none", + "hideFrom": { + "legend": false, + "tooltip": false, + "viz": false + }, + "lineInterpolation": "linear", + "lineWidth": 1, + "pointSize": 5, + "scaleDistribution": { + "type": "linear" + }, + "showPoints": "auto", + "spanNulls": false, + "stacking": { + "group": "A", + "mode": "none" + }, + "thresholdsStyle": { + "mode": "off" + } + }, + "mappings": [], + "thresholds": { + "mode": "absolute", + "steps": [ + { + "color": "green", + "value": null + }, + { + "color": "red", + "value": 80 + } + ] + } + }, + "overrides": [] + }, + "gridPos": { + "h": 9, + "w": 12, + "x": 0, + "y": 19 + }, + "id": 10, + "options": { + "legend": { + "calcs": [], + "displayMode": "list", + "placement": "bottom" + }, + "tooltip": { + "mode": "single", + "sort": "none" + } + }, + "targets": [ + { + "datasource": { + "type": "prometheus", + "uid": "PBFA97CFB590B2093" + }, + "exemplar": true, + "expr": "storage_error", + "interval": "", + "legendFormat": "", + "refId": "A" + } + ], + "title": "Storage Error", + "type": "timeseries" + }, + { + "collapsed": false, + "gridPos": { + "h": 1, + "w": 24, + "x": 0, + "y": 28 + }, + "id": 16, + "panels": [], + "title": "Latency Metrics", + "type": "row" + }, + { + "fieldConfig": { + "defaults": { + "color": { + "mode": "palette-classic" + }, + "custom": { + "axisLabel": "", + "axisPlacement": "auto", + "barAlignment": 0, + "drawStyle": "line", + "fillOpacity": 10, + "gradientMode": "none", + "hideFrom": { + "legend": false, + "tooltip": false, + "viz": false + }, + "lineInterpolation": "linear", + "lineWidth": 1, + "pointSize": 5, + "scaleDistribution": { + "type": "linear" + }, + "showPoints": "never", + "spanNulls": true, + "stacking": { + "group": "A", + "mode": "none" + }, + "thresholdsStyle": { + "mode": "off" + } + }, + "mappings": [], + "thresholds": { + "mode": "absolute", + "steps": [ + { + "color": "green", + "value": null + }, + { + "color": "red", + "value": 80 + } + ] + }, + "unit": "s" + }, + "overrides": [] + }, + "gridPos": { + "h": 8, + "w": 12, + "x": 0, + "y": 29 + }, + "id": 4, + "options": { + "legend": { + "calcs": [], + "displayMode": "list", + "placement": "bottom" + }, + "tooltip": { + "mode": "multi", + "sort": "none" + } + }, + "pluginVersion": "8.4.0", + "targets": [ + { + "datasource": { + "type": "prometheus", + "uid": "PBFA97CFB590B2093" + }, + "exemplar": true, + "expr": "storage_latency{ops=\"get\",role=\"slave\"}", + "instant": false, + "interval": "", + "legendFormat": " {{quantile}}p", + "refId": "A" + } + ], + "title": "Store Latency - GET", + "type": "timeseries" + }, + { + "fieldConfig": { + "defaults": { + "color": { + "mode": "palette-classic" + }, + "custom": { + "axisLabel": "", + "axisPlacement": "auto", + "barAlignment": 0, + "drawStyle": "line", + "fillOpacity": 10, + "gradientMode": "none", + "hideFrom": { + "legend": false, + "tooltip": false, + "viz": false + }, + "lineInterpolation": "linear", + "lineWidth": 1, + "pointSize": 5, + "scaleDistribution": { + "type": "linear" + }, + "showPoints": "never", + "spanNulls": true, + "stacking": { + "group": "A", + "mode": "none" + }, + "thresholdsStyle": { + "mode": "off" + } + }, + "mappings": [], + "thresholds": { + "mode": "absolute", + "steps": [ + { + "color": "green", + "value": null + }, + { + "color": "red", + "value": 80 + } + ] + }, + "unit": "s" + }, + "overrides": [] + }, + "gridPos": { + "h": 8, + "w": 12, + "x": 12, + "y": 29 + }, + "id": 6, + "options": { + "legend": { + "calcs": [], + "displayMode": "list", + "placement": "bottom" + }, + "tooltip": { + "mode": "multi", + "sort": "none" + } + }, + "pluginVersion": "8.4.0", + "targets": [ + { + "datasource": { + "type": "prometheus", + "uid": "PBFA97CFB590B2093" + }, + "exemplar": true, + "expr": "storage_latency{ops=\"saveChange\", role=\"slave\"}", + "interval": "", + "legendFormat": "{{quantile}}", + "refId": "A" + } + ], + "title": "Store Latency - Save", + "type": "timeseries" + } + ], + "refresh": "5s", + "schemaVersion": 35, + "style": "dark", + "tags": [], + "templating": { + "list": [] + }, + "time": { + "from": "now-1h", + "to": "now" + }, + "timepicker": { + "refresh_intervals": [ + "1s", + "5s", + "10s", + "30s", + "1m", + "5m", + "15m", + "30m", + "1h", + "2h", + "1d" + ] + }, + "timezone": "", + "title": "Dkv Slave Dashboard", + "uid": "lFpIV2fnk", + "version": 2, + "weekStart": "" +} \ No newline at end of file diff --git a/extras/prometheus-dashboard/grafana/provisioning/datasources/datasource.yml b/extras/prometheus-dashboard/grafana/provisioning/datasources/datasource.yml new file mode 100644 index 00000000..bb37f13d --- /dev/null +++ b/extras/prometheus-dashboard/grafana/provisioning/datasources/datasource.yml @@ -0,0 +1,11 @@ +apiVersion: 1 + +datasources: + - name: Prometheus + type: prometheus + access: proxy + orgId: 1 + url: http://prometheus:9090 + basicAuth: false + isDefault: true + editable: true \ No newline at end of file diff --git a/extras/prometheus-dashboard/prometheus/prometheus.yml b/extras/prometheus-dashboard/prometheus/prometheus.yml new file mode 100644 index 00000000..fd28feef --- /dev/null +++ b/extras/prometheus-dashboard/prometheus/prometheus.yml @@ -0,0 +1,11 @@ +global: + scrape_interval: 100ms +scrape_configs: + - job_name: "dkv" + static_configs: + - targets: [ "host.docker.internal:8181" ] + labels: + role: master + - targets: [ "host.docker.internal:7171" ] + labels: + role: slave \ No newline at end of file diff --git a/go.mod b/go.mod index 1e60c1aa..f835daa3 100644 --- a/go.mod +++ b/go.mod @@ -13,10 +13,12 @@ require ( github.com/flipkart-incubator/nexus v0.0.0-20220316072727-c44c4b25144a github.com/gogo/protobuf v1.3.2 github.com/golang/protobuf v1.5.2 + github.com/gorilla/mux v1.8.0 github.com/grpc-ecosystem/go-grpc-middleware v1.2.0 github.com/kpango/fastime v1.0.16 github.com/matttproud/golang_protobuf_extensions v1.0.1 - github.com/prometheus/client_golang v1.5.1 // indirect + github.com/prometheus/client_golang v1.5.1 + github.com/prometheus/client_model v0.2.0 github.com/prometheus/procfs v0.0.10 // indirect github.com/smira/go-statsd v1.3.1 github.com/spf13/pflag v1.0.5 diff --git a/go.sum b/go.sum index 86e525e4..fc7d73cb 100644 --- a/go.sum +++ b/go.sum @@ -176,6 +176,8 @@ github.com/googleapis/gax-go/v2 v2.0.5/go.mod h1:DWXyrwAJ9X0FpwwEdw+IPEYBICEFu5m github.com/googleapis/gax-go/v2 v2.1.0/go.mod h1:Q3nei7sK6ybPYH7twZdmQpAd1MKb7pfu6SK+H1/DsU0= github.com/googleapis/gax-go/v2 v2.1.1/go.mod h1:hddJymUZASv3XPyGkUpKj8pPO47Rmb0eJc8R6ouapiM= github.com/googleapis/google-cloud-go v0.26.0/go.mod h1:yJoOdPPE9UpqbamBhJvp7Ur6OUPPV4rUY3RnssPGNBA= +github.com/gorilla/mux v1.8.0 h1:i40aqfkR1h2SlN9hojwV5ZA91wcXFOvkdNIeFDP5koI= +github.com/gorilla/mux v1.8.0/go.mod h1:DVbg23sWSpFRCP0SfiEN6jmj59UnW/n46BH5rLB71So= github.com/grpc-ecosystem/go-grpc-middleware v1.2.0 h1:0IKlLyQ3Hs9nDaiK5cSHAGmcQEIC8l2Ts1u6x5Dfrqg= github.com/grpc-ecosystem/go-grpc-middleware v1.2.0/go.mod h1:mJzapYve32yjrKlk9GbyCZHuPgZsrbyIbyKhSzOpg6s= github.com/grpc-ecosystem/grpc-gateway v1.16.0/go.mod h1:BDjrQk3hbvj6Nolgz8mAMFbcEtjT1g+wF4CSlocrBnw= diff --git a/internal/discovery/client.go b/internal/discovery/client.go index 77e0d6f4..74221c15 100644 --- a/internal/discovery/client.go +++ b/internal/discovery/client.go @@ -7,14 +7,15 @@ This class contains the behaviour of propagating a nodes status updates to disco import ( "context" "fmt" + "strconv" + "time" + _ "github.com/Jille/grpc-multi-resolver" "github.com/flipkart-incubator/dkv/internal/hlc" "github.com/flipkart-incubator/dkv/pkg/serverpb" "go.uber.org/zap" "google.golang.org/grpc" "gopkg.in/ini.v1" - "strconv" - "time" ) type DiscoveryClientConfig struct { @@ -155,6 +156,8 @@ func (m *discoveryClient) pollClusterInfo() error { } } +// gets cluster info for the provided database and vBucket +// database and vBucket can be empty strings, in which case, the entire cluster set is returned func (m *discoveryClient) GetClusterStatus(database string, vBucket string) ([]*serverpb.RegionInfo, error) { if m.clusterInfo == nil { // When called before cluster info is initialised @@ -165,8 +168,10 @@ func (m *discoveryClient) GetClusterStatus(database string, vBucket string) ([]* } var regions []*serverpb.RegionInfo for _, region := range m.clusterInfo { - if region.Database == database && region.VBucket == vBucket { - regions = append(regions, region) + if region.Database == database || database == "" { + if region.VBucket == vBucket || vBucket == "" { + regions = append(regions, region) + } } } return regions, nil diff --git a/internal/discovery/client_test.go b/internal/discovery/client_test.go index 92fc75a5..5e2c28d4 100644 --- a/internal/discovery/client_test.go +++ b/internal/discovery/client_test.go @@ -22,6 +22,7 @@ var ( Logger: lgr, HealthCheckTickerInterval: opts.DefaultHealthCheckTickterInterval, StatsCli: stats.NewNoOpClient(), + PrometheusRegistry: stats.NewPromethousNoopRegistry(), } ) @@ -29,7 +30,7 @@ func TestDiscoveryClient(t *testing.T) { dkvSvc, grpcSrvr := serveStandaloneDKVWithDiscovery(discoverySvcPort, &serverpb.RegionInfo{}, dbFolder+"_DC") defer dkvSvc.Close() defer grpcSrvr.GracefulStop() - + <-time.After(time.Duration(10) * time.Second) clientConfig := &DiscoveryClientConfig{DiscoveryServiceAddr: fmt.Sprintf("%s:%d", dkvSvcHost, discoverySvcPort), PushStatusInterval: time.Duration(5), PollClusterInfoInterval: time.Duration(5)} @@ -99,13 +100,13 @@ func TestDiscoveryClient(t *testing.T) { } regionInfos, _ = dClient.GetClusterStatus("", "vbucket2") - if len(regionInfos) != 0 { - t.Errorf("GET Cluster Status Mismatch. Criteria: %s, Expected Value: %d, Actual Value: %d", "No database", 0, len(regionInfos)) + if len(regionInfos) != 1 { + t.Errorf("GET Cluster Status Mismatch. Criteria: %s, Expected Value: %d, Actual Value: %d", "No database", 1, len(regionInfos)) } regionInfos, _ = dClient.GetClusterStatus("db1", "") - if len(regionInfos) != 0 { - t.Errorf("GET Cluster Status Mismatch. Criteria: %s, Expected Value: %d, Actual Value: %d", "No vBucket", 0, len(regionInfos)) + if len(regionInfos) != 3 { + t.Errorf("GET Cluster Status Mismatch. Criteria: %s, Expected Value: %d, Actual Value: %d", "No vBucket", 3, len(regionInfos)) } regionInfos, _ = dClient.GetClusterStatus("db1", "vbucket3") diff --git a/internal/discovery/service.go b/internal/discovery/service.go index 9ccfb986..4e422f38 100644 --- a/internal/discovery/service.go +++ b/internal/discovery/service.go @@ -120,6 +120,7 @@ func (d *discoverService) GetClusterInfo(ctx context.Context, request *serverpb. continue } // Filter inactive regions and regions whose status was updated long time back and hence considered inactive + // This simplifies logic on consumers of this API (envoy, slaves) which don't need to filter by status if hlc.GetTimeAgo(statusUpdate.GetTimestamp()) < d.config.HeartbeatTimeout && statusUpdate.GetRegionInfo().GetStatus() != serverpb.RegionStatus_INACTIVE { regionsInfo = append(regionsInfo, statusUpdate.GetRegionInfo()) } diff --git a/internal/discovery/service_test.go b/internal/discovery/service_test.go index 85480fd2..ec0dcf82 100644 --- a/internal/discovery/service_test.go +++ b/internal/discovery/service_test.go @@ -2,6 +2,11 @@ package discovery import ( "fmt" + "net" + "os/exec" + "testing" + "time" + "github.com/flipkart-incubator/dkv/internal/master" "github.com/flipkart-incubator/dkv/internal/storage" "github.com/flipkart-incubator/dkv/internal/storage/badger" @@ -10,14 +15,10 @@ import ( "github.com/flipkart-incubator/dkv/pkg/serverpb" "go.uber.org/zap" "google.golang.org/grpc" - "net" - "os/exec" - "testing" - "time" ) const ( - dkvSvcPort = 8080 + dkvSvcPort = 8082 dkvSvcHost = "localhost" dbFolder = "/tmp/dkv_discovery_test_db" cacheSize = 3 << 30 diff --git a/internal/master/service.go b/internal/master/service.go index 03342cb7..6da8fc6e 100644 --- a/internal/master/service.go +++ b/internal/master/service.go @@ -12,6 +12,9 @@ import ( "sync" "time" + "github.com/flipkart-incubator/dkv/internal/stats" + "github.com/prometheus/client_golang/prometheus" + "github.com/flipkart-incubator/dkv/pkg/health" "github.com/flipkart-incubator/nexus/models" @@ -39,6 +42,28 @@ type DKVService interface { health.HealthServer } +type dkvServiceStat struct { + Latency *prometheus.SummaryVec + ResponseError *prometheus.CounterVec +} + +func newDKVServiceStat(registry prometheus.Registerer) *dkvServiceStat { + RequestLatency := prometheus.NewSummaryVec(prometheus.SummaryOpts{ + Namespace: "dkv", + Name: "latency", + Help: "Latency statistics for dkv service", + Objectives: map[float64]float64{0.5: 0.05, 0.9: 0.01, 0.99: 0.001}, + MaxAge: 10 * time.Second, + }, []string{"Ops"}) + ResponseError := prometheus.NewCounterVec(prometheus.CounterOpts{ + Namespace: "dkv", + Name: "error", + Help: "Error count for storage operations", + }, []string{"Ops"}) + registry.MustRegister(RequestLatency, ResponseError) + return &dkvServiceStat{RequestLatency, ResponseError} +} + type standaloneService struct { store storage.KVStore cp storage.ChangePropagator @@ -48,6 +73,7 @@ type standaloneService struct { isClosed bool shutdown chan struct{} opts *opts.ServerOpts + stat *dkvServiceStat } func (ss *standaloneService) GetStatus(ctx context.Context, request *emptypb.Empty) (*serverpb.RegionInfo, error) { @@ -87,10 +113,11 @@ func (ss *standaloneService) Watch(req *health.HealthCheckRequest, watcher healt func NewStandaloneService(store storage.KVStore, cp storage.ChangePropagator, br storage.Backupable, regionInfo *serverpb.RegionInfo, opts *opts.ServerOpts) DKVService { rwl := &sync.RWMutex{} regionInfo.Status = serverpb.RegionStatus_LEADER - return &standaloneService{store, cp, br, rwl, regionInfo, false, make(chan struct{}, 1), opts} + return &standaloneService{store, cp, br, rwl, regionInfo, false, make(chan struct{}, 1), opts, newDKVServiceStat(opts.PrometheusRegistry)} } func (ss *standaloneService) Put(ctx context.Context, putReq *serverpb.PutRequest) (*serverpb.PutResponse, error) { + defer stats.MeasureLatency(ss.stat.Latency.WithLabelValues(stats.Put), time.Now()) ss.rwl.RLock() defer ss.rwl.RUnlock() if err := ss.store.Put(&serverpb.KVPair{Key: putReq.Key, Value: putReq.Value, ExpireTS: putReq.ExpireTS}); err != nil { @@ -116,6 +143,7 @@ func (ss *standaloneService) MultiPut(ctx context.Context, putReq *serverpb.Mult } func (ss *standaloneService) Delete(ctx context.Context, delReq *serverpb.DeleteRequest) (*serverpb.DeleteResponse, error) { + defer stats.MeasureLatency(ss.stat.Latency.WithLabelValues(stats.Delete), time.Now()) ss.rwl.RLock() defer ss.rwl.RUnlock() @@ -127,9 +155,9 @@ func (ss *standaloneService) Delete(ctx context.Context, delReq *serverpb.Delete } func (ss *standaloneService) Get(ctx context.Context, getReq *serverpb.GetRequest) (*serverpb.GetResponse, error) { + defer stats.MeasureLatency(ss.stat.Latency.WithLabelValues(stats.Get+getReadConsistencySuffix(getReq.ReadConsistency)), time.Now()) ss.rwl.RLock() defer ss.rwl.RUnlock() - readResults, err := ss.store.Get(getReq.Key) res := &serverpb.GetResponse{Status: newEmptyStatus()} if err != nil { @@ -146,6 +174,7 @@ func (ss *standaloneService) Get(ctx context.Context, getReq *serverpb.GetReques } func (ss *standaloneService) MultiGet(ctx context.Context, multiGetReq *serverpb.MultiGetRequest) (*serverpb.MultiGetResponse, error) { + defer stats.MeasureLatency(ss.stat.Latency.WithLabelValues(stats.MultiGet+getReadConsistencySuffix(multiGetReq.ReadConsistency)), time.Now()) ss.rwl.RLock() defer ss.rwl.RUnlock() @@ -161,6 +190,7 @@ func (ss *standaloneService) MultiGet(ctx context.Context, multiGetReq *serverpb } func (ss *standaloneService) CompareAndSet(ctx context.Context, casReq *serverpb.CompareAndSetRequest) (*serverpb.CompareAndSetResponse, error) { + defer stats.MeasureLatency(ss.stat.Latency.WithLabelValues(stats.CompareAndSet), time.Now()) ss.rwl.RLock() defer ss.rwl.RUnlock() @@ -317,6 +347,7 @@ func (ss *standaloneService) Restore(ctx context.Context, restoreReq *serverpb.R } func (ss *standaloneService) Iterate(iterReq *serverpb.IterateRequest, dkvIterSrvr serverpb.DKV_IterateServer) error { + defer stats.MeasureLatency(ss.stat.Latency.WithLabelValues(stats.Iterate), time.Now()) ss.rwl.RLock() defer ss.rwl.RUnlock() @@ -359,6 +390,7 @@ type distributedService struct { // is not running shutdown chan struct{} opts *opts.ServerOpts + stat *dkvServiceStat } // NewDistributedService creates a distributed variant of the DKV service @@ -370,10 +402,12 @@ func NewDistributedService(kvs storage.KVStore, cp storage.ChangePropagator, br raftRepl: raftRepl, shutdown: make(chan struct{}, 1), opts: opts, + stat: newDKVServiceStat(stats.NewPromethousNoopRegistry()), } } func (ds *distributedService) Put(ctx context.Context, putReq *serverpb.PutRequest) (*serverpb.PutResponse, error) { + defer stats.MeasureLatency(ds.stat.Latency.WithLabelValues(stats.Put), time.Now()) reqBts, err := proto.Marshal(&raftpb.InternalRaftRequest{Put: putReq}) res := &serverpb.PutResponse{Status: newEmptyStatus()} if err != nil { @@ -404,6 +438,7 @@ func (ds *distributedService) MultiPut(ctx context.Context, multiPutReq *serverp } func (ds *distributedService) CompareAndSet(ctx context.Context, casReq *serverpb.CompareAndSetRequest) (*serverpb.CompareAndSetResponse, error) { + defer stats.MeasureLatency(ds.stat.Latency.WithLabelValues(stats.CompareAndSet), time.Now()) reqBts, _ := proto.Marshal(&raftpb.InternalRaftRequest{Cas: casReq}) res := &serverpb.CompareAndSetResponse{Status: newEmptyStatus()} casRes, err := ds.raftRepl.Save(ctx, reqBts) @@ -418,6 +453,7 @@ func (ds *distributedService) CompareAndSet(ctx context.Context, casReq *serverp } func (ds *distributedService) Delete(ctx context.Context, delReq *serverpb.DeleteRequest) (*serverpb.DeleteResponse, error) { + defer stats.MeasureLatency(ds.stat.Latency.WithLabelValues(stats.Delete), time.Now()) reqBts, err := proto.Marshal(&raftpb.InternalRaftRequest{Delete: delReq}) res := &serverpb.DeleteResponse{Status: newEmptyStatus()} if err != nil { @@ -433,6 +469,7 @@ func (ds *distributedService) Delete(ctx context.Context, delReq *serverpb.Delet } func (ds *distributedService) Get(ctx context.Context, getReq *serverpb.GetRequest) (*serverpb.GetResponse, error) { + defer stats.MeasureLatency(ds.stat.Latency.WithLabelValues(stats.Get+getReadConsistencySuffix(getReq.GetReadConsistency())), time.Now()) switch getReq.ReadConsistency { case serverpb.ReadConsistency_SEQUENTIAL: return ds.DKVService.Get(ctx, getReq) @@ -460,6 +497,7 @@ func (ds *distributedService) Get(ctx context.Context, getReq *serverpb.GetReque } func (ds *distributedService) MultiGet(ctx context.Context, multiGetReq *serverpb.MultiGetRequest) (*serverpb.MultiGetResponse, error) { + defer stats.MeasureLatency(ds.stat.Latency.WithLabelValues(stats.MultiGet+getReadConsistencySuffix(multiGetReq.GetReadConsistency())), time.Now()) switch multiGetReq.ReadConsistency { case serverpb.ReadConsistency_SEQUENTIAL: return ds.DKVService.MultiGet(ctx, multiGetReq) @@ -490,6 +528,7 @@ func gobDecodeAsKVPairs(val []byte) ([]*serverpb.KVPair, error) { } func (ds *distributedService) Iterate(iterReq *serverpb.IterateRequest, dkvIterSrvr serverpb.DKV_IterateServer) error { + defer stats.MeasureLatency(ds.stat.Latency.WithLabelValues(stats.Iterate), time.Now()) return ds.DKVService.Iterate(iterReq, dkvIterSrvr) } @@ -602,3 +641,13 @@ func newErrorStatus(err error) *serverpb.Status { func newEmptyStatus() *serverpb.Status { return &serverpb.Status{Code: 0, Message: ""} } + +func getReadConsistencySuffix(rc serverpb.ReadConsistency) string { + switch rc { + case serverpb.ReadConsistency_SEQUENTIAL: + return "Seq" + case serverpb.ReadConsistency_LINEARIZABLE: + return "Lin" + } + return "" +} diff --git a/internal/master/ss_service_test.go b/internal/master/ss_service_test.go index af0e9d60..e27a6560 100644 --- a/internal/master/ss_service_test.go +++ b/internal/master/ss_service_test.go @@ -32,6 +32,7 @@ var ( serverOpts = &opts.ServerOpts{ HealthCheckTickerInterval: opts.DefaultHealthCheckTickterInterval, StatsCli: stats.NewNoOpClient(), + PrometheusRegistry: stats.NewPromethousNoopRegistry(), Logger: lgr, } ) diff --git a/internal/opts/config.go b/internal/opts/config.go index a54ddde4..1e3dec3c 100644 --- a/internal/opts/config.go +++ b/internal/opts/config.go @@ -22,7 +22,7 @@ type Config struct { NodeName string `mapstructure:"node-name" desc:"Node Name"` DbEngine string `mapstructure:"db-engine" desc:"Underlying DB engine for storing data - badger|rocksdb"` DbEngineIni string `mapstructure:"db-engine-ini" desc:"An .ini file for configuring the underlying storage engine. Refer badger.ini or rocks.ini for more details."` - DbRole string `mapstructure:"role" desc:"Role of the node - master|slave|standalone"` + DbRole string `mapstructure:"role" desc:"Role of the node - master|slave|standalone|discovery"` ReplPollIntervalString string `mapstructure:"repl-poll-interval" desc:"Interval used for polling changes from master. Eg., 10s, 5ms, 2h, etc."` BlockCacheSize uint64 `mapstructure:"block-cache-size" desc:"Amount of cache (in bytes) to set aside for data blocks. A value of 0 disables block caching altogether."` DcID string `mapstructure:"dc-id" desc:"DC / Availability zone identifier"` @@ -34,8 +34,9 @@ type Config struct { DbFolder string `mapstructure:"db-folder" desc:"DB folder path for storing data files"` // Server Configuration - ListenAddr string `mapstructure:"listen-addr" desc:"Address on which the DKV service binds"` - StatsdAddr string `mapstructure:"statsd-addr" desc:"StatsD service address in host:port format"` + ListenAddr string `mapstructure:"listen-addr" desc:"Address on which the DKV service binds"` + HttpListenAddr string `mapstructure:"http-listen-addr" desc:"Address on which the DKV service binds for http"` + StatsdAddr string `mapstructure:"statsd-addr" desc:"StatsD service address in host:port format"` //Service discovery related params DiscoveryServiceConfig string `mapstructure:"discovery-service-config" desc:"A .ini file for configuring discovery service parameters"` diff --git a/internal/opts/serverOpts.go b/internal/opts/serverOpts.go index 7b84863b..c67aa987 100644 --- a/internal/opts/serverOpts.go +++ b/internal/opts/serverOpts.go @@ -2,6 +2,7 @@ package opts import ( "github.com/flipkart-incubator/dkv/internal/stats" + "github.com/prometheus/client_golang/prometheus" "go.uber.org/zap" ) @@ -12,6 +13,7 @@ type ServerOpts struct { HealthCheckTickerInterval uint StatsCli stats.Client Logger *zap.Logger + PrometheusRegistry prometheus.Registerer } const ( diff --git a/internal/slave/service.go b/internal/slave/service.go index 4333fc71..f43f16d3 100644 --- a/internal/slave/service.go +++ b/internal/slave/service.go @@ -4,6 +4,7 @@ import ( "context" "errors" "fmt" + dto "github.com/prometheus/client_model/go" "io" "math/rand" "strings" @@ -17,6 +18,7 @@ import ( "github.com/flipkart-incubator/dkv/internal/storage" "github.com/flipkart-incubator/dkv/pkg/ctl" "github.com/flipkart-incubator/dkv/pkg/serverpb" + "github.com/prometheus/client_golang/prometheus" "go.uber.org/zap" "google.golang.org/protobuf/types/known/emptypb" ) @@ -55,6 +57,9 @@ type replInfo struct { lastReplTime uint64 replConfig *ReplicationConfig fromChngNum uint64 + //replDelay is an approximation of the delay in time units of the changes seen + // in the master and the same changes seen in the slave + replDelay float64 } type slaveService struct { @@ -65,6 +70,44 @@ type slaveService struct { isClosed bool replInfo *replInfo serveropts *opts.ServerOpts + stat *stat +} +type stat struct { + ReplicationLag prometheus.Gauge + ReplicationDelay prometheus.Gauge + ReplicationStatus *prometheus.SummaryVec + ReplicationSpeed prometheus.Histogram +} + +func newStat(registry prometheus.Registerer) *stat { + replicationLag := prometheus.NewGauge(prometheus.GaugeOpts{ + Namespace: "slave", + Name: "replication_lag", + Help: "replication lag of the slave", + }) + replicationDelay := prometheus.NewGauge(prometheus.GaugeOpts{ + Namespace: "slave", + Name: "replication_delay", + Help: "replication delay of the slave", + }) + replicationStatus := prometheus.NewSummaryVec(prometheus.SummaryOpts{ + Namespace: "slave", + Name: "replication_status", + Help: "replication status of the slave", + MaxAge: 5 * time.Second, + }, []string{"masterAddr"}) + replicationSpeed := prometheus.NewHistogram(prometheus.HistogramOpts{ + Namespace: "slave", + Name: "replication_speed", + Help: "replication speed of the slave", + }) + registry.MustRegister(replicationLag, replicationDelay, replicationSpeed, replicationStatus) + return &stat{ + ReplicationLag: replicationLag, + ReplicationDelay: replicationDelay, + ReplicationStatus: replicationStatus, + ReplicationSpeed: replicationSpeed, + } } // NewService creates a slave DKVService that periodically polls @@ -83,7 +126,7 @@ func NewService(store storage.KVStore, ca storage.ChangeApplier, regionInfo *ser func newSlaveService(store storage.KVStore, ca storage.ChangeApplier, info *serverpb.RegionInfo, replConf *ReplicationConfig, clusterInfo discovery.ClusterInfoGetter, serveropts *opts.ServerOpts) *slaveService { ri := &replInfo{replConfig: replConf} - ss := &slaveService{store: store, ca: ca, regionInfo: info, replInfo: ri, clusterInfo: clusterInfo, serveropts: serveropts} + ss := &slaveService{store: store, ca: ca, regionInfo: info, replInfo: ri, clusterInfo: clusterInfo, serveropts: serveropts, stat: newStat(serveropts.PrometheusRegistry)} ss.findAndConnectToMaster() ss.startReplication() return ss @@ -208,13 +251,18 @@ func (ss *slaveService) pollAndApplyChanges() { case <-ss.replInfo.replTckr.C: ss.serveropts.Logger.Info("Current replication lag", zap.Uint64("ReplicationLag", ss.replInfo.replLag)) ss.serveropts.StatsCli.Gauge("replication.lag", int64(ss.replInfo.replLag)) + ss.stat.ReplicationLag.Set(float64(ss.replInfo.replLag)) + ss.stat.ReplicationDelay.Set(ss.replInfo.replDelay) if err := ss.applyChangesFromMaster(ss.replInfo.replConfig.MaxNumChngs); err != nil { + ss.stat.ReplicationStatus.WithLabelValues("no-master").Observe(1) ss.serveropts.Logger.Error("Unable to retrieve changes from master", zap.Error(err)) if err := ss.replaceMasterIfInactive(); err != nil { ss.serveropts.Logger.Error("Unable to replace master", zap.Error(err)) } } + ss.stat.ReplicationStatus.WithLabelValues(ss.replInfo.replConfig.ReplMasterAddr).Observe(1) case <-ss.replInfo.replStop: + ss.stat.ReplicationStatus.WithLabelValues("no-master").Observe(0) ss.serveropts.Logger.Info("Stopping the change poller") break } @@ -271,15 +319,38 @@ func (ss *slaveService) applyChanges(chngsRes *serverpb.GetChangesResponse) erro if err != nil { return err } + if timeBwRepl := (hlc.UnixNow() - ss.replInfo.lastReplTime); ss.replInfo.lastReplTime != 0 && timeBwRepl != 0 && actChngNum > ss.replInfo.fromChngNum { + currentReplSpeed := (float64(actChngNum-ss.replInfo.fromChngNum) / float64(timeBwRepl)) + ss.stat.ReplicationSpeed.Observe(currentReplSpeed) + } ss.replInfo.fromChngNum = actChngNum + 1 ss.serveropts.Logger.Info("Changes applied to local storage", zap.Uint64("FromChangeNumber", ss.replInfo.fromChngNum)) - ss.replInfo.replLag = chngsRes.MasterChangeNumber - actChngNum + if chngsRes.MasterChangeNumber >= actChngNum { + ss.replInfo.replLag = chngsRes.MasterChangeNumber - actChngNum + } else { + ss.replInfo.replLag = 0 //replication lag can be negative when master has returned every change that was available to it + } + replicationSpeedMetric := getReplicationSpeed(ss) + if cnt := replicationSpeedMetric.Histogram.GetSampleCount(); cnt != 0 { + replSpeedAvg := replicationSpeedMetric.Histogram.GetSampleSum() / float64(cnt) + if replSpeedAvg > float64(1e-9) { + ss.replInfo.replDelay = float64(ss.replInfo.replLag) / replSpeedAvg + } + } } else { ss.serveropts.Logger.Info("Not received any changes from master") } return nil } +func getReplicationSpeed(ss *slaveService) *dto.Metric { + metric := &dto.Metric{} + if err := ss.stat.ReplicationSpeed.Write(metric); err != nil { + ss.serveropts.Logger.Warn("Error while writing out %s metric", zap.String("Metric", "ReplicationSpeed")) + } + return metric +} + func newErrorStatus(err error) *serverpb.Status { return &serverpb.Status{Code: -1, Message: err.Error()} } diff --git a/internal/slave/service_test.go b/internal/slave/service_test.go index 5151b4d5..f6157162 100644 --- a/internal/slave/service_test.go +++ b/internal/slave/service_test.go @@ -66,6 +66,7 @@ var ( serverOpts = &opts.ServerOpts{ Logger: lgr, StatsCli: stats.NewNoOpClient(), + PrometheusRegistry: stats.NewPromethousNoopRegistry(), HealthCheckTickerInterval: opts.DefaultHealthCheckTickterInterval, } @@ -898,6 +899,7 @@ func serveStandaloneDKVSlave(wg *sync.WaitGroup, store storage.KVStore, ca stora specialOpts := &opts.ServerOpts{ Logger: lgr, StatsCli: stats.NewNoOpClient(), + PrometheusRegistry: stats.NewPromethousNoopRegistry(), HealthCheckTickerInterval: uint(1), } diff --git a/internal/stats/aggregate/aggregator.go b/internal/stats/aggregate/aggregator.go new file mode 100644 index 00000000..a3b3abf4 --- /dev/null +++ b/internal/stats/aggregate/aggregator.go @@ -0,0 +1,178 @@ +package aggregate + +import ( + "context" + "fmt" + "sync" + "time" + + "github.com/flipkart-incubator/dkv/internal/stats" + "github.com/flipkart-incubator/dkv/pkg/serverpb" +) + +var ( + MAX_STAT_BUFFER = 5 +) + +type StatAggregatorRegistry struct { + statsListener *StatListener + statAggregatorMap map[int64]*StatAggregator + /* Mutex for Safe Access */ + mapMutex sync.Mutex +} + +type MetricTag interface { + GetTag(region *serverpb.RegionInfo) string +} + +func NewStatAggregatorRegistry() *StatAggregatorRegistry { + statsListener := NewStatListener() + return &StatAggregatorRegistry{statsListener: statsListener, statAggregatorMap: make(map[int64]*StatAggregator)} +} + +func (sr *StatAggregatorRegistry) Register(regions []*serverpb.RegionInfo, tagger func(*serverpb.RegionInfo) string, outputChannel chan map[string]*stats.DKVMetrics) int64 { + hostMap := map[string]string{} + for _, region := range regions { + hostMap[region.GetHttpAddress()] = tagger(region) + } + id := time.Now().UnixNano() + statAggregator := NewStatAggregator(outputChannel, hostMap) + go statAggregator.Start(sr.statsListener) + + sr.mapMutex.Lock() + defer sr.mapMutex.Unlock() + + sr.statAggregatorMap[id] = statAggregator + + return id +} + +func (sr *StatAggregatorRegistry) DeRegister(id int64) { + sr.mapMutex.Lock() + defer sr.mapMutex.Unlock() + if statAggregator, exist := sr.statAggregatorMap[id]; exist { + statAggregator.Stop() + delete(sr.statAggregatorMap, id) + } +} + +type StatAggregator struct { + outputChannel chan map[string]*stats.DKVMetrics + aggregatedStatMap map[int64]map[string]*stats.DKVMetrics + hostMap map[string]string + channelIds map[string]int64 + ctx context.Context + cancelFunc context.CancelFunc +} + +func NewStatAggregator(outputChannel chan map[string]*stats.DKVMetrics, hostMap map[string]string) *StatAggregator { + ctx, cancelFunc := context.WithCancel(context.Background()) + return &StatAggregator{outputChannel: outputChannel, hostMap: hostMap, channelIds: map[string]int64{}, ctx: ctx, cancelFunc: cancelFunc} +} + +func (sa *StatAggregator) Start(listener *StatListener) { + sa.aggregatedStatMap = make(map[int64]map[string]*stats.DKVMetrics, MAX_STAT_BUFFER) + + channels := make([]chan MetricEvent, 2) + for host, _ := range sa.hostMap { + channel := make(chan MetricEvent, MAX_STAT_BUFFER) + channelId, _ := listener.Register(host, channel) + sa.channelIds[host] = channelId + channels = append(channels, channel) + } + aggregatedEventChannel := sa.getMultiplexedChannel(channels) + for { + select { + case event := <-aggregatedEventChannel: + tag := sa.hostMap[event.host] + metric := event.metric + + /* ensuring that we have upper buffer size of 5 sec */ + if _, exist := sa.aggregatedStatMap[metric.TimeStamp]; !exist { + if len(sa.aggregatedStatMap) >= MAX_STAT_BUFFER { + index := getMinIndex(sa.aggregatedStatMap) + populateConsolidatedStat(sa.aggregatedStatMap[index]) + sa.outputChannel <- sa.aggregatedStatMap[index] + delete(sa.aggregatedStatMap, index) + } + sa.aggregatedStatMap[metric.TimeStamp] = make(map[string]*stats.DKVMetrics) + } + + /* merging metrics*/ + if _, exist := sa.aggregatedStatMap[metric.TimeStamp][tag]; !exist { + metric.Count = 1 + sa.aggregatedStatMap[metric.TimeStamp][tag] = &metric + } else { + sa.aggregatedStatMap[metric.TimeStamp][tag].Merge(metric) + + } + + /* flushing when all metrics are aggregated */ + if getStatCount(sa.aggregatedStatMap[metric.TimeStamp]) == len(sa.hostMap) { + populateConsolidatedStat(sa.aggregatedStatMap[metric.TimeStamp]) + sa.outputChannel <- sa.aggregatedStatMap[metric.TimeStamp] + delete(sa.aggregatedStatMap, metric.TimeStamp) + } + + case <-sa.ctx.Done(): + for host, channelId := range sa.channelIds { + listener.DeRegister(host, channelId) + } + close(sa.outputChannel) + return + } + } +} + +func (sa *StatAggregator) Stop() { + sa.cancelFunc() +} + +func getStatCount(metricMap map[string]*stats.DKVMetrics) int { + count := 0 + for _, metric := range metricMap { + count = count + int(metric.Count) + } + return count +} + +func getMinIndex(m map[int64]map[string]*stats.DKVMetrics) int64 { + var min int64 + for index, _ := range m { + if min == 0 || min > index { + min = index + } + } + return min +} + +func populateConsolidatedStat(m map[string]*stats.DKVMetrics) { + dkvMetrics := stats.NewDKVMetric() + for _, dm := range m { + /* it should contain ts of the metric that it is combining */ + dkvMetrics.TimeStamp = dm.TimeStamp + dkvMetrics.Merge(*dm) + } + m["global"] = dkvMetrics +} +func (sa *StatAggregator) getMultiplexedChannel(channels []chan MetricEvent) chan MetricEvent { + /* Channel to Write Multiplexed Events */ + aggregatedSseEvents := make(chan MetricEvent, MAX_STAT_BUFFER) + + /* Start all Multiplexing Go Routines with Context */ + for _, channel := range channels { + go func(evntChan chan MetricEvent) { + for { + select { + case <-sa.ctx.Done(): + fmt.Println("Context Signal Received Exiting Multiplexer Routine") + return + case event := <-evntChan: + /* Write received event onto aggregated channel */ + aggregatedSseEvents <- event + } + } + }(channel) + } + return aggregatedSseEvents +} diff --git a/internal/stats/aggregate/collector.go b/internal/stats/aggregate/collector.go new file mode 100644 index 00000000..6161c391 --- /dev/null +++ b/internal/stats/aggregate/collector.go @@ -0,0 +1,182 @@ +package aggregate + +import ( + "bufio" + "bytes" + "context" + "encoding/json" + "errors" + "net/http" + "sync" + "time" + + "github.com/flipkart-incubator/dkv/internal/stats" +) + +type StatListener struct { + /* Time Stamp to Channel Map for Writing Output */ + listenerInfoMap map[string]*ListenerInfo + /* Mutex for Safe Access */ + mapMutex sync.Mutex +} + +type MetricEvent struct { + metric stats.DKVMetrics + host string +} + +func NewStatListener() *StatListener { + return &StatListener{listenerInfoMap: make(map[string]*ListenerInfo)} +} + +func (sl *StatListener) Register(host string, outputChannel chan MetricEvent) (int64, error) { + sl.mapMutex.Lock() + defer sl.mapMutex.Unlock() + if _, exists := sl.listenerInfoMap[host]; !exists { + newListenerInfo := NewListenerInfo(host) + if err := newListenerInfo.Start(); err != nil { + return 0, err + } + sl.listenerInfoMap[host] = newListenerInfo + } + return sl.listenerInfoMap[host].Register(outputChannel), nil +} + +func (sl *StatListener) DeRegister(host string, id int64) { + sl.mapMutex.Lock() + if li, exists := sl.listenerInfoMap[host]; exists { + li.DeRegister(id) + if li.GetChannelCount() == 0 { + li.Stop() + delete(sl.listenerInfoMap, host) + } + } + defer sl.mapMutex.Unlock() +} + +type ListenerInfo struct { + host string + outputChannelMap map[int64]chan MetricEvent + inputChannel <-chan MetricEvent + mapMutex sync.Mutex + ctx context.Context + cancelFunc context.CancelFunc +} + +func NewListenerInfo(host string) *ListenerInfo { + ctx, cancelFunc := context.WithCancel(context.Background()) + return &ListenerInfo{host: host, outputChannelMap: make(map[int64]chan MetricEvent, 2), inputChannel: make(<-chan MetricEvent, 5), ctx: ctx, cancelFunc: cancelFunc} +} + +func (li *ListenerInfo) GetChannelCount() int { + return len(li.outputChannelMap) +} + +func (li *ListenerInfo) Start() error { + var err error + if li.inputChannel, err = getStreamChannel(li.host, li.ctx); err == nil { + go li.BroadCast() + } + return err +} + +func (li *ListenerInfo) BroadCast() { + for { + select { + case e := <-li.inputChannel: + li.mapMutex.Lock() + for _, outputChan := range li.outputChannelMap { + outputChan <- e + } + li.mapMutex.Unlock() + case <-li.ctx.Done(): + li.mapMutex.Lock() + for _, outputChan := range li.outputChannelMap { + close(outputChan) + } + li.mapMutex.Unlock() + } + } +} + +func getStreamChannel(host string, ctx context.Context) (<-chan MetricEvent, error) { + + client := &http.Client{} + transport := &http.Transport{} + transport.DisableCompression = true + client.Transport = transport + request, err := http.NewRequest("GET", "http://"+host+"/metrics/stream", nil) + if err != nil { + return nil, err + } + /* Add Header to accept streaming events */ + request.Header.Set("Accept", "text/event-stream") + /* Make Channel to Report Events */ + eventChannel := make(chan MetricEvent, 5) + /* Ensure Request gets Cancelled along with Context */ + requestWithContext := request.WithContext(ctx) + /* Fire Request */ + if response, err := client.Do(requestWithContext); err == nil { + /* Open a Reader on Response Body */ + go parseEvent(response, eventChannel, host, ctx) + } else { + return nil, err + } + return eventChannel, nil +} + +func parseEvent(response *http.Response, eventChannel chan MetricEvent, host string, ctx context.Context) { + defer response.Body.Close() + br := bufio.NewReader(response.Body) + for { + select { + case <-ctx.Done(): + close(eventChannel) + return + default: + /* Read Lines Upto Delimiter */ + if readBytes, err := br.ReadBytes('\n'); err == nil { + if event, err := buildEvent(readBytes); err == nil { + eventChannel <- MetricEvent{metric: *event, host: host} + } + } + } + } +} + +func buildEvent(byts []byte) (*stats.DKVMetrics, error) { + splits := bytes.Split(byts, []byte{':', ' '}) + if len(splits) == 2 { + dkvMetrics := &stats.DKVMetrics{} + err := json.Unmarshal(splits[1], dkvMetrics) + return dkvMetrics, err + } + return nil, errors.New("Invalid Response") +} + +func (li *ListenerInfo) Stop() { + li.cancelFunc() +} + +func (li *ListenerInfo) Register(outputChannel chan MetricEvent) int64 { + channelId := time.Now().UnixNano() + li.mapMutex.Lock() + li.outputChannelMap[channelId] = outputChannel + li.mapMutex.Unlock() + return channelId +} + +func (li *ListenerInfo) DeRegister(id int64) { + li.mapMutex.Lock() + if outputChannel, ok := li.outputChannelMap[id]; ok { + li.unsafeDeregister(outputChannel, id) + } + li.mapMutex.Unlock() +} + +func (li *ListenerInfo) unsafeDeregister(outputChannel chan MetricEvent, id int64) { + /* Close Channel */ + close(outputChannel) + /* Delete current Channel from Broadcast Map */ + delete(li.outputChannelMap, id) +} diff --git a/internal/stats/models.go b/internal/stats/models.go new file mode 100644 index 00000000..064c58c1 --- /dev/null +++ b/internal/stats/models.go @@ -0,0 +1,113 @@ +package stats + +import ( + "time" + + dto "github.com/prometheus/client_model/go" +) + +const ( + Ops = "ops" + Put = "put" + Get = "get" + MultiGet = "mget" + Delete = "del" + GetSnapShot = "getSnapShot" + PutSnapShot = "putSnapShot" + Iterate = "iter" + CompareAndSet = "cas" + LoadChange = "loadChange" + SaveChange = "saveChange" +) + +type DKVMetrics struct { + TimeStamp int64 `json:"ts"` + StoreLatency map[string]*Percentile `json:"storage_latency"` + NexusLatency map[string]*Percentile `json:"nexus_latency"` + DKVLatency map[string]*Percentile `json:"dkv_latency"` + StorageOpsCount map[string]uint64 `json:"storage_ops_count"` + StorageOpsErrorCount map[string]float64 `json:"storage_ops_error_count"` + NexusOpsCount map[string]uint64 `json:"nexus_ops_count"` + DKVReqCount map[string]uint64 `json:"dkv_req_count"` + Count float64 `json:"count"` +} + +func (dm *DKVMetrics) Merge(dm1 DKVMetrics) { + dm.StoreLatency = MergeMapPercentile(dm.StoreLatency, dm1.StoreLatency, dm.Count) + dm.NexusLatency = MergeMapPercentile(dm.NexusLatency, dm1.NexusLatency, dm.Count) + dm.DKVLatency = MergeMapPercentile(dm.DKVLatency, dm1.DKVLatency, dm.Count) + dm.StorageOpsCount = MergeMapUint64(dm.StorageOpsCount, dm1.StorageOpsCount) + dm.StorageOpsErrorCount = MergeMapFloat64(dm.StorageOpsErrorCount, dm.StorageOpsErrorCount) + dm.NexusOpsCount = MergeMapUint64(dm.NexusOpsCount, dm1.NexusOpsCount) + dm.DKVReqCount = MergeMapUint64(dm.DKVReqCount, dm1.DKVReqCount) + dm.Count = dm.Count + 1 +} + +func MergeMapUint64(m1, m2 map[string]uint64) map[string]uint64 { + for k, v := range m2 { + if _, exist := m1[k]; exist { + m1[k] = m1[k] + v + } else { + m1[k] = v + } + } + return m1 +} + +func MergeMapFloat64(m1, m2 map[string]float64) map[string]float64 { + for k, v := range m2 { + if _, exist := m1[k]; exist { + m1[k] = m1[k] + v + } else { + m1[k] = v + } + } + return m1 +} + +func MergeMapPercentile(m1, m2 map[string]*Percentile, count float64) map[string]*Percentile { + for k, v := range m2 { + if _, exist := m1[k]; exist { + m1[k].P50 = (m1[k].P50*count + m2[k].P50) / (count + 1) + m1[k].P90 = (m1[k].P90*count + m2[k].P90) / (count + 1) + m1[k].P99 = (m1[k].P99*count + m2[k].P99) / (count + 1) + } else { + m1[k] = v + } + } + return m1 +} + +type Percentile struct { + P50 float64 `json:"p50"` + P90 float64 `json:"p90"` + P99 float64 `json:"p99"` +} + +func NewDKVMetric() *DKVMetrics { + return &DKVMetrics{ + TimeStamp: time.Now().Unix(), + StoreLatency: make(map[string]*Percentile), + NexusLatency: make(map[string]*Percentile), + DKVLatency: make(map[string]*Percentile), + StorageOpsCount: make(map[string]uint64), + StorageOpsErrorCount: make(map[string]float64), + NexusOpsCount: make(map[string]uint64), + DKVReqCount: make(map[string]uint64), + Count: 1, + } +} +func NewPercentile(quantile []*dto.Quantile) *Percentile { + percentile := &Percentile{} + for _, q := range quantile { + switch *q.Quantile { + case 0.5: + percentile.P50 = *q.Value + case 0.9: + percentile.P90 = *q.Value + case 0.99: + percentile.P99 = *q.Value + } + } + return percentile +} diff --git a/internal/stats/prometheus.go b/internal/stats/prometheus.go new file mode 100644 index 00000000..28a8be0f --- /dev/null +++ b/internal/stats/prometheus.go @@ -0,0 +1,84 @@ +package stats + +import ( + "log" + "time" + + "github.com/prometheus/client_golang/prometheus" +) + +type promethousRegistry struct{} + +func (*promethousRegistry) Register(c prometheus.Collector) error { + return prometheus.DefaultRegisterer.Register(c) +} + +func (r *promethousRegistry) MustRegister(cs ...prometheus.Collector) { + for _, c := range cs { + if err := r.Register(c); err != nil { + if metric, ok := c.(prometheus.Metric); ok { + log.Printf("Failed to register collector %s: %s", metric.Desc().String(), err) + } else { + log.Printf("Failed to register collector: %s", err) + } + } + } +} + +func (*promethousRegistry) Unregister(c prometheus.Collector) bool { + return prometheus.DefaultRegisterer.Unregister(c) +} + +func NewPromethousRegistry() prometheus.Registerer { + return &promethousRegistry{} +} + +func (*noopClient) Register(collector prometheus.Collector) error { + return nil +} + +func (*noopClient) MustRegister(collectors ...prometheus.Collector) {} + +func (*noopClient) Unregister(collector prometheus.Collector) bool { + return true +} + +func NewPromethousNoopRegistry() prometheus.Registerer { + return &noopClient{} +} + +func MeasureLatency(observer prometheus.Observer, startTime time.Time) { + observer.Observe(time.Since(startTime).Seconds()) +} + +func GetMetrics() (*DKVMetrics, error) { + dkvMetrics := NewDKVMetric() + mfs, err := prometheus.DefaultGatherer.Gather() + if err != nil { + return dkvMetrics, err + } + for _, mf := range mfs { + switch mf.GetName() { + case "storage_latency": + for _, m := range mf.GetMetric() { + dkvMetrics.StoreLatency[m.Label[0].GetValue()] = NewPercentile(m.GetSummary().GetQuantile()) + dkvMetrics.StorageOpsCount[m.Label[0].GetValue()] = m.GetSummary().GetSampleCount() + } + case "nexus_latency": + for _, m := range mf.GetMetric() { + dkvMetrics.NexusLatency[m.Label[0].GetValue()] = NewPercentile(m.GetSummary().GetQuantile()) + dkvMetrics.NexusOpsCount[m.Label[0].GetValue()] = m.GetSummary().GetSampleCount() + } + case "dkv_latency": + for _, m := range mf.GetMetric() { + dkvMetrics.DKVLatency[m.Label[0].GetValue()] = NewPercentile(m.GetSummary().GetQuantile()) + dkvMetrics.DKVReqCount[m.Label[0].GetValue()] = m.GetSummary().GetSampleCount() + } + case "storage_error": + for _, m := range mf.GetMetric() { + dkvMetrics.StorageOpsErrorCount[m.Label[0].GetValue()] = m.GetCounter().GetValue() + } + } + } + return dkvMetrics, nil +} diff --git a/internal/stats/streamer.go b/internal/stats/streamer.go new file mode 100644 index 00000000..6a8320f3 --- /dev/null +++ b/internal/stats/streamer.go @@ -0,0 +1,61 @@ +package stats + +import ( + "sync" + "time" +) + +type StatStreamer struct { + /* Time Stamp to Channel Map for Writing Output */ + outputChannelMap map[int64]chan DKVMetrics + /* Mutex for Safe Access */ + mapMutex sync.Mutex +} + +func NewStatStreamer() *StatStreamer { + return &StatStreamer{ + outputChannelMap: make(map[int64]chan DKVMetrics, 10), + } +} + +func (sp *StatStreamer) Register(outputChannel chan DKVMetrics) int64 { + channelId := time.Now().UnixNano() + sp.mapMutex.Lock() + sp.outputChannelMap[channelId] = outputChannel + sp.mapMutex.Unlock() + return channelId +} + +func (sp *StatStreamer) DeRegister(id int64) { + sp.mapMutex.Lock() + if outputChannel, ok := sp.outputChannelMap[id]; ok { + sp.unsafeDeregister(outputChannel, id) + } + sp.mapMutex.Unlock() +} + +func (sp *StatStreamer) unsafeDeregister(outputChannel chan DKVMetrics, id int64) { + /* Close Channel */ + close(outputChannel) + /* Delete current Channel from Broadcast Map */ + delete(sp.outputChannelMap, id) +} + +func (sp *StatStreamer) Run() { + ticker := time.NewTicker(time.Second) + for { + select { + case <-ticker.C: + dkvMetrics, _ := GetMetrics() + sp.mapMutex.Lock() + for id, outputChannel := range sp.outputChannelMap { + select { + case outputChannel <- *dkvMetrics: + default: + sp.unsafeDeregister(outputChannel, id) + } + } + sp.mapMutex.Unlock() + } + } +} diff --git a/internal/storage/badger/store.go b/internal/storage/badger/store.go index 31171920..3bc10f09 100644 --- a/internal/storage/badger/store.go +++ b/internal/storage/badger/store.go @@ -17,6 +17,7 @@ import ( "time" "github.com/matttproud/golang_protobuf_extensions/pbutil" + "github.com/prometheus/client_golang/prometheus" "github.com/dgraph-io/badger/v3" "github.com/flipkart-incubator/dkv/internal/stats" @@ -39,6 +40,7 @@ type DB interface { type badgerDB struct { db *badger.DB opts *bdgrOpts + stat *storage.Stat // Indicates a global mutation like backup and restore that // require exclusivity. Shall be manipulated using atomics. @@ -50,6 +52,7 @@ type bdgrOpts struct { lgr *zap.Logger statsCli stats.Client sstDirectory string + promRegistry prometheus.Registerer } // DBOption is used to configure the Badger @@ -77,6 +80,17 @@ func WithStats(statsCli stats.Client) DBOption { } } +// WithPromStats is used to inject a prometheus metrics instance +func WithPromStats(registry prometheus.Registerer) DBOption { + return func(opts *bdgrOpts) { + if registry != nil { + opts.promRegistry = registry + } else { + opts.promRegistry = stats.NewPromethousNoopRegistry() + } + } +} + // WithSyncWrites configures Badger to ensure every // write is flushed to disk before acking back. func WithSyncWrites() DBOption { @@ -156,9 +170,10 @@ func WithMemTableSize(size int64) DBOption { func OpenDB(dbOpts ...DBOption) (kvs DB, err error) { noopLgr := zap.NewNop() opts := &bdgrOpts{ - opts: badger.DefaultOptions("").WithLogger(&zapBadgerLogger{lgr: noopLgr}), - lgr: noopLgr, - statsCli: stats.NewNoOpClient(), + opts: badger.DefaultOptions("").WithLogger(&zapBadgerLogger{lgr: noopLgr}), + lgr: noopLgr, + statsCli: stats.NewNoOpClient(), + promRegistry: stats.NewPromethousNoopRegistry(), } for _, dbOpt := range dbOpts { dbOpt(opts) @@ -171,7 +186,7 @@ func openStore(bdbOpts *bdgrOpts) (*badgerDB, error) { if err != nil { return nil, err } - return &badgerDB{db, bdbOpts, 0}, nil + return &badgerDB{db, bdbOpts, storage.NewStat(bdbOpts.promRegistry), 0}, nil } func (bdb *badgerDB) Close() error { @@ -180,6 +195,7 @@ func (bdb *badgerDB) Close() error { } func (bdb *badgerDB) Put(pairs ...*serverpb.KVPair) error { + /* todo stat computation */ metricsPrefix := "badger.put.multi" if len(pairs) == 1 { metricsPrefix = "badger.put.single" @@ -210,17 +226,22 @@ func (bdb *badgerDB) Put(pairs ...*serverpb.KVPair) error { func (bdb *badgerDB) Delete(key []byte) error { defer bdb.opts.statsCli.Timing("badger.delete.latency.ms", time.Now()) + defer stats.MeasureLatency(bdb.stat.RequestLatency.WithLabelValues(stats.Delete), time.Now()) + err := bdb.db.Update(func(txn *badger.Txn) error { return txn.Delete(key) }) if err != nil { bdb.opts.statsCli.Incr("badger.delete.errors", 1) + bdb.stat.ResponseError.WithLabelValues(stats.Delete).Inc() } return err } func (bdb *badgerDB) Get(keys ...[]byte) ([]*serverpb.KVPair, error) { defer bdb.opts.statsCli.Timing("badger.get.latency.ms", time.Now()) + defer stats.MeasureLatency(bdb.stat.RequestLatency.WithLabelValues(stats.Get), time.Now()) + var results []*serverpb.KVPair err := bdb.db.View(func(txn *badger.Txn) error { for _, key := range keys { @@ -239,12 +260,15 @@ func (bdb *badgerDB) Get(keys ...[]byte) ([]*serverpb.KVPair, error) { }) if err != nil { bdb.opts.statsCli.Incr("badger.get.errors", 1) + bdb.stat.ResponseError.WithLabelValues(stats.Get).Inc() } return results, err } func (bdb *badgerDB) CompareAndSet(key, expect, update []byte) (bool, error) { defer bdb.opts.statsCli.Timing("badger.cas.latency.ms", time.Now()) + defer stats.MeasureLatency(bdb.stat.RequestLatency.WithLabelValues(stats.CompareAndSet), time.Now()) + casTrxn := bdb.db.NewTransaction(true) defer casTrxn.Discard() @@ -256,6 +280,7 @@ func (bdb *badgerDB) CompareAndSet(key, expect, update []byte) (bool, error) { } case err != nil: bdb.opts.statsCli.Incr("badger.cas.get.errors", 1) + bdb.stat.ResponseError.WithLabelValues(stats.CompareAndSet).Inc() return false, err default: existVal, _ := exist.ValueCopy(nil) @@ -266,6 +291,7 @@ func (bdb *badgerDB) CompareAndSet(key, expect, update []byte) (bool, error) { err = casTrxn.Set(key, update) if err != nil { bdb.opts.statsCli.Incr("badger.cas.set.errors", 1) + bdb.stat.ResponseError.WithLabelValues(stats.CompareAndSet).Inc() return false, err } err = casTrxn.Commit() @@ -281,6 +307,7 @@ const ( func (bdb *badgerDB) GetSnapshot() (io.ReadCloser, error) { defer bdb.opts.statsCli.Timing("badger.snapshot.get.latency.ms", time.Now()) + defer stats.MeasureLatency(bdb.stat.RequestLatency.WithLabelValues(stats.GetSnapShot), time.Now()) sstFile, err := storage.CreateTempFile(bdb.opts.sstDirectory, badgerSSTPrefix) if err != nil { @@ -318,6 +345,7 @@ func (bdb *badgerDB) GetSnapshot() (io.ReadCloser, error) { func (bdb *badgerDB) PutSnapshot(snap io.ReadCloser) error { defer bdb.opts.statsCli.Timing("badger.snapshot.put.latency.ms", time.Now()) + defer stats.MeasureLatency(bdb.stat.RequestLatency.WithLabelValues(stats.PutSnapShot), time.Now()) wb := bdb.db.NewWriteBatch() defer wb.Cancel() @@ -476,6 +504,8 @@ func (bdb *badgerDB) GetLatestAppliedChangeNumber() (uint64, error) { func (bdb *badgerDB) SaveChanges(changes []*serverpb.ChangeRecord) (uint64, error) { defer bdb.opts.statsCli.Timing("badger.save.changes.latency.ms", time.Now()) + defer stats.MeasureLatency(bdb.stat.RequestLatency.WithLabelValues(stats.SaveChange), time.Now()) + var appldChngNum uint64 var lastErr error diff --git a/internal/storage/rocksdb/store.go b/internal/storage/rocksdb/store.go index 9592104f..48638856 100644 --- a/internal/storage/rocksdb/store.go +++ b/internal/storage/rocksdb/store.go @@ -16,6 +16,7 @@ import ( "github.com/flipkart-incubator/dkv/internal/hlc" "github.com/flipkart-incubator/dkv/internal/storage/iterators" "github.com/flipkart-incubator/dkv/internal/storage/utils" + "github.com/prometheus/client_golang/prometheus" "github.com/vmihailenco/msgpack/v5" "github.com/flipkart-incubator/dkv/internal/stats" @@ -41,6 +42,7 @@ type rocksDB struct { ttlCF *gorocksdb.ColumnFamilyHandle optimTrxnDB *gorocksdb.OptimisticTransactionDB opts *rocksDBOpts + stat *storage.Stat // Indicates a global mutation like backup and restore that // require exclusivity. Shall be manipulated using atomics. @@ -58,6 +60,7 @@ type rocksDBOpts struct { lgr *zap.Logger statsCli stats.Client cfNames []string + promRegistry prometheus.Registerer } // DBOption is used to configure the RocksDB @@ -75,6 +78,17 @@ func WithLogger(lgr *zap.Logger) DBOption { } } +// WithPromStats is used to inject a prometheus stats instance +func WithPromStats(registry prometheus.Registerer) DBOption { + return func(opts *rocksDBOpts) { + if registry != nil { + opts.promRegistry = registry + } else { + opts.promRegistry = stats.NewPromethousNoopRegistry() + } + } +} + // WithStats is used to inject a metrics client. func WithStats(statsCli stats.Client) DBOption { return func(opts *rocksDBOpts) { @@ -175,6 +189,7 @@ func newOptions(dbFolder string) *rocksDBOpts { opts := gorocksdb.NewDefaultOptions() opts.SetCreateIfMissing(true) opts.SetCreateIfMissingColumnFamilies(true) + opts.SetWALTtlSeconds(uint64(600)) opts.SetBlockBasedTableFactory(bbto) rstOpts := gorocksdb.NewRestoreOptions() wrOpts := gorocksdb.NewDefaultWriteOptions() @@ -190,6 +205,7 @@ func newOptions(dbFolder string) *rocksDBOpts { writeOpts: wrOpts, statsCli: stats.NewNoOpClient(), cfNames: cfNames, + promRegistry: stats.NewPromethousNoopRegistry(), } } @@ -221,6 +237,7 @@ func openStore(opts *rocksDBOpts) (*rocksDB, error) { optimTrxnDB: optimTrxnDB, opts: opts, globalMutation: 0, + stat: storage.NewStat(opts.promRegistry), } //TODO: revisit this later after understanding what is the impact of manually triggered compaction //go rocksdb.Compaction() @@ -323,6 +340,8 @@ func (rdb *rocksDB) Put(pairs ...*serverpb.KVPair) error { func (rdb *rocksDB) Delete(key []byte) error { defer rdb.opts.statsCli.Timing("rocksdb.delete.latency.ms", time.Now()) + defer stats.MeasureLatency(rdb.stat.RequestLatency.WithLabelValues(stats.Delete), time.Now()) + wb := gorocksdb.NewWriteBatch() defer wb.Destroy() wb.DeleteCF(rdb.ttlCF, key) @@ -330,6 +349,7 @@ func (rdb *rocksDB) Delete(key []byte) error { err := rdb.db.Write(rdb.opts.writeOpts, wb) if err != nil { rdb.opts.statsCli.Incr("rocksdb.delete.errors", 1) + rdb.stat.ResponseError.WithLabelValues(stats.Delete).Inc() } return err } @@ -346,6 +366,8 @@ func (rdb *rocksDB) Get(keys ...[]byte) ([]*serverpb.KVPair, error) { func (rdb *rocksDB) CompareAndSet(key, expect, update []byte) (bool, error) { defer rdb.opts.statsCli.Timing("rocksdb.cas.latency.ms", time.Now()) + defer stats.MeasureLatency(rdb.stat.RequestLatency.WithLabelValues(stats.CompareAndSet), time.Now()) + ro := rdb.opts.readOpts wo := rdb.opts.writeOpts to := gorocksdb.NewDefaultOptimisticTransactionOptions() @@ -371,6 +393,7 @@ func (rdb *rocksDB) CompareAndSet(key, expect, update []byte) (bool, error) { err = txn.Put(key, update) if err != nil { rdb.opts.statsCli.Incr("rocksdb.cas.set.errors", 1) + rdb.stat.ResponseError.WithLabelValues(stats.CompareAndSet).Inc() return false, err } err = txn.Commit() @@ -444,6 +467,7 @@ func (r *checkPointSnapshot) Close() error { func (rdb *rocksDB) GetSnapshot() (io.ReadCloser, error) { defer rdb.opts.statsCli.Timing("rocksdb.snapshot.get.latency.ms", time.Now()) + defer stats.MeasureLatency(rdb.stat.RequestLatency.WithLabelValues(stats.GetSnapShot), time.Now()) //Prevent any other backups or restores err := rdb.beginGlobalMutation() @@ -507,6 +531,7 @@ func (rdb *rocksDB) PutSnapshot(snap io.ReadCloser) error { return nil } defer rdb.opts.statsCli.Timing("rocksdb.snapshot.put.latency.ms", time.Now()) + defer stats.MeasureLatency(rdb.stat.RequestLatency.WithLabelValues(stats.PutSnapShot), time.Now()) defer snap.Close() //Prevent any other backups or restores @@ -616,6 +641,8 @@ func (rdb *rocksDB) GetLatestCommittedChangeNumber() (uint64, error) { func (rdb *rocksDB) LoadChanges(fromChangeNumber uint64, maxChanges int) ([]*serverpb.ChangeRecord, error) { defer rdb.opts.statsCli.Timing("rocksdb.load.changes.latency.ms", time.Now()) + defer stats.MeasureLatency(rdb.stat.RequestLatency.WithLabelValues(stats.LoadChange), time.Now()) + chngIter, err := rdb.db.GetUpdatesSince(fromChangeNumber) if err != nil { return nil, err @@ -638,6 +665,8 @@ func (rdb *rocksDB) GetLatestAppliedChangeNumber() (uint64, error) { func (rdb *rocksDB) SaveChanges(changes []*serverpb.ChangeRecord) (uint64, error) { defer rdb.opts.statsCli.Timing("rocksdb.save.changes.latency.ms", time.Now()) + defer stats.MeasureLatency(rdb.stat.RequestLatency.WithLabelValues(stats.SaveChange), time.Now()) + appldChngNum := uint64(0) for _, chng := range changes { wb := gorocksdb.WriteBatchFrom(chng.SerialisedForm) @@ -806,9 +835,12 @@ func parseTTLMsgPackData(valueWithTTL []byte) (*ttlDataFormat, error) { func (rdb *rocksDB) getSingleKey(ro *gorocksdb.ReadOptions, key []byte) ([]*serverpb.KVPair, error) { defer rdb.opts.statsCli.Timing("rocksdb.single.get.latency.ms", time.Now()) + defer stats.MeasureLatency(rdb.stat.RequestLatency.WithLabelValues(stats.Get), time.Now()) + values, err := rdb.db.MultiGetCFMultiCF(ro, []*gorocksdb.ColumnFamilyHandle{rdb.normalCF, rdb.ttlCF}, [][]byte{key, key}) if err != nil { rdb.opts.statsCli.Incr("rocksdb.single.get.errors", 1) + rdb.stat.ResponseError.WithLabelValues(stats.Get).Inc() return nil, err } value1, value2 := values[0], values[1] @@ -851,6 +883,7 @@ func (rdb *rocksDB) extractResult(value1 *gorocksdb.Slice, value2 *gorocksdb.Sli func (rdb *rocksDB) getMultipleKeys(ro *gorocksdb.ReadOptions, keys [][]byte) ([]*serverpb.KVPair, error) { defer rdb.opts.statsCli.Timing("rocksdb.multi.get.latency.ms", time.Now()) + defer stats.MeasureLatency(rdb.stat.RequestLatency.WithLabelValues(stats.MultiGet), time.Now()) kl := len(keys) reqCFs := make([]*gorocksdb.ColumnFamilyHandle, kl<<1) @@ -862,6 +895,7 @@ func (rdb *rocksDB) getMultipleKeys(ro *gorocksdb.ReadOptions, keys [][]byte) ([ values, err := rdb.db.MultiGetCFMultiCF(ro, reqCFs, append(keys, keys...)) if err != nil { rdb.opts.statsCli.Incr("rocksdb.multi.get.errors", 1) + rdb.stat.ResponseError.WithLabelValues(stats.MultiGet).Inc() return nil, err } diff --git a/internal/storage/store.go b/internal/storage/store.go index f9e3b072..15c5c64c 100644 --- a/internal/storage/store.go +++ b/internal/storage/store.go @@ -6,9 +6,34 @@ import ( "os" "time" + "github.com/flipkart-incubator/dkv/internal/stats" + "github.com/prometheus/client_golang/prometheus" + "github.com/flipkart-incubator/dkv/pkg/serverpb" ) +type Stat struct { + RequestLatency *prometheus.SummaryVec + ResponseError *prometheus.CounterVec +} + +func NewStat(registry prometheus.Registerer) *Stat { + RequestLatency := prometheus.NewSummaryVec(prometheus.SummaryOpts{ + Namespace: "storage", + Name: "latency", + Help: "Latency statistics for storage operations", + Objectives: map[float64]float64{0.5: 0.05, 0.9: 0.01, 0.99: 0.001}, + MaxAge: 10 * time.Second, + }, []string{stats.Ops}) + ResponseError := prometheus.NewCounterVec(prometheus.CounterOpts{ + Namespace: "storage", + Name: "error", + Help: "Error count for storage operations", + }, []string{stats.Ops}) + registry.MustRegister(RequestLatency, ResponseError) + return &Stat{RequestLatency, ResponseError} +} + // A KVStore represents the key value store that provides // the underlying storage implementation for the various // DKV operations. diff --git a/pkg/serverpb/admin.pb.go b/pkg/serverpb/admin.pb.go index b07244b1..453da423 100644 --- a/pkg/serverpb/admin.pb.go +++ b/pkg/serverpb/admin.pb.go @@ -1039,6 +1039,9 @@ type RegionInfo struct { // Nexus cluster url of the region // Will be used by new followers to discover the raft cluster NexusClusterUrl *string `protobuf:"bytes,7,opt,name=nexusClusterUrl,proto3,oneof" json:"nexusClusterUrl,omitempty"` + // http listener of the node. + // http endpoint is useful for admin API interactions + HttpAddress string `protobuf:"bytes,8,opt,name=httpAddress,proto3" json:"httpAddress,omitempty"` } func (x *RegionInfo) Reset() { @@ -1122,6 +1125,13 @@ func (x *RegionInfo) GetNexusClusterUrl() string { return "" } +func (x *RegionInfo) GetHttpAddress() string { + if x != nil { + return x.HttpAddress + } + return "" +} + var File_pkg_serverpb_admin_proto protoreflect.FileDescriptor var file_pkg_serverpb_admin_proto_rawDesc = []byte{ @@ -1235,7 +1245,7 @@ var file_pkg_serverpb_admin_proto_rawDesc = []byte{ 0x12, 0x3a, 0x0a, 0x0b, 0x72, 0x65, 0x67, 0x69, 0x6f, 0x6e, 0x49, 0x6e, 0x66, 0x6f, 0x73, 0x18, 0x01, 0x20, 0x03, 0x28, 0x0b, 0x32, 0x18, 0x2e, 0x64, 0x6b, 0x76, 0x2e, 0x73, 0x65, 0x72, 0x76, 0x65, 0x72, 0x70, 0x62, 0x2e, 0x52, 0x65, 0x67, 0x69, 0x6f, 0x6e, 0x49, 0x6e, 0x66, 0x6f, 0x52, - 0x0b, 0x72, 0x65, 0x67, 0x69, 0x6f, 0x6e, 0x49, 0x6e, 0x66, 0x6f, 0x73, 0x22, 0xa3, 0x02, 0x0a, + 0x0b, 0x72, 0x65, 0x67, 0x69, 0x6f, 0x6e, 0x49, 0x6e, 0x66, 0x6f, 0x73, 0x22, 0xc5, 0x02, 0x0a, 0x0a, 0x52, 0x65, 0x67, 0x69, 0x6f, 0x6e, 0x49, 0x6e, 0x66, 0x6f, 0x12, 0x12, 0x0a, 0x04, 0x64, 0x63, 0x49, 0x44, 0x18, 0x01, 0x20, 0x01, 0x28, 0x09, 0x52, 0x04, 0x64, 0x63, 0x49, 0x44, 0x12, 0x20, 0x0a, 0x0b, 0x6e, 0x6f, 0x64, 0x65, 0x41, 0x64, 0x64, 0x72, 0x65, 0x73, 0x73, 0x18, 0x02, @@ -1251,78 +1261,80 @@ var file_pkg_serverpb_admin_proto_rawDesc = []byte{ 0x00, 0x52, 0x0a, 0x6d, 0x61, 0x73, 0x74, 0x65, 0x72, 0x48, 0x6f, 0x73, 0x74, 0x88, 0x01, 0x01, 0x12, 0x2d, 0x0a, 0x0f, 0x6e, 0x65, 0x78, 0x75, 0x73, 0x43, 0x6c, 0x75, 0x73, 0x74, 0x65, 0x72, 0x55, 0x72, 0x6c, 0x18, 0x07, 0x20, 0x01, 0x28, 0x09, 0x48, 0x01, 0x52, 0x0f, 0x6e, 0x65, 0x78, - 0x75, 0x73, 0x43, 0x6c, 0x75, 0x73, 0x74, 0x65, 0x72, 0x55, 0x72, 0x6c, 0x88, 0x01, 0x01, 0x42, - 0x0d, 0x0a, 0x0b, 0x5f, 0x6d, 0x61, 0x73, 0x74, 0x65, 0x72, 0x48, 0x6f, 0x73, 0x74, 0x42, 0x12, - 0x0a, 0x10, 0x5f, 0x6e, 0x65, 0x78, 0x75, 0x73, 0x43, 0x6c, 0x75, 0x73, 0x74, 0x65, 0x72, 0x55, - 0x72, 0x6c, 0x2a, 0x68, 0x0a, 0x0c, 0x52, 0x65, 0x67, 0x69, 0x6f, 0x6e, 0x53, 0x74, 0x61, 0x74, - 0x75, 0x73, 0x12, 0x0c, 0x0a, 0x08, 0x49, 0x4e, 0x41, 0x43, 0x54, 0x49, 0x56, 0x45, 0x10, 0x00, - 0x12, 0x0a, 0x0a, 0x06, 0x4c, 0x45, 0x41, 0x44, 0x45, 0x52, 0x10, 0x01, 0x12, 0x14, 0x0a, 0x10, - 0x50, 0x52, 0x49, 0x4d, 0x41, 0x52, 0x59, 0x5f, 0x46, 0x4f, 0x4c, 0x4c, 0x4f, 0x57, 0x45, 0x52, - 0x10, 0x02, 0x12, 0x16, 0x0a, 0x12, 0x53, 0x45, 0x43, 0x4f, 0x4e, 0x44, 0x41, 0x52, 0x59, 0x5f, - 0x46, 0x4f, 0x4c, 0x4c, 0x4f, 0x57, 0x45, 0x52, 0x10, 0x03, 0x12, 0x10, 0x0a, 0x0c, 0x41, 0x43, - 0x54, 0x49, 0x56, 0x45, 0x5f, 0x53, 0x4c, 0x41, 0x56, 0x45, 0x10, 0x04, 0x32, 0xae, 0x02, 0x0a, - 0x0e, 0x44, 0x4b, 0x56, 0x52, 0x65, 0x70, 0x6c, 0x69, 0x63, 0x61, 0x74, 0x69, 0x6f, 0x6e, 0x12, - 0x4f, 0x0a, 0x0a, 0x47, 0x65, 0x74, 0x43, 0x68, 0x61, 0x6e, 0x67, 0x65, 0x73, 0x12, 0x1f, 0x2e, - 0x64, 0x6b, 0x76, 0x2e, 0x73, 0x65, 0x72, 0x76, 0x65, 0x72, 0x70, 0x62, 0x2e, 0x47, 0x65, 0x74, - 0x43, 0x68, 0x61, 0x6e, 0x67, 0x65, 0x73, 0x52, 0x65, 0x71, 0x75, 0x65, 0x73, 0x74, 0x1a, 0x20, - 0x2e, 0x64, 0x6b, 0x76, 0x2e, 0x73, 0x65, 0x72, 0x76, 0x65, 0x72, 0x70, 0x62, 0x2e, 0x47, 0x65, - 0x74, 0x43, 0x68, 0x61, 0x6e, 0x67, 0x65, 0x73, 0x52, 0x65, 0x73, 0x70, 0x6f, 0x6e, 0x73, 0x65, - 0x12, 0x39, 0x0a, 0x0a, 0x41, 0x64, 0x64, 0x52, 0x65, 0x70, 0x6c, 0x69, 0x63, 0x61, 0x12, 0x15, + 0x75, 0x73, 0x43, 0x6c, 0x75, 0x73, 0x74, 0x65, 0x72, 0x55, 0x72, 0x6c, 0x88, 0x01, 0x01, 0x12, + 0x20, 0x0a, 0x0b, 0x68, 0x74, 0x74, 0x70, 0x41, 0x64, 0x64, 0x72, 0x65, 0x73, 0x73, 0x18, 0x08, + 0x20, 0x01, 0x28, 0x09, 0x52, 0x0b, 0x68, 0x74, 0x74, 0x70, 0x41, 0x64, 0x64, 0x72, 0x65, 0x73, + 0x73, 0x42, 0x0d, 0x0a, 0x0b, 0x5f, 0x6d, 0x61, 0x73, 0x74, 0x65, 0x72, 0x48, 0x6f, 0x73, 0x74, + 0x42, 0x12, 0x0a, 0x10, 0x5f, 0x6e, 0x65, 0x78, 0x75, 0x73, 0x43, 0x6c, 0x75, 0x73, 0x74, 0x65, + 0x72, 0x55, 0x72, 0x6c, 0x2a, 0x68, 0x0a, 0x0c, 0x52, 0x65, 0x67, 0x69, 0x6f, 0x6e, 0x53, 0x74, + 0x61, 0x74, 0x75, 0x73, 0x12, 0x0c, 0x0a, 0x08, 0x49, 0x4e, 0x41, 0x43, 0x54, 0x49, 0x56, 0x45, + 0x10, 0x00, 0x12, 0x0a, 0x0a, 0x06, 0x4c, 0x45, 0x41, 0x44, 0x45, 0x52, 0x10, 0x01, 0x12, 0x14, + 0x0a, 0x10, 0x50, 0x52, 0x49, 0x4d, 0x41, 0x52, 0x59, 0x5f, 0x46, 0x4f, 0x4c, 0x4c, 0x4f, 0x57, + 0x45, 0x52, 0x10, 0x02, 0x12, 0x16, 0x0a, 0x12, 0x53, 0x45, 0x43, 0x4f, 0x4e, 0x44, 0x41, 0x52, + 0x59, 0x5f, 0x46, 0x4f, 0x4c, 0x4c, 0x4f, 0x57, 0x45, 0x52, 0x10, 0x03, 0x12, 0x10, 0x0a, 0x0c, + 0x41, 0x43, 0x54, 0x49, 0x56, 0x45, 0x5f, 0x53, 0x4c, 0x41, 0x56, 0x45, 0x10, 0x04, 0x32, 0xae, + 0x02, 0x0a, 0x0e, 0x44, 0x4b, 0x56, 0x52, 0x65, 0x70, 0x6c, 0x69, 0x63, 0x61, 0x74, 0x69, 0x6f, + 0x6e, 0x12, 0x4f, 0x0a, 0x0a, 0x47, 0x65, 0x74, 0x43, 0x68, 0x61, 0x6e, 0x67, 0x65, 0x73, 0x12, + 0x1f, 0x2e, 0x64, 0x6b, 0x76, 0x2e, 0x73, 0x65, 0x72, 0x76, 0x65, 0x72, 0x70, 0x62, 0x2e, 0x47, + 0x65, 0x74, 0x43, 0x68, 0x61, 0x6e, 0x67, 0x65, 0x73, 0x52, 0x65, 0x71, 0x75, 0x65, 0x73, 0x74, + 0x1a, 0x20, 0x2e, 0x64, 0x6b, 0x76, 0x2e, 0x73, 0x65, 0x72, 0x76, 0x65, 0x72, 0x70, 0x62, 0x2e, + 0x47, 0x65, 0x74, 0x43, 0x68, 0x61, 0x6e, 0x67, 0x65, 0x73, 0x52, 0x65, 0x73, 0x70, 0x6f, 0x6e, + 0x73, 0x65, 0x12, 0x39, 0x0a, 0x0a, 0x41, 0x64, 0x64, 0x52, 0x65, 0x70, 0x6c, 0x69, 0x63, 0x61, + 0x12, 0x15, 0x2e, 0x64, 0x6b, 0x76, 0x2e, 0x73, 0x65, 0x72, 0x76, 0x65, 0x72, 0x70, 0x62, 0x2e, + 0x52, 0x65, 0x70, 0x6c, 0x69, 0x63, 0x61, 0x1a, 0x14, 0x2e, 0x64, 0x6b, 0x76, 0x2e, 0x73, 0x65, + 0x72, 0x76, 0x65, 0x72, 0x70, 0x62, 0x2e, 0x53, 0x74, 0x61, 0x74, 0x75, 0x73, 0x12, 0x3c, 0x0a, + 0x0d, 0x52, 0x65, 0x6d, 0x6f, 0x76, 0x65, 0x52, 0x65, 0x70, 0x6c, 0x69, 0x63, 0x61, 0x12, 0x15, 0x2e, 0x64, 0x6b, 0x76, 0x2e, 0x73, 0x65, 0x72, 0x76, 0x65, 0x72, 0x70, 0x62, 0x2e, 0x52, 0x65, 0x70, 0x6c, 0x69, 0x63, 0x61, 0x1a, 0x14, 0x2e, 0x64, 0x6b, 0x76, 0x2e, 0x73, 0x65, 0x72, 0x76, - 0x65, 0x72, 0x70, 0x62, 0x2e, 0x53, 0x74, 0x61, 0x74, 0x75, 0x73, 0x12, 0x3c, 0x0a, 0x0d, 0x52, - 0x65, 0x6d, 0x6f, 0x76, 0x65, 0x52, 0x65, 0x70, 0x6c, 0x69, 0x63, 0x61, 0x12, 0x15, 0x2e, 0x64, - 0x6b, 0x76, 0x2e, 0x73, 0x65, 0x72, 0x76, 0x65, 0x72, 0x70, 0x62, 0x2e, 0x52, 0x65, 0x70, 0x6c, - 0x69, 0x63, 0x61, 0x1a, 0x14, 0x2e, 0x64, 0x6b, 0x76, 0x2e, 0x73, 0x65, 0x72, 0x76, 0x65, 0x72, - 0x70, 0x62, 0x2e, 0x53, 0x74, 0x61, 0x74, 0x75, 0x73, 0x12, 0x52, 0x0a, 0x0b, 0x47, 0x65, 0x74, - 0x52, 0x65, 0x70, 0x6c, 0x69, 0x63, 0x61, 0x73, 0x12, 0x20, 0x2e, 0x64, 0x6b, 0x76, 0x2e, 0x73, - 0x65, 0x72, 0x76, 0x65, 0x72, 0x70, 0x62, 0x2e, 0x47, 0x65, 0x74, 0x52, 0x65, 0x70, 0x6c, 0x69, - 0x63, 0x61, 0x73, 0x52, 0x65, 0x71, 0x75, 0x65, 0x73, 0x74, 0x1a, 0x21, 0x2e, 0x64, 0x6b, 0x76, + 0x65, 0x72, 0x70, 0x62, 0x2e, 0x53, 0x74, 0x61, 0x74, 0x75, 0x73, 0x12, 0x52, 0x0a, 0x0b, 0x47, + 0x65, 0x74, 0x52, 0x65, 0x70, 0x6c, 0x69, 0x63, 0x61, 0x73, 0x12, 0x20, 0x2e, 0x64, 0x6b, 0x76, 0x2e, 0x73, 0x65, 0x72, 0x76, 0x65, 0x72, 0x70, 0x62, 0x2e, 0x47, 0x65, 0x74, 0x52, 0x65, 0x70, - 0x6c, 0x69, 0x63, 0x61, 0x73, 0x52, 0x65, 0x73, 0x70, 0x6f, 0x6e, 0x73, 0x65, 0x32, 0x8e, 0x01, - 0x0a, 0x10, 0x44, 0x4b, 0x56, 0x42, 0x61, 0x63, 0x6b, 0x75, 0x70, 0x52, 0x65, 0x73, 0x74, 0x6f, - 0x72, 0x65, 0x12, 0x3b, 0x0a, 0x06, 0x42, 0x61, 0x63, 0x6b, 0x75, 0x70, 0x12, 0x1b, 0x2e, 0x64, - 0x6b, 0x76, 0x2e, 0x73, 0x65, 0x72, 0x76, 0x65, 0x72, 0x70, 0x62, 0x2e, 0x42, 0x61, 0x63, 0x6b, - 0x75, 0x70, 0x52, 0x65, 0x71, 0x75, 0x65, 0x73, 0x74, 0x1a, 0x14, 0x2e, 0x64, 0x6b, 0x76, 0x2e, - 0x73, 0x65, 0x72, 0x76, 0x65, 0x72, 0x70, 0x62, 0x2e, 0x53, 0x74, 0x61, 0x74, 0x75, 0x73, 0x12, - 0x3d, 0x0a, 0x07, 0x52, 0x65, 0x73, 0x74, 0x6f, 0x72, 0x65, 0x12, 0x1c, 0x2e, 0x64, 0x6b, 0x76, - 0x2e, 0x73, 0x65, 0x72, 0x76, 0x65, 0x72, 0x70, 0x62, 0x2e, 0x52, 0x65, 0x73, 0x74, 0x6f, 0x72, - 0x65, 0x52, 0x65, 0x71, 0x75, 0x65, 0x73, 0x74, 0x1a, 0x14, 0x2e, 0x64, 0x6b, 0x76, 0x2e, 0x73, - 0x65, 0x72, 0x76, 0x65, 0x72, 0x70, 0x62, 0x2e, 0x53, 0x74, 0x61, 0x74, 0x75, 0x73, 0x32, 0xd6, - 0x01, 0x0a, 0x0a, 0x44, 0x4b, 0x56, 0x43, 0x6c, 0x75, 0x73, 0x74, 0x65, 0x72, 0x12, 0x3d, 0x0a, - 0x07, 0x41, 0x64, 0x64, 0x4e, 0x6f, 0x64, 0x65, 0x12, 0x1c, 0x2e, 0x64, 0x6b, 0x76, 0x2e, 0x73, - 0x65, 0x72, 0x76, 0x65, 0x72, 0x70, 0x62, 0x2e, 0x41, 0x64, 0x64, 0x4e, 0x6f, 0x64, 0x65, 0x52, - 0x65, 0x71, 0x75, 0x65, 0x73, 0x74, 0x1a, 0x14, 0x2e, 0x64, 0x6b, 0x76, 0x2e, 0x73, 0x65, 0x72, - 0x76, 0x65, 0x72, 0x70, 0x62, 0x2e, 0x53, 0x74, 0x61, 0x74, 0x75, 0x73, 0x12, 0x43, 0x0a, 0x0a, - 0x52, 0x65, 0x6d, 0x6f, 0x76, 0x65, 0x4e, 0x6f, 0x64, 0x65, 0x12, 0x1f, 0x2e, 0x64, 0x6b, 0x76, - 0x2e, 0x73, 0x65, 0x72, 0x76, 0x65, 0x72, 0x70, 0x62, 0x2e, 0x52, 0x65, 0x6d, 0x6f, 0x76, 0x65, - 0x4e, 0x6f, 0x64, 0x65, 0x52, 0x65, 0x71, 0x75, 0x65, 0x73, 0x74, 0x1a, 0x14, 0x2e, 0x64, 0x6b, - 0x76, 0x2e, 0x73, 0x65, 0x72, 0x76, 0x65, 0x72, 0x70, 0x62, 0x2e, 0x53, 0x74, 0x61, 0x74, 0x75, - 0x73, 0x12, 0x44, 0x0a, 0x09, 0x4c, 0x69, 0x73, 0x74, 0x4e, 0x6f, 0x64, 0x65, 0x73, 0x12, 0x16, - 0x2e, 0x67, 0x6f, 0x6f, 0x67, 0x6c, 0x65, 0x2e, 0x70, 0x72, 0x6f, 0x74, 0x6f, 0x62, 0x75, 0x66, - 0x2e, 0x45, 0x6d, 0x70, 0x74, 0x79, 0x1a, 0x1f, 0x2e, 0x64, 0x6b, 0x76, 0x2e, 0x73, 0x65, 0x72, - 0x76, 0x65, 0x72, 0x70, 0x62, 0x2e, 0x4c, 0x69, 0x73, 0x74, 0x4e, 0x6f, 0x64, 0x65, 0x73, 0x52, - 0x65, 0x73, 0x70, 0x6f, 0x6e, 0x73, 0x65, 0x32, 0xb4, 0x01, 0x0a, 0x0c, 0x44, 0x4b, 0x56, 0x44, - 0x69, 0x73, 0x63, 0x6f, 0x76, 0x65, 0x72, 0x79, 0x12, 0x47, 0x0a, 0x0c, 0x55, 0x70, 0x64, 0x61, - 0x74, 0x65, 0x53, 0x74, 0x61, 0x74, 0x75, 0x73, 0x12, 0x21, 0x2e, 0x64, 0x6b, 0x76, 0x2e, 0x73, - 0x65, 0x72, 0x76, 0x65, 0x72, 0x70, 0x62, 0x2e, 0x55, 0x70, 0x64, 0x61, 0x74, 0x65, 0x53, 0x74, - 0x61, 0x74, 0x75, 0x73, 0x52, 0x65, 0x71, 0x75, 0x65, 0x73, 0x74, 0x1a, 0x14, 0x2e, 0x64, 0x6b, + 0x6c, 0x69, 0x63, 0x61, 0x73, 0x52, 0x65, 0x71, 0x75, 0x65, 0x73, 0x74, 0x1a, 0x21, 0x2e, 0x64, + 0x6b, 0x76, 0x2e, 0x73, 0x65, 0x72, 0x76, 0x65, 0x72, 0x70, 0x62, 0x2e, 0x47, 0x65, 0x74, 0x52, + 0x65, 0x70, 0x6c, 0x69, 0x63, 0x61, 0x73, 0x52, 0x65, 0x73, 0x70, 0x6f, 0x6e, 0x73, 0x65, 0x32, + 0x8e, 0x01, 0x0a, 0x10, 0x44, 0x4b, 0x56, 0x42, 0x61, 0x63, 0x6b, 0x75, 0x70, 0x52, 0x65, 0x73, + 0x74, 0x6f, 0x72, 0x65, 0x12, 0x3b, 0x0a, 0x06, 0x42, 0x61, 0x63, 0x6b, 0x75, 0x70, 0x12, 0x1b, + 0x2e, 0x64, 0x6b, 0x76, 0x2e, 0x73, 0x65, 0x72, 0x76, 0x65, 0x72, 0x70, 0x62, 0x2e, 0x42, 0x61, + 0x63, 0x6b, 0x75, 0x70, 0x52, 0x65, 0x71, 0x75, 0x65, 0x73, 0x74, 0x1a, 0x14, 0x2e, 0x64, 0x6b, 0x76, 0x2e, 0x73, 0x65, 0x72, 0x76, 0x65, 0x72, 0x70, 0x62, 0x2e, 0x53, 0x74, 0x61, 0x74, 0x75, - 0x73, 0x12, 0x5b, 0x0a, 0x0e, 0x47, 0x65, 0x74, 0x43, 0x6c, 0x75, 0x73, 0x74, 0x65, 0x72, 0x49, - 0x6e, 0x66, 0x6f, 0x12, 0x23, 0x2e, 0x64, 0x6b, 0x76, 0x2e, 0x73, 0x65, 0x72, 0x76, 0x65, 0x72, - 0x70, 0x62, 0x2e, 0x47, 0x65, 0x74, 0x43, 0x6c, 0x75, 0x73, 0x74, 0x65, 0x72, 0x49, 0x6e, 0x66, - 0x6f, 0x52, 0x65, 0x71, 0x75, 0x65, 0x73, 0x74, 0x1a, 0x24, 0x2e, 0x64, 0x6b, 0x76, 0x2e, 0x73, - 0x65, 0x72, 0x76, 0x65, 0x72, 0x70, 0x62, 0x2e, 0x47, 0x65, 0x74, 0x43, 0x6c, 0x75, 0x73, 0x74, - 0x65, 0x72, 0x49, 0x6e, 0x66, 0x6f, 0x52, 0x65, 0x73, 0x70, 0x6f, 0x6e, 0x73, 0x65, 0x32, 0x51, - 0x0a, 0x10, 0x44, 0x4b, 0x56, 0x44, 0x69, 0x73, 0x63, 0x6f, 0x76, 0x65, 0x72, 0x79, 0x4e, 0x6f, - 0x64, 0x65, 0x12, 0x3d, 0x0a, 0x09, 0x47, 0x65, 0x74, 0x53, 0x74, 0x61, 0x74, 0x75, 0x73, 0x12, - 0x16, 0x2e, 0x67, 0x6f, 0x6f, 0x67, 0x6c, 0x65, 0x2e, 0x70, 0x72, 0x6f, 0x74, 0x6f, 0x62, 0x75, - 0x66, 0x2e, 0x45, 0x6d, 0x70, 0x74, 0x79, 0x1a, 0x18, 0x2e, 0x64, 0x6b, 0x76, 0x2e, 0x73, 0x65, - 0x72, 0x76, 0x65, 0x72, 0x70, 0x62, 0x2e, 0x52, 0x65, 0x67, 0x69, 0x6f, 0x6e, 0x49, 0x6e, 0x66, - 0x6f, 0x42, 0x30, 0x5a, 0x2e, 0x67, 0x69, 0x74, 0x68, 0x75, 0x62, 0x2e, 0x63, 0x6f, 0x6d, 0x2f, - 0x66, 0x6c, 0x69, 0x70, 0x6b, 0x61, 0x72, 0x74, 0x2d, 0x69, 0x6e, 0x63, 0x75, 0x62, 0x61, 0x74, - 0x6f, 0x72, 0x2f, 0x64, 0x6b, 0x76, 0x2f, 0x70, 0x6b, 0x67, 0x2f, 0x73, 0x65, 0x72, 0x76, 0x65, - 0x72, 0x70, 0x62, 0x62, 0x06, 0x70, 0x72, 0x6f, 0x74, 0x6f, 0x33, + 0x73, 0x12, 0x3d, 0x0a, 0x07, 0x52, 0x65, 0x73, 0x74, 0x6f, 0x72, 0x65, 0x12, 0x1c, 0x2e, 0x64, + 0x6b, 0x76, 0x2e, 0x73, 0x65, 0x72, 0x76, 0x65, 0x72, 0x70, 0x62, 0x2e, 0x52, 0x65, 0x73, 0x74, + 0x6f, 0x72, 0x65, 0x52, 0x65, 0x71, 0x75, 0x65, 0x73, 0x74, 0x1a, 0x14, 0x2e, 0x64, 0x6b, 0x76, + 0x2e, 0x73, 0x65, 0x72, 0x76, 0x65, 0x72, 0x70, 0x62, 0x2e, 0x53, 0x74, 0x61, 0x74, 0x75, 0x73, + 0x32, 0xd6, 0x01, 0x0a, 0x0a, 0x44, 0x4b, 0x56, 0x43, 0x6c, 0x75, 0x73, 0x74, 0x65, 0x72, 0x12, + 0x3d, 0x0a, 0x07, 0x41, 0x64, 0x64, 0x4e, 0x6f, 0x64, 0x65, 0x12, 0x1c, 0x2e, 0x64, 0x6b, 0x76, + 0x2e, 0x73, 0x65, 0x72, 0x76, 0x65, 0x72, 0x70, 0x62, 0x2e, 0x41, 0x64, 0x64, 0x4e, 0x6f, 0x64, + 0x65, 0x52, 0x65, 0x71, 0x75, 0x65, 0x73, 0x74, 0x1a, 0x14, 0x2e, 0x64, 0x6b, 0x76, 0x2e, 0x73, + 0x65, 0x72, 0x76, 0x65, 0x72, 0x70, 0x62, 0x2e, 0x53, 0x74, 0x61, 0x74, 0x75, 0x73, 0x12, 0x43, + 0x0a, 0x0a, 0x52, 0x65, 0x6d, 0x6f, 0x76, 0x65, 0x4e, 0x6f, 0x64, 0x65, 0x12, 0x1f, 0x2e, 0x64, + 0x6b, 0x76, 0x2e, 0x73, 0x65, 0x72, 0x76, 0x65, 0x72, 0x70, 0x62, 0x2e, 0x52, 0x65, 0x6d, 0x6f, + 0x76, 0x65, 0x4e, 0x6f, 0x64, 0x65, 0x52, 0x65, 0x71, 0x75, 0x65, 0x73, 0x74, 0x1a, 0x14, 0x2e, + 0x64, 0x6b, 0x76, 0x2e, 0x73, 0x65, 0x72, 0x76, 0x65, 0x72, 0x70, 0x62, 0x2e, 0x53, 0x74, 0x61, + 0x74, 0x75, 0x73, 0x12, 0x44, 0x0a, 0x09, 0x4c, 0x69, 0x73, 0x74, 0x4e, 0x6f, 0x64, 0x65, 0x73, + 0x12, 0x16, 0x2e, 0x67, 0x6f, 0x6f, 0x67, 0x6c, 0x65, 0x2e, 0x70, 0x72, 0x6f, 0x74, 0x6f, 0x62, + 0x75, 0x66, 0x2e, 0x45, 0x6d, 0x70, 0x74, 0x79, 0x1a, 0x1f, 0x2e, 0x64, 0x6b, 0x76, 0x2e, 0x73, + 0x65, 0x72, 0x76, 0x65, 0x72, 0x70, 0x62, 0x2e, 0x4c, 0x69, 0x73, 0x74, 0x4e, 0x6f, 0x64, 0x65, + 0x73, 0x52, 0x65, 0x73, 0x70, 0x6f, 0x6e, 0x73, 0x65, 0x32, 0xb4, 0x01, 0x0a, 0x0c, 0x44, 0x4b, + 0x56, 0x44, 0x69, 0x73, 0x63, 0x6f, 0x76, 0x65, 0x72, 0x79, 0x12, 0x47, 0x0a, 0x0c, 0x55, 0x70, + 0x64, 0x61, 0x74, 0x65, 0x53, 0x74, 0x61, 0x74, 0x75, 0x73, 0x12, 0x21, 0x2e, 0x64, 0x6b, 0x76, + 0x2e, 0x73, 0x65, 0x72, 0x76, 0x65, 0x72, 0x70, 0x62, 0x2e, 0x55, 0x70, 0x64, 0x61, 0x74, 0x65, + 0x53, 0x74, 0x61, 0x74, 0x75, 0x73, 0x52, 0x65, 0x71, 0x75, 0x65, 0x73, 0x74, 0x1a, 0x14, 0x2e, + 0x64, 0x6b, 0x76, 0x2e, 0x73, 0x65, 0x72, 0x76, 0x65, 0x72, 0x70, 0x62, 0x2e, 0x53, 0x74, 0x61, + 0x74, 0x75, 0x73, 0x12, 0x5b, 0x0a, 0x0e, 0x47, 0x65, 0x74, 0x43, 0x6c, 0x75, 0x73, 0x74, 0x65, + 0x72, 0x49, 0x6e, 0x66, 0x6f, 0x12, 0x23, 0x2e, 0x64, 0x6b, 0x76, 0x2e, 0x73, 0x65, 0x72, 0x76, + 0x65, 0x72, 0x70, 0x62, 0x2e, 0x47, 0x65, 0x74, 0x43, 0x6c, 0x75, 0x73, 0x74, 0x65, 0x72, 0x49, + 0x6e, 0x66, 0x6f, 0x52, 0x65, 0x71, 0x75, 0x65, 0x73, 0x74, 0x1a, 0x24, 0x2e, 0x64, 0x6b, 0x76, + 0x2e, 0x73, 0x65, 0x72, 0x76, 0x65, 0x72, 0x70, 0x62, 0x2e, 0x47, 0x65, 0x74, 0x43, 0x6c, 0x75, + 0x73, 0x74, 0x65, 0x72, 0x49, 0x6e, 0x66, 0x6f, 0x52, 0x65, 0x73, 0x70, 0x6f, 0x6e, 0x73, 0x65, + 0x32, 0x51, 0x0a, 0x10, 0x44, 0x4b, 0x56, 0x44, 0x69, 0x73, 0x63, 0x6f, 0x76, 0x65, 0x72, 0x79, + 0x4e, 0x6f, 0x64, 0x65, 0x12, 0x3d, 0x0a, 0x09, 0x47, 0x65, 0x74, 0x53, 0x74, 0x61, 0x74, 0x75, + 0x73, 0x12, 0x16, 0x2e, 0x67, 0x6f, 0x6f, 0x67, 0x6c, 0x65, 0x2e, 0x70, 0x72, 0x6f, 0x74, 0x6f, + 0x62, 0x75, 0x66, 0x2e, 0x45, 0x6d, 0x70, 0x74, 0x79, 0x1a, 0x18, 0x2e, 0x64, 0x6b, 0x76, 0x2e, + 0x73, 0x65, 0x72, 0x76, 0x65, 0x72, 0x70, 0x62, 0x2e, 0x52, 0x65, 0x67, 0x69, 0x6f, 0x6e, 0x49, + 0x6e, 0x66, 0x6f, 0x42, 0x30, 0x5a, 0x2e, 0x67, 0x69, 0x74, 0x68, 0x75, 0x62, 0x2e, 0x63, 0x6f, + 0x6d, 0x2f, 0x66, 0x6c, 0x69, 0x70, 0x6b, 0x61, 0x72, 0x74, 0x2d, 0x69, 0x6e, 0x63, 0x75, 0x62, + 0x61, 0x74, 0x6f, 0x72, 0x2f, 0x64, 0x6b, 0x76, 0x2f, 0x70, 0x6b, 0x67, 0x2f, 0x73, 0x65, 0x72, + 0x76, 0x65, 0x72, 0x70, 0x62, 0x62, 0x06, 0x70, 0x72, 0x6f, 0x74, 0x6f, 0x33, } var ( diff --git a/pkg/serverpb/admin.proto b/pkg/serverpb/admin.proto index 03280aec..db13a6b9 100644 --- a/pkg/serverpb/admin.proto +++ b/pkg/serverpb/admin.proto @@ -191,6 +191,9 @@ message RegionInfo { // Nexus cluster url of the region // Will be used by new followers to discover the raft cluster optional string nexusClusterUrl = 7; + // http listener of the node. + // http endpoint is useful for admin API interactions + string httpAddress = 8; } enum RegionStatus {