diff --git a/internal/impl/babysitter.go b/internal/impl/babysitter.go index a7755a0..7e3139b 100644 --- a/internal/impl/babysitter.go +++ b/internal/impl/babysitter.go @@ -25,7 +25,6 @@ import ( "github.com/ServiceWeaver/weaver-kube/internal/proto" "github.com/ServiceWeaver/weaver/runtime" - "github.com/ServiceWeaver/weaver/runtime/colors" "github.com/ServiceWeaver/weaver/runtime/envelope" "github.com/ServiceWeaver/weaver/runtime/logging" "github.com/ServiceWeaver/weaver/runtime/metrics" @@ -126,7 +125,7 @@ func RunBabysitter(ctx context.Context) error { envelope: e, exportTraces: exportTraces, clientset: clientset, - printer: logging.NewPrettyPrinter(colors.Enabled()), + printer: logging.NewPrettyPrinter(false /*colors disabled*/), watching: map[string]struct{}{}, } diff --git a/internal/impl/dashboard.txt b/internal/impl/dashboard.txt new file mode 100644 index 0000000..c0d56fd --- /dev/null +++ b/internal/impl/dashboard.txt @@ -0,0 +1,501 @@ +{ + "annotations": { + "list": [ + { + "builtIn": 1, + "datasource": { + "type": "grafana", + "uid": "-- Grafana --" + }, + "enable": true, + "hide": true, + "iconColor": "rgba(0, 211, 255, 1)", + "name": "Annotations & Alerts", + "type": "dashboard" + } + ] + }, + "description": "Basic Service Weaver dashboard", + "editable": true, + "fiscalYearStartMonth": 0, + "graphTooltip": 0, + "id": 1, + "links": [], + "liveNow": false, + "panels": [ + { + "collapsed": false, + "gridPos": { + "h": 1, + "w": 24, + "x": 0, + "y": 0 + }, + "id": 7, + "panels": [], + "title": "Metrics", + "type": "row" + }, + { + "datasource": { + "type": "loki", + "uid": "P8E80F9AEF21F6940" + }, + "gridPos": { + "h": 40, + "w": 12, + "x": 0, + "y": 1 + }, + "id": 2, + "options": { + "dedupStrategy": "none", + "enableLogDetails": true, + "prettifyLogMessage": false, + "showCommonLabels": false, + "showLabels": false, + "showTime": false, + "sortOrder": "Descending", + "wrapLogMessage": false + }, + "targets": [ + { + "datasource": { + "type": "loki", + "uid": "P8E80F9AEF21F6940" + }, + "editorMode": "builder", + "expr": "{app=\"%s\"}", + "key": "Q-f5ed4dd4-34d6-449d-b547-a020c1407c9d-0", + "queryType": "range", + "refId": "A" + } + ], + "title": "New Panel", + "type": "logs" + }, + { + "datasource": { + "type": "prometheus", + "uid": "PBFA97CFB590B2093" + }, + "fieldConfig": { + "defaults": { + "color": { + "mode": "palette-classic" + }, + "custom": { + "axisCenteredZero": false, + "axisColorMode": "text", + "axisLabel": "", + "axisPlacement": "auto", + "barAlignment": 0, + "drawStyle": "line", + "fillOpacity": 0, + "gradientMode": "none", + "hideFrom": { + "legend": false, + "tooltip": false, + "viz": false + }, + "lineInterpolation": "linear", + "lineWidth": 1, + "pointSize": 5, + "scaleDistribution": { + "type": "linear" + }, + "showPoints": "auto", + "spanNulls": false, + "stacking": { + "group": "A", + "mode": "none" + }, + "thresholdsStyle": { + "mode": "off" + } + }, + "mappings": [], + "thresholds": { + "mode": "absolute", + "steps": [ + { + "color": "green", + "value": null + }, + { + "color": "red", + "value": 80 + } + ] + } + }, + "overrides": [] + }, + "gridPos": { + "h": 8, + "w": 12, + "x": 12, + "y": 1 + }, + "id": 3, + "options": { + "legend": { + "calcs": [], + "displayMode": "list", + "placement": "bottom", + "showLegend": true + }, + "tooltip": { + "mode": "single", + "sort": "none" + } + }, + "targets": [ + { + "datasource": { + "type": "prometheus", + "uid": "PBFA97CFB590B2093" + }, + "editorMode": "builder", + "expr": "serviceweaver_method_count", + "instant": false, + "range": true, + "refId": "A" + } + ], + "title": "# Method Invocations", + "type": "timeseries" + }, + { + "datasource": { + "type": "prometheus", + "uid": "PBFA97CFB590B2093" + }, + "fieldConfig": { + "defaults": { + "color": { + "mode": "palette-classic" + }, + "custom": { + "axisCenteredZero": false, + "axisColorMode": "text", + "axisLabel": "", + "axisPlacement": "auto", + "barAlignment": 0, + "drawStyle": "line", + "fillOpacity": 0, + "gradientMode": "none", + "hideFrom": { + "legend": false, + "tooltip": false, + "viz": false + }, + "lineInterpolation": "linear", + "lineWidth": 1, + "pointSize": 5, + "scaleDistribution": { + "type": "linear" + }, + "showPoints": "auto", + "spanNulls": false, + "stacking": { + "group": "A", + "mode": "none" + }, + "thresholdsStyle": { + "mode": "off" + } + }, + "mappings": [], + "thresholds": { + "mode": "absolute", + "steps": [ + { + "color": "green", + "value": null + }, + { + "color": "red", + "value": 80 + } + ] + } + }, + "overrides": [] + }, + "gridPos": { + "h": 8, + "w": 12, + "x": 12, + "y": 9 + }, + "id": 9, + "options": { + "legend": { + "calcs": [], + "displayMode": "list", + "placement": "bottom", + "showLegend": true + }, + "tooltip": { + "mode": "single", + "sort": "none" + } + }, + "targets": [ + { + "datasource": { + "type": "prometheus", + "uid": "PBFA97CFB590B2093" + }, + "editorMode": "builder", + "expr": "serviceweaver_method_error_count", + "instant": false, + "range": true, + "refId": "A" + } + ], + "title": "# Method Invocations with errors", + "type": "timeseries" + }, + { + "datasource": { + "type": "prometheus", + "uid": "PBFA97CFB590B2093" + }, + "fieldConfig": { + "defaults": { + "color": { + "mode": "thresholds" + }, + "custom": { + "fillOpacity": 80, + "gradientMode": "none", + "hideFrom": { + "legend": false, + "tooltip": false, + "viz": false + }, + "lineWidth": 1 + }, + "mappings": [], + "thresholds": { + "mode": "absolute", + "steps": [ + { + "color": "green", + "value": null + }, + { + "color": "red", + "value": 80 + } + ] + } + }, + "overrides": [] + }, + "gridPos": { + "h": 8, + "w": 12, + "x": 12, + "y": 17 + }, + "id": 10, + "options": { + "bucketOffset": 0, + "legend": { + "calcs": [ + "lastNotNull", + "min", + "max", + "variance" + ], + "displayMode": "list", + "placement": "bottom", + "showLegend": true + } + }, + "targets": [ + { + "datasource": { + "type": "prometheus", + "uid": "PBFA97CFB590B2093" + }, + "editorMode": "builder", + "expr": "serviceweaver_method_latency_micros_bucket", + "instant": false, + "range": true, + "refId": "A" + } + ], + "title": "Distribution of latency (us) of Service Weaver component execution", + "type": "histogram" + }, + { + "datasource": { + "type": "prometheus", + "uid": "PBFA97CFB590B2093" + }, + "fieldConfig": { + "defaults": { + "color": { + "mode": "thresholds" + }, + "custom": { + "fillOpacity": 80, + "gradientMode": "none", + "hideFrom": { + "legend": false, + "tooltip": false, + "viz": false + }, + "lineWidth": 1 + }, + "mappings": [], + "thresholds": { + "mode": "absolute", + "steps": [ + { + "color": "green", + "value": null + }, + { + "color": "red", + "value": 80 + } + ] + } + }, + "overrides": [] + }, + "gridPos": { + "h": 8, + "w": 12, + "x": 12, + "y": 25 + }, + "id": 4, + "options": { + "bucketOffset": 0, + "legend": { + "calcs": [ + "lastNotNull", + "min", + "max", + "variance" + ], + "displayMode": "list", + "placement": "bottom", + "showLegend": true + } + }, + "targets": [ + { + "datasource": { + "type": "prometheus", + "uid": "PBFA97CFB590B2093" + }, + "editorMode": "builder", + "expr": "serviceweaver_method_bytes_request_bucket", + "instant": false, + "range": true, + "refId": "A" + } + ], + "title": "Distribution of #bytes in Service Weaver component method requests", + "type": "histogram" + }, + { + "datasource": { + "type": "prometheus", + "uid": "PBFA97CFB590B2093" + }, + "fieldConfig": { + "defaults": { + "color": { + "mode": "thresholds" + }, + "custom": { + "fillOpacity": 80, + "gradientMode": "none", + "hideFrom": { + "legend": false, + "tooltip": false, + "viz": false + }, + "lineWidth": 1 + }, + "mappings": [], + "thresholds": { + "mode": "absolute", + "steps": [ + { + "color": "green", + "value": null + }, + { + "color": "red", + "value": 80 + } + ] + } + }, + "overrides": [] + }, + "gridPos": { + "h": 8, + "w": 12, + "x": 12, + "y": 33 + }, + "id": 8, + "options": { + "bucketOffset": 0, + "legend": { + "calcs": [ + "lastNotNull", + "min", + "max", + "variance" + ], + "displayMode": "list", + "placement": "bottom", + "showLegend": true + } + }, + "targets": [ + { + "datasource": { + "type": "prometheus", + "uid": "PBFA97CFB590B2093" + }, + "editorMode": "builder", + "expr": "serviceweaver_method_bytes_reply_bucket", + "instant": false, + "range": true, + "refId": "A" + } + ], + "title": "Distribution of #bytes in Service Weaver component method replies", + "type": "histogram" + } + ], + "refresh": "5s", + "schemaVersion": 38, + "style": "dark", + "tags": [], + "templating": { + "list": [] + }, + "time": { + "from": "now-6h", + "to": "now" + }, + "timepicker": {}, + "timezone": "", + "title": "Service Weaver", + "uid": "c7fcf5e5-00e9-4af5-a246-0eaf3068ac29", + "version": 16, + "weekStart": "" +} diff --git a/internal/impl/kube.go b/internal/impl/kube.go index bede4ee..a1831fe 100644 --- a/internal/impl/kube.go +++ b/internal/impl/kube.go @@ -16,9 +16,11 @@ package impl import ( "bytes" + _ "embed" "fmt" "os" "path/filepath" + "time" "github.com/ServiceWeaver/weaver-kube/internal/proto" "github.com/ServiceWeaver/weaver/runtime/bin" @@ -64,6 +66,16 @@ const ( // [1] https://prometheus.io/ prometheusImageName = "prom/prometheus:v2.30.3" + // The name of the Loki [1] image used to handle the logs. + // + // [1] https://grafana.com/oss/loki/ + lokiImageName = "grafana/loki" + + // The name of the Promtail [1] image used to scrape the logs. + // + // [1] https://grafana.com/docs/loki/latest/clients/promtail/ + promtailImageName = "grafana/promtail" + // The name of the Grafana [1] image used to display metrics, traces, and logs. // // [1] https://grafana.com/ @@ -82,14 +94,17 @@ const ( // The port on which the weavelets are exporting the metrics. metricsPort = 9090 + // The port on which Loki is exporting the logs. + lokiPort = 3100 + // The default Grafana web server port. grafanaPort = 3000 -) -// Port used by the weavelets to listen for internal traffic. -// -// TODO(mwhittaker): Remove internal port from kube.proto. -const internalPort = 10000 + // Port used by the weavelets to listen for internal traffic. + // + // TODO(mwhittaker): Remove internal port from kube.proto. + internalPort = 10000 +) var ( // Start value for ports used by the public and private listeners. @@ -103,6 +118,12 @@ var ( memoryUnit = resource.MustParse("128Mi") ) +// dashboard was generated using the Grafana UI. Then, we saved the content as +// a JSON file. +// +//go:embed dashboard.txt +var dashboardContent string + // replicaSetInfo contains information associated with a replica set. type replicaSetInfo struct { name string // name of the replica set @@ -432,6 +453,20 @@ func GenerateKubeDeployment(image string, dep *protos.Deployment, cfg *KubeConfi } generated = append(generated, content...) + // Generate the Loki deployment info. + content, err = generateLokiDeployment(dep) + if err != nil { + return fmt.Errorf("unable to create kube deployment for the Loki service: %w", err) + } + generated = append(generated, content...) + + // Generate the Promtail deployment info. + content, err = generatePromtailDeployment(dep) + if err != nil { + return fmt.Errorf("unable to create kube deployment for Promtail: %w", err) + } + generated = append(generated, content...) + // Generate the Grafana deployment info. content, err = generateGrafanaDeployment(dep) if err != nil { @@ -695,6 +730,9 @@ scrape_configs: metrics_path: %s kubernetes_sd_configs: - role: pod + namespaces: + names: + - default scheme: http relabel_configs: - source_labels: [__meta_kubernetes_pod_label_metrics] @@ -746,14 +784,14 @@ scrape_configs: Image: prometheusImageName, ImagePullPolicy: corev1.PullIfNotPresent, Args: []string{ - fmt.Sprintf("--config.file=/etc/%s/prometheus.yml", pname), + fmt.Sprintf("--config.file=/etc/%s/prometheus.yaml", pname), fmt.Sprintf("--storage.tsdb.path=/%s", pname), }, Ports: []corev1.ContainerPort{{ContainerPort: metricsPort}}, VolumeMounts: []corev1.VolumeMount{ { Name: cname, - MountPath: fmt.Sprintf("/etc/%s/prometheus.yml", pname), + MountPath: fmt.Sprintf("/etc/%s/prometheus.yaml", pname), SubPath: "prometheus.yaml", }, { @@ -830,6 +868,379 @@ scrape_configs: return generated, nil } +// generateLokiDeployment generates the kubernetes configurations to deploy +// a Loki service for a given app. +// +// TODO(rgrandl): check if we can simplify the config map, and the deployment info. +// +// TODO(rgrandl): We run a single instance of Loki for now. We might want +// to scale it up if it becomes a bottleneck. +func generateLokiDeployment(dep *protos.Deployment) ([]byte, error) { + cname := name{dep.App.Name, "loki", "config"}.DNSLabel() + lname := name{dep.App.Name, "loki"}.DNSLabel() + + timeSchemaEnabledFromFn := func() string { + current := time.Now() + year, month, day := current.Date() + return fmt.Sprintf("%d-%02d-%02d", year, month, day) + } + + // Build the config map that holds the Loki configuration file. In the + // config we specify how to store the logs and the schema. Right now we have + // a very simple in-memory store [1]. + // + // TODO(rgrandl): There are millions of knobs to tune the config. We might revisit + // this in the future. + // + // [1] https://grafana.com/docs/loki/latest/operations/storage/boltdb-shipper/ + config := fmt.Sprintf(` +auth_enabled: false +server: + http_listen_port: %d + +common: + instance_addr: 127.0.0.1 + path_prefix: /tmp/%s + storage: + filesystem: + chunks_directory: /tmp/%s/chunks + rules_directory: /tmp/%s/rules + replication_factor: 1 + ring: + kvstore: + store: inmemory + +schema_config: + configs: + - from: %s # Marks the starting point of this schema + store: boltdb-shipper + object_store: filesystem + schema: v11 + index: + prefix: index_ + period: 24h +`, lokiPort, lname, lname, lname, timeSchemaEnabledFromFn()) + + // Create a config map to store the Loki config. + cm := corev1.ConfigMap{ + TypeMeta: metav1.TypeMeta{ + Kind: "ConfigMap", + APIVersion: "v1", + }, + ObjectMeta: metav1.ObjectMeta{Name: cname}, + Data: map[string]string{ + "loki.yaml": config, + }, + } + content, err := yaml.Marshal(cm) + if err != nil { + return nil, err + } + var generated []byte + generated = append(generated, []byte(fmt.Sprintf("\n# Config Map %s\n", cname))...) + generated = append(generated, content...) + generated = append(generated, []byte("\n---\n")...) + fmt.Fprintf(os.Stderr, "Generated kube deployment for config map %s\n", cname) + + // Build the kubernetes Loki deployment. + d := &appsv1.Deployment{ + TypeMeta: metav1.TypeMeta{ + APIVersion: "apps/v1", + Kind: "Deployment", + }, + ObjectMeta: metav1.ObjectMeta{Name: lname}, + Spec: appsv1.DeploymentSpec{ + Replicas: ptrOf(int32(1)), + Selector: &metav1.LabelSelector{ + MatchLabels: map[string]string{"loki": lname}, + }, + Template: corev1.PodTemplateSpec{ + ObjectMeta: metav1.ObjectMeta{ + Labels: map[string]string{"loki": lname}, + }, + Spec: corev1.PodSpec{ + Containers: []corev1.Container{ + { + Name: lname, + Image: fmt.Sprintf("%s:latest", lokiImageName), + ImagePullPolicy: corev1.PullIfNotPresent, + Args: []string{ + fmt.Sprintf("--config.file=/etc/%s/loki.yaml", lname), + }, + Ports: []corev1.ContainerPort{{ContainerPort: lokiPort}}, + VolumeMounts: []corev1.VolumeMount{ + { + Name: cname, + MountPath: fmt.Sprintf("/etc/%s/", lname), + }, + }, + }, + }, + Volumes: []corev1.Volume{ + { + Name: cname, + VolumeSource: corev1.VolumeSource{ + ConfigMap: &corev1.ConfigMapVolumeSource{ + LocalObjectReference: corev1.LocalObjectReference{ + Name: cname, + }, + Items: []corev1.KeyToPath{ + { + Key: "loki.yaml", + Path: "loki.yaml", + }, + }, + }, + }, + }, + }, + }, + }, + Strategy: v1.DeploymentStrategy{ + Type: "RollingUpdate", + RollingUpdate: &v1.RollingUpdateDeployment{}, + }, + }, + } + content, err = yaml.Marshal(d) + if err != nil { + return nil, err + } + generated = append(generated, []byte(fmt.Sprintf("\n# Loki Deployment %s\n", lname))...) + generated = append(generated, content...) + generated = append(generated, []byte("\n---\n")...) + fmt.Fprintf(os.Stderr, "Generated kube deployment for Loki %s\n", lname) + + // Build the kubernetes Loki service. + s := &corev1.Service{ + TypeMeta: metav1.TypeMeta{ + APIVersion: "v1", + Kind: "Service", + }, + ObjectMeta: metav1.ObjectMeta{Name: lname}, + Spec: corev1.ServiceSpec{ + Selector: map[string]string{"loki": lname}, + Ports: []corev1.ServicePort{ + { + Port: lokiPort, + Protocol: "TCP", + TargetPort: intstr.IntOrString{IntVal: int32(lokiPort)}, + }, + }, + }, + } + content, err = yaml.Marshal(s) + if err != nil { + return nil, err + } + generated = append(generated, []byte(fmt.Sprintf("\n# Loki Service %s\n", lname))...) + generated = append(generated, content...) + generated = append(generated, []byte("\n---\n")...) + fmt.Fprintf(os.Stderr, "Generated kube service for Loki %s\n", lname) + return generated, nil +} + +// generatePromtailDeployment generates information that is needed to run a +// Promtail agent on each node in order to scrape the logs. +func generatePromtailDeployment(dep *protos.Deployment) ([]byte, error) { + promName := name{dep.App.Name, "promtail"}.DNSLabel() + lname := name{dep.App.Name, "loki"}.DNSLabel() + + // This configuration is a simplified version of the Promtail config generated + // by helm [1]. Right now we scrape only logs from the pods. We may want to + // scrape system information and nodes info as well. + // + // The scraped logs are sent to Loki for indexing and being stored. + // + // [1] https://helm.sh/docs/topics/charts/. + config := fmt.Sprintf(` +server: + log_format: logfmt + http_listen_port: 3101 + +clients: + - url: http://%s:%d/loki/api/v1/push + +positions: + filename: /run/promtail/positions.yaml + +scrape_configs: + - job_name: kubernetes-pods + kubernetes_sd_configs: + - role: pod + namespaces: + names: + - default + relabel_configs: + - source_labels: + - __meta_kubernetes_pod_label_appName + regex: ^.*%s.*$ + action: keep + - source_labels: + - __meta_kubernetes_pod_label_appName + action: replace + target_label: app + - source_labels: + - __meta_kubernetes_pod_name + action: replace + target_label: pod + - action: replace + replacement: /var/log/pods/*$1/*.log + separator: / + source_labels: + - __meta_kubernetes_pod_uid + - __meta_kubernetes_pod_container_name + target_label: __path__ +`, lname, lokiPort, dep.App.Name) + + // Config is stored as a config map in the daemonset. + cm := corev1.ConfigMap{ + TypeMeta: metav1.TypeMeta{ + Kind: "ConfigMap", + APIVersion: "v1", + }, + ObjectMeta: metav1.ObjectMeta{Name: promName}, + Data: map[string]string{ + "promtail.yaml": config, + }, + } + + content, err := yaml.Marshal(cm) + if err != nil { + return nil, err + } + var generated []byte + generated = append(generated, []byte(fmt.Sprintf("\n# Config Map %s\n", cm.Name))...) + generated = append(generated, content...) + generated = append(generated, []byte("\n---\n")...) + fmt.Fprintf(os.Stderr, "Generated kube deployment for config map %s\n", cm.Name) + + // Create a daemonset that will run on each node. The daemonset will run promtail + // in order to scrape the pods running on each node. + dset := appsv1.DaemonSet{ + TypeMeta: metav1.TypeMeta{ + Kind: "DaemonSet", + APIVersion: "apps/v1", + }, + ObjectMeta: metav1.ObjectMeta{ + Name: promName, + }, + Spec: appsv1.DaemonSetSpec{ + Selector: &metav1.LabelSelector{ + MatchLabels: map[string]string{ + "promtail": promName, + }, + }, + Template: corev1.PodTemplateSpec{ + ObjectMeta: metav1.ObjectMeta{ + Labels: map[string]string{ + "promtail": promName, + }, + }, + Spec: corev1.PodSpec{ + ServiceAccountName: "default", + Containers: []corev1.Container{ + { + Name: promName, + Image: fmt.Sprintf("%s:latest", promtailImageName), + ImagePullPolicy: corev1.PullIfNotPresent, + Args: []string{ + fmt.Sprintf("--config.file=/etc/%s/promtail.yaml", promName), + }, + Ports: []corev1.ContainerPort{{ContainerPort: 3101}}, + Env: []corev1.EnvVar{ + { + Name: "HOSTNAME", + ValueFrom: &corev1.EnvVarSource{ + FieldRef: &corev1.ObjectFieldSelector{ + FieldPath: "spec.nodeName", + }, + }, + }, + }, + VolumeMounts: []corev1.VolumeMount{ + { + Name: "config", + MountPath: fmt.Sprintf("/etc/%s", promName), + }, + { + Name: "run", + MountPath: "/run/promtail", + }, + { + Name: "containers", + MountPath: "/var/lib/docker/containers", + ReadOnly: true, + }, + { + Name: "pods", + MountPath: "/var/log/pods", + ReadOnly: true, + }, + }, + }, + }, + Volumes: []corev1.Volume{ + { + Name: "config", + VolumeSource: corev1.VolumeSource{ + ConfigMap: &corev1.ConfigMapVolumeSource{ + LocalObjectReference: corev1.LocalObjectReference{ + Name: cm.Name, + }, + Items: []corev1.KeyToPath{ + { + Key: "promtail.yaml", + Path: "promtail.yaml", + }, + }, + }, + }, + }, + { + Name: "run", + VolumeSource: corev1.VolumeSource{ + HostPath: &corev1.HostPathVolumeSource{ + Path: "/run/promtail", + }, + }, + }, + { + Name: "containers", + VolumeSource: corev1.VolumeSource{ + HostPath: &corev1.HostPathVolumeSource{ + Path: "/var/lib/docker/containers", + }, + }, + }, + { + Name: "pods", + VolumeSource: corev1.VolumeSource{ + HostPath: &corev1.HostPathVolumeSource{ + Path: "/var/log/pods", + }, + }, + }, + }, + }, + }, + UpdateStrategy: v1.DaemonSetUpdateStrategy{ + Type: "RollingUpdate", + RollingUpdate: &v1.RollingUpdateDaemonSet{}, + }, + }, + } + content, err = yaml.Marshal(dset) + if err != nil { + return nil, err + } + generated = append(generated, []byte(fmt.Sprintf("\n# Promtail DaemonSet %s\n", promName))...) + generated = append(generated, content...) + generated = append(generated, []byte("\n---\n")...) + fmt.Fprintf(os.Stderr, "Generated kube daemonset for Promtail %s\n", promName) + return generated, nil +} + // generateGrafanaDeployment generates the kubernetes configurations to deploy // a Grafana service for a given app. // @@ -840,6 +1251,7 @@ func generateGrafanaDeployment(dep *protos.Deployment) ([]byte, error) { gname := name{dep.App.Name, "grafana"}.DNSLabel() pname := name{dep.App.Name, "prometheus"}.DNSLabel() jname := name{dep.App.Name, jaegerAppName}.DNSLabel() + lname := name{dep.App.Name, "loki"}.DNSLabel() // Build the config map that holds the Grafana configuration file. In the // config we specify which data source connections the Grafana service should @@ -859,9 +1271,25 @@ datasources: - name: Jaeger type: jaeger url: http://%s:%d -`, pname, jname, jaegerUIPort) + - name: Loki + type: loki + access: proxy + url: http://%s:%d +`, pname, jname, jaegerUIPort, lname, lokiPort) - // Create a config map to store the Grafana config. + // It contains the list of dashboard providers that load dashboards into + // Grafana from the local filesystem [1]. + // + // https://grafana.com/docs/grafana/latest/administration/provisioning/#dashboards + const dashboard = ` +apiVersion: 1 +providers: + - name: 'Service Weaver Dashboard' + options: + path: /etc/grafana/dashboards/default-dashboard.json +` + + // Create a config map to store the Grafana configs and the default dashboards. cm := corev1.ConfigMap{ TypeMeta: metav1.TypeMeta{ Kind: "ConfigMap", @@ -869,7 +1297,9 @@ datasources: }, ObjectMeta: metav1.ObjectMeta{Name: cname}, Data: map[string]string{ - "grafana.yaml": config, + "grafana.yaml": config, + "dashboard-config.yaml": dashboard, + "default-dashboard.json": fmt.Sprintf(dashboardContent, dep.App.Name), }, } content, err := yaml.Marshal(cm) @@ -912,6 +1342,21 @@ datasources: Name: "datasource-volume", MountPath: "/etc/grafana/provisioning/datasources/", }, + { + // By default, we need to store the dashboards config files under + // provisioning/dashboards directory. Each config file can contain + // a list of dashboards providers that load dashboards into Grafana + // from the local filesystem. More info here [1]. + // + // [1] https://grafana.com/docs/grafana/latest/administration/provisioning/#dashboards + Name: "dashboards-config", + MountPath: "/etc/grafana/provisioning/dashboards/", + }, + { + // Mount the volume that stores the predefined dashboards. + Name: "dashboards", + MountPath: "/etc/grafana/dashboards/", + }, }, Env: []corev1.EnvVar{ // TODO(rgrandl): we may want to enable the user to specify their @@ -944,6 +1389,38 @@ datasources: }, }, }, + { + Name: "dashboards-config", + VolumeSource: corev1.VolumeSource{ + ConfigMap: &corev1.ConfigMapVolumeSource{ + LocalObjectReference: corev1.LocalObjectReference{ + Name: cname, + }, + Items: []corev1.KeyToPath{ + { + Key: "dashboard-config.yaml", + Path: "dashboard-config.yaml", + }, + }, + }, + }, + }, + { + Name: "dashboards", + VolumeSource: corev1.VolumeSource{ + ConfigMap: &corev1.ConfigMapVolumeSource{ + LocalObjectReference: corev1.LocalObjectReference{ + Name: cname, + }, + Items: []corev1.KeyToPath{ + { + Key: "default-dashboard.json", + Path: "default-dashboard.json", + }, + }, + }, + }, + }, }, }, },