From 078b61561fb39b07248c847631948ad7c81997b0 Mon Sep 17 00:00:00 2001 From: wildum Date: Fri, 28 Jul 2023 16:45:29 +0200 Subject: [PATCH 1/4] update image versions and replace the traces generator --- example/docker-compose/docker-compose.yaml | 24 +- .../load-generator/load-generator.json | 630 ------------------ 2 files changed, 11 insertions(+), 643 deletions(-) delete mode 100644 example/docker-compose/load-generator/load-generator.json diff --git a/example/docker-compose/docker-compose.yaml b/example/docker-compose/docker-compose.yaml index 81fef8ad8df9..86f073b4439d 100644 --- a/example/docker-compose/docker-compose.yaml +++ b/example/docker-compose/docker-compose.yaml @@ -9,7 +9,7 @@ services: # grafana: - image: grafana/grafana:9.2.3 + image: grafana/grafana:10.0.3 entrypoint: - /usr/share/grafana/bin/grafana-server - --homepath=/usr/share/grafana @@ -23,7 +23,7 @@ services: - "3000:3000" loki: - image: grafana/loki:2.6.1 + image: grafana/loki:2.8.3 command: -config.file=/etc/loki/local-config.yaml ports: - "3100:3100" @@ -39,9 +39,8 @@ services: - "9009:9009" tempo: - image: grafana/tempo:1.5.0 + image: grafana/tempo:2.1.0 command: - - "-search.enabled=true" - "-storage.trace.backend=local" # tell tempo where to permanently put traces - "-storage.trace.local.path=/tmp/tempo/traces" - "-storage.trace.wal.path=/tmp/tempo/wal" # tell tempo where to store the wal @@ -60,15 +59,14 @@ services: ports: - "9001:9001" - # tracing load generator - synthetic-load-generator: - profiles: [agent] # Should only be run if the Agent is present - image: omnition/synthetic-load-generator:1.0.25 - volumes: - - ./load-generator:/etc/load-generator - environment: - - TOPOLOGY_FILE=/etc/load-generator/load-generator.json - - JAEGER_COLLECTOR_URL=http://agent:14268 + hotrod: + profiles: [agent] + image: yurishkuro/microsim:latest + ports: + - "8080:8080" + command: + - "-j=http://agent:14268/api/traces" + - "-d=1h" depends_on: - agent diff --git a/example/docker-compose/load-generator/load-generator.json b/example/docker-compose/load-generator/load-generator.json deleted file mode 100644 index 0205037a05c5..000000000000 --- a/example/docker-compose/load-generator/load-generator.json +++ /dev/null @@ -1,630 +0,0 @@ -{ - "topology": { - "services": [ - { - "serviceName": "frontend", - "tagSets": [ - { - "weight": 1, - "tags": { - "version": "v127", - "region": "us-east-1" - }, - "tagGenerators": [], - "inherit": [], - "maxLatency": 100 - }, - { - "weight": 1, - "tags": { - "version": "v125", - "region": "us-east-1" - }, - "tagGenerators": [], - "inherit": [], - "maxLatency": 100 - }, - { - "weight": 2, - "tags": { - "version": "v125", - "region": "us-west-1" - }, - "tagGenerators": [], - "inherit": [], - "maxLatency": 100 - } - ], - "routes": [ - { - "route": "/product", - "downstreamCalls": { - "productcatalogservice": "/GetProducts", - "recommendationservice": "/GetRecommendations", - "adservice": "/AdRequest" - }, - "tagSets": [ - { - "weight": 1, - "tags": { - "starter": "charmander" - }, - "tagGenerators": [ - { - "rand": { - "seed": 179867746078676, - "nextNextGaussian": 0, - "haveNextNextGaussian": false - }, - "tagGen": {}, - "valLength": 16, - "numTags": 50, - "numVals": 3000 - } - ], - "inherit": [] - }, - { - "weight": 1, - "tags": { - "starter": "squirtle" - }, - "tagGenerators": [], - "inherit": [] - }, - { - "weight": 1, - "tags": { - "starter": "bulbasaur" - }, - "tagGenerators": [], - "inherit": [] - } - ] - }, - { - "route": "/cart", - "downstreamCalls": { - "cartservice": "/GetCart", - "recommendationservice": "/GetRecommendations" - }, - "tagSets": [] - }, - { - "route": "/checkout", - "downstreamCalls": { - "checkoutservice": "/PlaceOrder" - }, - "tagSets": [ - { - "tags": {}, - "tagGenerators": [], - "inherit": [], - "maxLatency": 800 - } - ] - }, - { - "route": "/shipping", - "downstreamCalls": { - "shippingservice": "/GetQuote" - }, - "tagSets": [ - { - "tags": {}, - "tagGenerators": [], - "inherit": [], - "maxLatency": 50 - } - ] - }, - { - "route": "/currency", - "downstreamCalls": { - "currencyservice": "/GetConversion" - }, - "tagSets": [ - { - "tags": {}, - "tagGenerators": [], - "inherit": [], - "maxLatency": 50 - } - ] - } - ], - "instances": [ - "frontend-6b654dbf57-zq8dt", - "frontend-d847fdcf5-j6s2f", - "frontend-79d8c8d6c8-9sbff" - ], - "mergedTagSets": {}, - "random": { - "seed": 187004238864083, - "nextNextGaussian": 0, - "haveNextNextGaussian": false - } - }, - { - "serviceName": "productcatalogservice", - "tagSets": [ - { - "tags": { - "version": "v52" - }, - "tagGenerators": [], - "inherit": [ - "region" - ] - } - ], - "routes": [ - { - "route": "/GetProducts", - "downstreamCalls": {}, - "tagSets": [ - { - "tags": {}, - "tagGenerators": [], - "inherit": [ - "starter" - ], - "maxLatency": 100 - } - ] - }, - { - "route": "/SearchProducts", - "downstreamCalls": {}, - "tagSets": [ - { - "weight": 15, - "tags": { - "error": true, - "http.status_code": 503 - }, - "tagGenerators": [], - "inherit": [], - "maxLatency": 400 - }, - { - "weight": 85, - "tags": {}, - "tagGenerators": [], - "inherit": [], - "maxLatency": 400 - } - ] - } - ], - "instances": [ - "productcatalogservice-6b654dbf57-zq8dt", - "productcatalogservice-d847fdcf5-j6s2f" - ], - "mergedTagSets": {}, - "random": { - "seed": 238238032670139, - "nextNextGaussian": 0, - "haveNextNextGaussian": false - } - }, - { - "serviceName": "recommendationservice", - "tagSets": [ - { - "tags": { - "version": "v234", - "region": "us-east-1" - }, - "tagGenerators": [], - "inherit": [] - } - ], - "routes": [ - { - "route": "/GetRecommendations", - "downstreamCalls": { - "productcatalogservice": "/GetProducts" - }, - "tagSets": [ - { - "tags": {}, - "tagGenerators": [], - "inherit": [], - "maxLatency": 200 - } - ] - } - ], - "instances": [ - "recommendationservice-6b654dbf57-zq8dt", - "recommendationservice-d847fdcf5-j6s2f" - ], - "mergedTagSets": {}, - "random": { - "seed": 66295214032801, - "nextNextGaussian": 0, - "haveNextNextGaussian": false - } - }, - { - "serviceName": "cartservice", - "tagSets": [ - { - "tags": { - "version": "v5", - "region": "us-east-1" - }, - "tagGenerators": [], - "inherit": [] - } - ], - "routes": [ - { - "route": "/GetCart", - "downstreamCalls": {}, - "tagSets": [ - { - "tags": {}, - "tagGenerators": [], - "inherit": [], - "maxLatency": 200 - } - ] - } - ], - "instances": [ - "cartservice-6b654dbf57-zq8dt", - "cartservice-d847fdcf5-j6s2f" - ], - "mergedTagSets": {}, - "random": { - "seed": 234194353561392, - "nextNextGaussian": 0, - "haveNextNextGaussian": false - } - }, - { - "serviceName": "checkoutservice", - "tagSets": [ - { - "tags": { - "version": "v37", - "region": "us-east-1" - }, - "tagGenerators": [], - "inherit": [], - "maxLatency": 500 - } - ], - "routes": [ - { - "route": "/PlaceOrder", - "downstreamCalls": { - "paymentservice": "/CreditCardInfo", - "shippingservice": "/Address", - "currencyservice": "/GetConversion", - "cartservice": "/GetCart", - "emailservice": "/SendOrderConfirmation" - }, - "tagSets": [ - { - "weight": 25, - "tags": { - "error": true, - "http.status_code": 503 - }, - "tagGenerators": [], - "inherit": [] - }, - { - "weight": 85, - "tags": {}, - "tagGenerators": [], - "inherit": [] - } - ] - } - ], - "instances": [ - "checkoutservice-6b654dbf57-zq8dt", - "checkoutservice-d847fdcf5-j6s2f" - ], - "mergedTagSets": {}, - "random": { - "seed": 60782549660568, - "nextNextGaussian": 0, - "haveNextNextGaussian": false - } - }, - { - "serviceName": "paymentservice", - "tagSets": [ - { - "tags": { - "version": "v177", - "region": "us-east-1" - }, - "tagGenerators": [], - "inherit": [] - } - ], - "routes": [ - { - "route": "/ChargeRequest", - "downstreamCalls": { - "paymentservice": "/CreditCardInfo" - }, - "tagSets": [ - { - "tags": {}, - "tagGenerators": [], - "inherit": [], - "maxLatency": 700 - } - ] - }, - { - "route": "/CreditCardInfo", - "downstreamCalls": {}, - "tagSets": [ - { - "tags": {}, - "tagGenerators": [], - "inherit": [], - "maxLatency": 50 - } - ] - } - ], - "instances": [ - "paymentservice-6b654dbf57-zq8dt", - "paymentservice-d847fdcf5-j6s2f" - ], - "mergedTagSets": {}, - "random": { - "seed": 174850031049111, - "nextNextGaussian": 0, - "haveNextNextGaussian": false - } - }, - { - "serviceName": "shippingservice", - "tagSets": [ - { - "tags": { - "version": "v127", - "region": "us-east-1" - }, - "tagGenerators": [], - "inherit": [] - } - ], - "routes": [ - { - "route": "/GetQuote", - "downstreamCalls": { - "shippingservice": "/Address" - }, - "tagSets": [ - { - "tags": {}, - "tagGenerators": [], - "inherit": [], - "maxLatency": 250 - } - ] - }, - { - "route": "/ShipOrder", - "downstreamCalls": { - "shippingservice": "/Address" - }, - "tagSets": [ - { - "tags": {}, - "tagGenerators": [], - "inherit": [], - "maxLatency": 500 - } - ] - }, - { - "route": "/Address", - "downstreamCalls": {}, - "tagSets": [ - { - "tags": {}, - "tagGenerators": [], - "inherit": [], - "maxLatency": 100 - } - ] - } - ], - "instances": [ - "shippingservice-6b654dbf57-zq8dt", - "shippingservice-d847fdcf5-j6s2f" - ], - "mergedTagSets": {}, - "random": { - "seed": 107892261530518, - "nextNextGaussian": 0, - "haveNextNextGaussian": false - } - }, - { - "serviceName": "emailservice", - "tagSets": [ - { - "tags": { - "version": "v27", - "region": "us-east-1" - }, - "tagGenerators": [], - "inherit": [], - "maxLatency": 500 - } - ], - "routes": [ - { - "route": "/SendOrderConfirmation", - "downstreamCalls": { - "emailservice": "/OrderResult" - }, - "tagSets": [ - { - "weight": 15, - "tags": { - "error": true, - "http.status_code": 503 - }, - "tagGenerators": [], - "inherit": [] - }, - { - "weight": 85, - "tags": {}, - "tagGenerators": [], - "inherit": [] - } - ] - }, - { - "route": "/OrderResult", - "downstreamCalls": {}, - "tagSets": [ - { - "tags": {}, - "tagGenerators": [], - "inherit": [], - "maxLatency": 100 - } - ] - } - ], - "instances": [ - "emailservice-6b654dbf57-zq8dt", - "emailservice-d847fdcf5-j6s2f" - ], - "mergedTagSets": {}, - "random": { - "seed": 61175057559946, - "nextNextGaussian": 0, - "haveNextNextGaussian": false - } - }, - { - "serviceName": "currencyservice", - "tagSets": [ - { - "tags": { - "version": "v27", - "region": "us-east-1" - }, - "tagGenerators": [], - "inherit": [] - } - ], - "routes": [ - { - "route": "/GetConversion", - "downstreamCalls": { - "currencyservice": "/Money" - }, - "tagSets": [ - { - "tags": {}, - "tagGenerators": [], - "inherit": [], - "maxLatency": 100 - } - ] - }, - { - "route": "/Money", - "downstreamCalls": {}, - "tagSets": [ - { - "tags": {}, - "tagGenerators": [], - "inherit": [], - "maxLatency": 100 - } - ] - } - ], - "instances": [ - "currencyservice-6b654dbf57-zq8dt", - "currencyservice-d847fdcf5-j6s2f" - ], - "mergedTagSets": {}, - "random": { - "seed": 66219471499700, - "nextNextGaussian": 0, - "haveNextNextGaussian": false - } - }, - { - "serviceName": "adservice", - "tagSets": [ - { - "tags": {}, - "tagGenerators": [], - "inherit": [], - "maxLatency": 500 - } - ], - "routes": [ - { - "route": "/AdRequest", - "downstreamCalls": {}, - "tagSets": [] - }, - { - "route": "/Ad", - "downstreamCalls": {}, - "tagSets": [] - } - ], - "instances": [ - "adservice-6b654dbf57-zq8dt", - "adservice-d847fdcf5-j6s2f" - ], - "mergedTagSets": {}, - "random": { - "seed": 22694143111805, - "nextNextGaussian": 0, - "haveNextNextGaussian": false - } - } - ] - }, - "rootRoutes": [ - { - "service": "frontend", - "route": "/product", - "tracesPerHour": 288 - }, - { - "service": "frontend", - "route": "/cart", - "tracesPerHour": 1440 - }, - { - "service": "frontend", - "route": "/shipping", - "tracesPerHour": 48 - }, - { - "service": "frontend", - "route": "/currency", - "tracesPerHour": 20 - }, - { - "service": "frontend", - "route": "/checkout", - "tracesPerHour": 48 - } - ] - } \ No newline at end of file From 5aca92a5a0d2f6f358bb529ac8ecf82e1e94d627 Mon Sep 17 00:00:00 2001 From: William Dumont Date: Mon, 7 Aug 2023 13:37:05 +0200 Subject: [PATCH 2/4] replace cortex by mimir and uncomment logs config --- example/docker-compose/README.md | 4 +- .../docker-compose/agent/config/agent.yaml | 14 +-- example/docker-compose/docker-compose.yaml | 18 ++-- .../grafana/datasources/datasource.yml | 6 +- .../cortex.yaml => mimir/config/mimir.yaml} | 95 ++++++++----------- 5 files changed, 59 insertions(+), 78 deletions(-) rename example/docker-compose/{cortex/config/cortex.yaml => mimir/config/mimir.yaml} (55%) diff --git a/example/docker-compose/README.md b/example/docker-compose/README.md index 817be88dd986..5480f028ff2d 100644 --- a/example/docker-compose/README.md +++ b/example/docker-compose/README.md @@ -5,7 +5,7 @@ Grafana Agent. By default, the following services are exposed: -1. Cortex for storing metrics (localhost:9009) +1. Mimir for storing metrics (localhost:9009) 2. Grafana for visualizing telemetry (localhost:3000) 3. Loki for storing logs (localhost:3100) 4. Tempo for storing traces (localhost:3200) @@ -48,7 +48,7 @@ dashboards: * The `Agent` dashboard gives a very high-level overview of running agents. * The `Agent Prometheus Remote Write` dashboard visualizes the current state of - writing metrics to Cortex. + writing metrics to Mimir. * The `Agent Tracing Pipeline` dashboard visualizes the current state of the tracing pipeline (if spans are being processed). diff --git a/example/docker-compose/agent/config/agent.yaml b/example/docker-compose/agent/config/agent.yaml index 07cf1fb67903..e611de88f5bb 100644 --- a/example/docker-compose/agent/config/agent.yaml +++ b/example/docker-compose/agent/config/agent.yaml @@ -24,7 +24,7 @@ metrics: global: scrape_interval: 60s remote_write: - - url: http://${REMOTE_WRITE_HOST:-localhost:9009}/api/prom/push + - url: http://${REMOTE_WRITE_HOST:-localhost:9009}/api/v1/push configs: - name: default scrape_configs: @@ -41,12 +41,12 @@ logs: filename: /tmp/positions.yaml scrape_configs: ## Uncomment to read logs from /var/log - #- job_name: system - # static_configs: - # - targets: [localhost] - # labels: - # job: varlogs - # __path__: /var/log/*log + - job_name: system + static_configs: + - targets: [localhost] + labels: + job: varlogs + __path__: /var/log/*log traces: configs: diff --git a/example/docker-compose/docker-compose.yaml b/example/docker-compose/docker-compose.yaml index 86f073b4439d..3a8c3d6d5ed9 100644 --- a/example/docker-compose/docker-compose.yaml +++ b/example/docker-compose/docker-compose.yaml @@ -4,7 +4,7 @@ services: # Core services. These services allow a Grafana Agent to send data somewhere # and visualize it in Grafana. # - # Backends: grafana, loki, cortex, tempo + # Backends: grafana, loki, mimir, tempo # Example services: avalanche # @@ -28,13 +28,13 @@ services: ports: - "3100:3100" - cortex: - image: cortexproject/cortex:v1.8.1 + mimir: + image: grafana/mimir:2.9.0 volumes: - - ./cortex/config:/etc/cortex-config + - ./mimir/config:/etc/mimir-config entrypoint: - - /bin/cortex - - -config.file=/etc/cortex-config/cortex.yaml + - /bin/mimir + - -config.file=/etc/mimir-config/mimir.yaml ports: - "9009:9009" @@ -72,7 +72,7 @@ services: # # Optional Grafana Agent which can collect telemetry and send it to - # Loki/Cortex/Tempo. + # Loki/Mimir/Tempo. # # Enable with the "agent" profile. # @@ -92,7 +92,7 @@ services: - -config.enable-read-api environment: HOSTNAME: agent - REMOTE_WRITE_HOST: cortex:9009 + REMOTE_WRITE_HOST: mimir:9009 LOKI_HOST: loki:3100 TEMPO_HOST: tempo:4317 AVALANCHE_HOST: avalanche:9001 @@ -108,7 +108,7 @@ services: ports: - "12345:12345" depends_on: - - cortex + - mimir - loki - tempo diff --git a/example/docker-compose/grafana/datasources/datasource.yml b/example/docker-compose/grafana/datasources/datasource.yml index 1eeb48552b2b..0a1efc20bcea 100644 --- a/example/docker-compose/grafana/datasources/datasource.yml +++ b/example/docker-compose/grafana/datasources/datasource.yml @@ -1,14 +1,14 @@ apiVersion: 1 deleteDatasources: - - name: Cortex + - name: Mimir datasources: -- name: Cortex +- name: Mimir type: prometheus access: proxy orgId: 1 - url: http://cortex:9009/api/prom + url: http://mimir:9009/prometheus basicAuth: false isDefault: false version: 1 diff --git a/example/docker-compose/cortex/config/cortex.yaml b/example/docker-compose/mimir/config/mimir.yaml similarity index 55% rename from example/docker-compose/cortex/config/cortex.yaml rename to example/docker-compose/mimir/config/mimir.yaml index dfe8243d27f1..8fe56d3ccb81 100644 --- a/example/docker-compose/cortex/config/cortex.yaml +++ b/example/docker-compose/mimir/config/mimir.yaml @@ -1,66 +1,47 @@ -auth_enabled: false - -server: - http_listen_port: 9009 - - # Configure the server to allow messages up to 100MB. - grpc_server_max_recv_msg_size: 104857600 - grpc_server_max_send_msg_size: 104857600 - grpc_server_max_concurrent_streams: 1000 - -distributor: - shard_by_all_labels: true - pool: - health_check_ingesters: true - -ingester_client: - grpc_client_config: - max_recv_msg_size: 104857600 - max_send_msg_size: 104857600 - grpc_compression: gzip - -ingester: - lifecycler: - join_after: 0 - min_ready_duration: 0s - final_sleep: 0s - num_tokens: 512 - - ring: - kvstore: - store: inmemory - replication_factor: 1 - -storage: - engine: blocks - +activity_tracker: {} +alertmanager: {} +alertmanager_storage: + backend: local blocks_storage: - tsdb: - dir: /tmp/cortex/tsdb - bucket_store: - sync_dir: /tmp/cortex/tsdb-sync - backend: filesystem + bucket_store: + sync_dir: /tmp/mimir/tsdb-sync filesystem: - dir: /tmp/cortex/blocks - + dir: /tmp/mimir/blocks + tsdb: + dir: /tmp/mimir/tsdb compactor: - data_dir: /tmp/cortex/compactor sharding_ring: kvstore: store: inmemory - -frontend_worker: - match_max_concurrent: true - -ruler: - enable_api: true - enable_sharding: false - storage: - type: local - local: - directory: /tmp/cortex/rules - +distributor: + pool: + health_check_ingesters: true +ingester: + ring: + final_sleep: 0s + kvstore: + store: inmemory + min_ready_duration: 0s + num_tokens: 512 + replication_factor: 1 +ingester_client: + grpc_client_config: + grpc_compression: gzip + max_recv_msg_size: 104857600 + max_send_msg_size: 104857600 limits: - ingestion_rate: 250000 ingestion_burst_size: 500000 + ingestion_rate: 250000 +multitenancy_enabled: false +ruler: + enable_api: true +ruler_storage: + backend: filesystem + local: + directory: /tmp/mimir/rules +server: + grpc_server_max_concurrent_streams: 1000 + grpc_server_max_recv_msg_size: 104857600 + grpc_server_max_send_msg_size: 104857600 + http_listen_port: 9009 From ec000126b5fa16d40575839cf11f6a1a01c487fa Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Marc=20Tudur=C3=AD?= Date: Fri, 28 Jul 2023 11:14:02 +0200 Subject: [PATCH 3/4] flow: Add openstack discovery component (#4606) Co-authored-by: Mischa Thompson Co-authored-by: Clayton Cornell <131809008+clayton-cornell@users.noreply.github.com> --- CHANGELOG.md | 5 +- component/all/all.go | 1 + component/discovery/openstack/openstack.go | 107 ++++++++++++ .../discovery/openstack/openstack_test.go | 94 +++++++++++ .../components/discovery.openstack.md | 153 ++++++++++++++++++ 5 files changed, 358 insertions(+), 2 deletions(-) create mode 100644 component/discovery/openstack/openstack.go create mode 100644 component/discovery/openstack/openstack_test.go create mode 100644 docs/sources/flow/reference/components/discovery.openstack.md diff --git a/CHANGELOG.md b/CHANGELOG.md index 646f469644fb..05843d733587 100644 --- a/CHANGELOG.md +++ b/CHANGELOG.md @@ -23,11 +23,12 @@ Main (unreleased) - New Grafana Agent Flow components: - - `prometheus.exporter.gcp` - scrape GCP metrics (@tburgessdev) + - `prometheus.exporter.gcp` - scrape GCP metrics. (@tburgessdev) - `otelcol.processor.span` - accepts traces telemetry data from other `otelcol` components and modifies the names and attributes of the spans. (@ptodev) - - `discovery.uyuni` discovers scrape targets from a Uyuni Server. (@spartan0x117) + - `discovery.uyuni` discovers scrape targets from a Uyuni Server. (@sparta0x117) - `discovery.eureka` discovers targets from a Eureka Service Registry. (@spartan0x117) + - `discovery.openstack` - service discovery for OpenStack. (@marctc) ### Bugfixes diff --git a/component/all/all.go b/component/all/all.go index ad620d22e650..008b282adb3c 100644 --- a/component/all/all.go +++ b/component/all/all.go @@ -15,6 +15,7 @@ import ( _ "github.com/grafana/agent/component/discovery/kubelet" // Import discovery.kubelet _ "github.com/grafana/agent/component/discovery/kubernetes" // Import discovery.kubernetes _ "github.com/grafana/agent/component/discovery/nomad" // Import discovery.nomad + _ "github.com/grafana/agent/component/discovery/openstack" // Import discovery.openstack _ "github.com/grafana/agent/component/discovery/relabel" // Import discovery.relabel _ "github.com/grafana/agent/component/discovery/uyuni" // Import discovery.uyuni _ "github.com/grafana/agent/component/local/file" // Import local.file diff --git a/component/discovery/openstack/openstack.go b/component/discovery/openstack/openstack.go new file mode 100644 index 000000000000..c32868c6e195 --- /dev/null +++ b/component/discovery/openstack/openstack.go @@ -0,0 +1,107 @@ +package openstack + +import ( + "fmt" + "time" + + "github.com/grafana/agent/component" + "github.com/grafana/agent/component/common/config" + "github.com/grafana/agent/component/discovery" + "github.com/grafana/agent/pkg/river/rivertypes" + config_util "github.com/prometheus/common/config" + "github.com/prometheus/common/model" + prom_discovery "github.com/prometheus/prometheus/discovery/openstack" +) + +func init() { + component.Register(component.Registration{ + Name: "discovery.openstack", + Args: Arguments{}, + Exports: discovery.Exports{}, + + Build: func(opts component.Options, args component.Arguments) (component.Component, error) { + return New(opts, args.(Arguments)) + }, + }) +} + +type Arguments struct { + IdentityEndpoint string `river:"identity_endpoint,attr,optional"` + Username string `river:"username,attr,optional"` + UserID string `river:"userid,attr,optional"` + Password rivertypes.Secret `river:"password,attr,optional"` + ProjectName string `river:"project_name,attr,optional"` + ProjectID string `river:"project_id,attr,optional"` + DomainName string `river:"domain_name,attr,optional"` + DomainID string `river:"domain_id,attr,optional"` + ApplicationCredentialName string `river:"application_credential_name,attr,optional"` + ApplicationCredentialID string `river:"application_credential_id,attr,optional"` + ApplicationCredentialSecret rivertypes.Secret `river:"application_credential_secret,attr,optional"` + Role string `river:"role,attr"` + Region string `river:"region,attr"` + RefreshInterval time.Duration `river:"refresh_interval,attr,optional"` + Port int `river:"port,attr,optional"` + AllTenants bool `river:"all_tenants,attr,optional"` + TLSConfig config.TLSConfig `river:"tls_config,attr,optional"` + Availability string `river:"availability,attr,optional"` +} + +var DefaultArguments = Arguments{ + Port: 80, + RefreshInterval: time.Duration(60), + Availability: "public", +} + +// SetToDefault implements river.Defaulter. +func (args *Arguments) SetToDefault() { + *args = DefaultArguments +} + +// Validate implements river.Validator. +func (args *Arguments) Validate() error { + switch args.Availability { + case "public", "internal", "admin": + default: + return fmt.Errorf("unknown availability %s, must be one of admin, internal or public", args.Availability) + } + + switch args.Role { + case "instance", "hypervisor": + default: + return fmt.Errorf("unknown availability %s, must be one of instance or hypervisor", args.Role) + } + return args.TLSConfig.Validate() +} + +func (args *Arguments) Convert() *prom_discovery.SDConfig { + tlsConfig := &args.TLSConfig + + return &prom_discovery.SDConfig{ + IdentityEndpoint: args.IdentityEndpoint, + Username: args.Username, + UserID: args.UserID, + Password: config_util.Secret(args.Password), + ProjectName: args.ProjectName, + ProjectID: args.ProjectID, + DomainName: args.DomainName, + DomainID: args.DomainID, + ApplicationCredentialName: args.ApplicationCredentialName, + ApplicationCredentialID: args.ApplicationCredentialID, + ApplicationCredentialSecret: config_util.Secret(args.ApplicationCredentialSecret), + Role: prom_discovery.Role(args.Role), + Region: args.Region, + RefreshInterval: model.Duration(args.RefreshInterval), + Port: args.Port, + AllTenants: args.AllTenants, + TLSConfig: *tlsConfig.Convert(), + Availability: args.Availability, + } +} + +// New returns a new instance of a discovery.openstack component. +func New(opts component.Options, args Arguments) (*discovery.Component, error) { + return discovery.New(opts, args, func(args component.Arguments) (discovery.Discoverer, error) { + newArgs := args.(Arguments) + return prom_discovery.NewDiscovery(newArgs.Convert(), opts.Logger) + }) +} diff --git a/component/discovery/openstack/openstack_test.go b/component/discovery/openstack/openstack_test.go new file mode 100644 index 000000000000..66032d3bf053 --- /dev/null +++ b/component/discovery/openstack/openstack_test.go @@ -0,0 +1,94 @@ +package openstack + +import ( + "testing" + "time" + + "github.com/grafana/agent/pkg/river" + "github.com/prometheus/common/config" + "github.com/prometheus/common/model" + "github.com/prometheus/prometheus/discovery/openstack" + "github.com/stretchr/testify/require" +) + +func TestUnmarshal(t *testing.T) { + cfg := ` + identity_endpoint = "http://openstack" + username = "exampleuser" + userid = "exampleuserid" + password = "examplepassword" + project_name = "exampleproject" + project_id = "exampleprojectid" + domain_name = "exampledomain" + domain_id = "exampledomainid" + application_credential_name = "exampleappcred" + application_credential_id = "exampleappcredid" + role = "hypervisor" + region = "us-east-1" + refresh_interval = "1m" + port = 80 + all_tenants = true + ` + var args Arguments + err := river.Unmarshal([]byte(cfg), &args) + require.NoError(t, err) +} + +func TestValidate(t *testing.T) { + wrongAvailability := ` + role = "hypervisor" + region = "us-east-1" + availability = "private"` + + var args Arguments + err := river.Unmarshal([]byte(wrongAvailability), &args) + require.ErrorContains(t, err, "unknown availability private, must be one of admin, internal or public") + + wrongRole := ` + role = "private" + region = "us-east-1" + availability = "public"` + + var args2 Arguments + err = river.Unmarshal([]byte(wrongRole), &args2) + require.ErrorContains(t, err, "unknown availability private, must be one of instance or hypervisor") +} + +func TestConvert(t *testing.T) { + args := Arguments{ + IdentityEndpoint: "http://openstack", + Username: "exampleuser", + UserID: "exampleuserid", + Password: "examplepassword", + ProjectName: "exampleproject", + ProjectID: "exampleprojectid", + DomainName: "exampledomain", + DomainID: "exampledomainid", + ApplicationCredentialName: "exampleappcred", + ApplicationCredentialID: "exampleappcredid", + Role: "hypervisor", + Region: "us-east-1", + RefreshInterval: time.Duration(60), + Port: 80, + AllTenants: true, + Availability: "public", + } + converted := args.Convert() + + require.Equal(t, "http://openstack", converted.IdentityEndpoint) + require.Equal(t, "exampleuser", converted.Username) + require.Equal(t, "exampleuserid", converted.UserID) + require.Equal(t, config.Secret("examplepassword"), converted.Password) + require.Equal(t, "exampleproject", converted.ProjectName) + require.Equal(t, "exampleprojectid", converted.ProjectID) + require.Equal(t, "exampledomain", converted.DomainName) + require.Equal(t, "exampledomainid", converted.DomainID) + require.Equal(t, "exampleappcred", converted.ApplicationCredentialName) + require.Equal(t, "exampleappcredid", converted.ApplicationCredentialID) + require.Equal(t, openstack.Role("hypervisor"), converted.Role) + require.Equal(t, "us-east-1", converted.Region) + require.Equal(t, model.Duration(60), converted.RefreshInterval) + require.Equal(t, 80, converted.Port) + require.Equal(t, true, converted.AllTenants) + require.Equal(t, "public", converted.Availability) +} diff --git a/docs/sources/flow/reference/components/discovery.openstack.md b/docs/sources/flow/reference/components/discovery.openstack.md new file mode 100644 index 000000000000..728ec95c6bfc --- /dev/null +++ b/docs/sources/flow/reference/components/discovery.openstack.md @@ -0,0 +1,153 @@ +--- +canonical: https://grafana.com/docs/agent/latest/flow/reference/components/discovery.openstack/ +title: discovery.openstack +--- + +# discovery.openstack + +`discovery.openstack` discovers [OpenStack][] Nova instances and exposes them as targets. + +[OpenStack]: https://docs.openstack.org/nova/latest/ + +## Usage + +```river +discovery.openstack "LABEL" { + role = "hypervisor" + region = "us-east-1" +} +``` + +## Arguments + +The following arguments are supported: + +Name | Type | Description | Default | Required +------------------- | ---------- | ---------------------------------------------------------------------- | -------------------- | -------- +`role` | `string` | Role of the discovered targets. | | yes +`region` | `string` | OpenStack region. | | yes +`identity_endpoint` | `string` | Specifies the HTTP endpoint that is required to work with te Identity API of the appropriate version | | no +`username` | `string` | OpenStack username for the Identity V2 and V3 APIs. | | no +`userid` | `string` | OpenStack userid for the Identity V2 and V3 APIs. | | no +`password` | `secret` | Password for the Identity V2 and V3 APIs. | | no +`domain_name` | `string` | OpenStack domain name for the Identity V2 and V3 APIs. | | no +`domain_id` | `string` | OpenStack domain ID for the Identity V2 and V3 APIs. | | no +`project_name` | `string` | OpenStack project name for the Identity V2 and V3 APIs. | | no +`project_id` | `string` | OpenStack project ID for the Identity V2 and V3 APIs. | | no +`application_credential_name` | `string` | OpenStack application credential name for the Identity V2 and V3 APIs. | | no +`application_credential_id` | `string` | OpenStack application credential ID for the Identity V2 and V3 APIs. | | no +`application_credential_secret` | `secret` | OpenStack application credential secret for the Identity V2 and V3 APIs. | | no +`all_tenants` | `bool` | Whether the service discovery should list all instances for all projects. | `false` | no +`refresh_interval` | `duration`| Refresh interval to re-read the instance list. | `60s` | no +`port` | `int` | The port to scrape metrics from. | `80` | no +`availability` | `string` | The availability of the endpoint to connect to. | `public` | no + +`role` must be one of `hypervisor` or `instance`. + +`username` is required if using Identity V2 API. In Identity V3, either `userid` or a combination of `username` and `domain_id` or `domain_name` are needed. + +`project_id` and `project_name` fields are optional for the Identity V2 API. Some providers allow you to specify a `project_name` instead of the `project_id`. Some require both. + +`application_credential_id` or `application_credential_name` fields are required if using an application credential to authenticate. Some providers allow you to create an application credential to authenticate rather than a password. + +`application_credential_secret` field is required if using an application credential to authenticate. + +`all_tenants` is only relevant for the `instance` role and usually requires admin permissions. + +`availability` must be one of `public`, `admin`, or `internal`. + +## Blocks +The following blocks are supported inside the definition of `discovery.openstack`: + +Hierarchy | Block | Description | Required +--------- | ----- | ----------- | -------- +tls_config | [tls_config][] | TLS configuration for requests to the OpenStack API. | no + +[tls_config]: #tls_config-block + +### tls_config block + +{{< docs/shared lookup="flow/reference/components/tls-config-block.md" source="agent" >}} + +## Exported fields + +The following fields are exported and can be referenced by other components: + +Name | Type | Description +--------- | ------------------- | ----------- +`targets` | `list(map(string))` | The set of targets discovered from the OpenStack API. + +#### `hypervisor` + +The `hypervisor` role discovers one target per Nova hypervisor node. The target +address defaults to the `host_ip` attribute of the hypervisor. + +* `__meta_openstack_hypervisor_host_ip`: the hypervisor node's IP address. +* `__meta_openstack_hypervisor_hostname`: the hypervisor node's name. +* `__meta_openstack_hypervisor_id`: the hypervisor node's ID. +* `__meta_openstack_hypervisor_state`: the hypervisor node's state. +* `__meta_openstack_hypervisor_status`: the hypervisor node's status. +* `__meta_openstack_hypervisor_type`: the hypervisor node's type. + +#### `instance` + +The `instance` role discovers one target per network interface of Nova +instance. The target address defaults to the private IP address of the network +interface. + +* `__meta_openstack_address_pool`: the pool of the private IP. +* `__meta_openstack_instance_flavor`: the flavor of the OpenStack instance. +* `__meta_openstack_instance_id`: the OpenStack instance ID. +* `__meta_openstack_instance_image`: the ID of the image the OpenStack instance is using. +* `__meta_openstack_instance_name`: the OpenStack instance name. +* `__meta_openstack_instance_status`: the status of the OpenStack instance. +* `__meta_openstack_private_ip`: the private IP of the OpenStack instance. +* `__meta_openstack_project_id`: the project (tenant) owning this instance. +* `__meta_openstack_public_ip`: the public IP of the OpenStack instance. +* `__meta_openstack_tag_`: each tag value of the instance. +* `__meta_openstack_user_id`: the user account owning the tenant. + +## Component health + +`discovery.openstack` is only reported as unhealthy when given an invalid +configuration. In those cases, exported fields retain their last healthy +values. + +## Debug information + +`discovery.openstack` does not expose any component-specific debug information. + +### Debug metrics + +`discovery.openstack` does not expose any component-specific debug metrics. + +## Example + +```river +discovery.openstack "example" { + role = OPENSTACK_ROLE + region = OPENSTACK_REGION +} + +prometheus.scrape "demo" { + targets = discovery.openstack.example.targets + forward_to = [prometheus.remote_write.demo.receiver] +} + +prometheus.remote_write "demo" { + endpoint { + url = PROMETHEUS_REMOTE_WRITE_URL + + basic_auth { + username = USERNAME + password = PASSWORD + } + } +} +``` +Replace the following: + - `OPENSTACK_ROLE`: Your OpenStack role. + - `OPENSTACK_REGION`: Your OpenStack region. + - `PROMETHEUS_REMOTE_WRITE_URL`: The URL of the Prometheus remote_write-compatible server to send metrics to. + - `USERNAME`: The username to use for authentication to the remote_write API. + - `PASSWORD`: The password to use for authentication to the remote_write API. From aaf5af94b085d0792c0474ec81a7220fba8b8465 Mon Sep 17 00:00:00 2001 From: William Dumont Date: Mon, 7 Aug 2023 14:26:38 +0200 Subject: [PATCH 4/4] reorder config in mimir.yaml to match previous file --- .../docker-compose/mimir/config/mimir.yaml | 63 +++++++++++-------- 1 file changed, 38 insertions(+), 25 deletions(-) diff --git a/example/docker-compose/mimir/config/mimir.yaml b/example/docker-compose/mimir/config/mimir.yaml index 8fe56d3ccb81..ff8ae8e66731 100644 --- a/example/docker-compose/mimir/config/mimir.yaml +++ b/example/docker-compose/mimir/config/mimir.yaml @@ -1,7 +1,37 @@ activity_tracker: {} + alertmanager: {} + alertmanager_storage: backend: local + +server: + http_listen_port: 9009 + + # Configure the server to allow messages up to 100MB. + grpc_server_max_recv_msg_size: 104857600 + grpc_server_max_send_msg_size: 104857600 + grpc_server_max_concurrent_streams: 1000 + +distributor: + pool: + health_check_ingesters: true + +ingester_client: + grpc_client_config: + grpc_compression: gzip + max_recv_msg_size: 104857600 + max_send_msg_size: 104857600 + +ingester: + ring: + final_sleep: 0s + kvstore: + store: inmemory + min_ready_duration: 0s + num_tokens: 512 + replication_factor: 1 + blocks_storage: backend: filesystem bucket_store: @@ -10,38 +40,21 @@ blocks_storage: dir: /tmp/mimir/blocks tsdb: dir: /tmp/mimir/tsdb + compactor: sharding_ring: kvstore: store: inmemory -distributor: - pool: - health_check_ingesters: true -ingester: - ring: - final_sleep: 0s - kvstore: - store: inmemory - min_ready_duration: 0s - num_tokens: 512 - replication_factor: 1 -ingester_client: - grpc_client_config: - grpc_compression: gzip - max_recv_msg_size: 104857600 - max_send_msg_size: 104857600 -limits: - ingestion_burst_size: 500000 - ingestion_rate: 250000 -multitenancy_enabled: false + ruler: enable_api: true + ruler_storage: backend: filesystem local: directory: /tmp/mimir/rules -server: - grpc_server_max_concurrent_streams: 1000 - grpc_server_max_recv_msg_size: 104857600 - grpc_server_max_send_msg_size: 104857600 - http_listen_port: 9009 + +limits: + ingestion_burst_size: 500000 + ingestion_rate: 250000 + multitenancy_enabled: false \ No newline at end of file