From 434bdd6fbcd880c2c14dfbeb9ae8f2cddadf44c9 Mon Sep 17 00:00:00 2001 From: Gregor Zeitlinger Date: Tue, 17 Dec 2024 08:48:41 +0100 Subject: [PATCH 01/10] remove unused code --- yaml/docker-compose-java-template.yml | 33 -------- yaml/java_generator.go | 114 -------------------------- yaml/model.go | 14 +--- 3 files changed, 4 insertions(+), 157 deletions(-) delete mode 100644 yaml/docker-compose-java-template.yml delete mode 100644 yaml/java_generator.go diff --git a/yaml/docker-compose-java-template.yml b/yaml/docker-compose-java-template.yml deleted file mode 100644 index 569f70a..0000000 --- a/yaml/docker-compose-java-template.yml +++ /dev/null @@ -1,33 +0,0 @@ -version: "3.9" -services: - application: - image: "{{ .Image }}" - volumes: - - "{{ .JavaAgent }}:/grafana-opentelemetry-java.jar" - - "{{ .ApplicationJar }}:/app.jar" - {{if .JmxConfig }}- "{{ .JmxConfig }}:/otel-jmx-config.yaml" {{end}} - environment: - GRAFANA_OTLP_DEBUG_LOGGING: "true" - OTEL_LOGS_EXPORTER: otlp # otherwise, it's too verbose - OTEL_SERVICE_NAME: "app" - OTEL_RESOURCE_ATTRIBUTES: 'deployment.environment=production,service.namespace=shop,service.version=1.1' - GRAFANA_OTEL_USE_TESTED_INSTRUMENTATIONS: "{{ not .UseAllInstrumentations }}" - GRAFANA_OTEL_APPLICATION_OBSERVABILITY_METRICS: "{{ not .DisableDataSaver }}" - OTEL_METRIC_EXPORT_INTERVAL: "5000" # so we don't have to wait 60s for metrics - OTEL_JAVAAGENT_DEBUG: "true" - OTEL_EXPORTER_OTLP_ENDPOINT: "http://lgtm:4318" - {{if .JmxConfig }}OTEL_JMX_CONFIG: /otel-jmx-config.yaml{{end}} - command: - - /bin/bash - - -c - - java {{ .JvmDebug }} -javaagent:grafana-opentelemetry-java.jar -jar /app.jar - ports: - - "{{ .ApplicationPort }}:8080" - {{if .JvmDebug }}- "5005:5005" {{end}} - lgtm: - image: grafana/otel-lgtm:latest - ports: - - "{{ .GrafanaHTTPPort }}:3000" - - "{{ .PrometheusHTTPPort }}:9090" - - "{{ .TempoHTTPPort }}:3200" - - "{{ .LokiHTTPPort }}:3100" diff --git a/yaml/java_generator.go b/yaml/java_generator.go deleted file mode 100644 index a7b4a19..0000000 --- a/yaml/java_generator.go +++ /dev/null @@ -1,114 +0,0 @@ -package yaml - -import ( - "fmt" - "github.com/onsi/ginkgo/v2" - . "github.com/onsi/gomega" - "os" - "os/exec" - "path/filepath" - "regexp" - "strings" - "time" -) - -func (c *TestCase) applicationJar() string { - t := time.Now() - build := os.Getenv("TESTCASE_SKIP_BUILD") != "true" - if build { - ginkgo.GinkgoWriter.Printf("building application jar in %s\n", c.Dir) - // create a new app.jar - only needed for local testing - maybe add an option to skip this in CI - cmd := exec.Command(filepath.FromSlash("../../../gradlew"), "clean", "build") - cmd.Dir = c.Dir - cmd.Stdout = os.Stdout - cmd.Stderr = os.Stdout - - err := cmd.Run() - Expect(err).ToNot(HaveOccurred(), "could not build application jar") - } - - pattern := c.Dir + filepath.FromSlash("/build/libs/*SNAPSHOT.jar") - matches, err := filepath.Glob(pattern) - Expect(err).ToNot(HaveOccurred(), "could not find application jar") - Expect(matches).To(HaveLen(1)) - - file := matches[0] - - if build { - fileinfo, err := os.Stat(file) - Expect(err).ToNot(HaveOccurred()) - Expect(fileinfo.ModTime()).To(BeTemporally(">=", t), "application jar was not built") - } - - return file -} - -func imageName(dir string) string { - content, err := os.ReadFile(filepath.Join(dir, ".tool-versions")) - Expect(err).ToNot(HaveOccurred(), "could not read .tool-versions") - for _, line := range strings.Split(string(content), "\n") { - if strings.HasPrefix(line, "java ") { - // find major version in java temurin-8.0.372+7 using regex - major := regexp.MustCompile("java temurin-(\\d+).*").FindStringSubmatch(line)[1] - return fmt.Sprintf("eclipse-temurin:%s-jre", major) - } - } - ginkgo.Fail("no java version found") - return "" -} - -func (c *TestCase) javaTemplateVars() (string, map[string]any) { - projectDir := strings.Split(c.Dir, filepath.FromSlash("examples/"))[0] - agent := filepath.Join(projectDir, filepath.FromSlash("agent/build/libs/grafana-opentelemetry-java.jar")) - - _, err := os.Stat(agent) - if err != nil { - buildAgent(projectDir) - } - - image := imageName(c.Dir) - params := c.Definition.DockerCompose.JavaGeneratorParams - return filepath.FromSlash("./docker-compose-java-template.yml"), map[string]any{ - "Image": image, - "JavaAgent": filepath.ToSlash(agent), - "ApplicationJar": filepath.ToSlash(c.applicationJar()), - "JmxConfig": jmxConfig(c.Dir, params.OtelJmxConfig), - "DisableDataSaver": params.DisableDataSaver, - "JvmDebug": jvmDebug(image), - "UseAllInstrumentations": os.Getenv("TESTCASE_INCLUDE_ALL_INSTRUMENTATIONS") == "true", - } -} - -func jvmDebug(image string) string { - if os.Getenv("TESTCASE_JVM_DEBUG") != "true" { - return "" - } - port := "" - if image == "eclipse-temurin:8-jre" { - port = "5005" - } else { - port = "*:5005" - } - ginkgo.GinkgoWriter.Printf("jvm debug port: %s\n", port) - return fmt.Sprintf("-agentlib:jdwp=transport=dt_socket,server=y,suspend=y,address=%s", port) -} - -func buildAgent(projectDir string) { - ginkgo.GinkgoWriter.Printf("building javaagent in %s\n", projectDir) - cmd := exec.Command(filepath.FromSlash("./gradlew"), "clean", "build") - cmd.Dir = projectDir - cmd.Stdout = os.Stdout - cmd.Stderr = os.Stdout - - err := cmd.Run() - Expect(err).ToNot(HaveOccurred(), "could not build javaagent jar") -} - -func jmxConfig(dir string, jmxConfig string) string { - if jmxConfig == "" { - return "" - } - p := filepath.Join(dir, jmxConfig) - Expect(p).To(BeAnExistingFile(), "jmx config file does not exist") - return filepath.ToSlash(p) -} diff --git a/yaml/model.go b/yaml/model.go index 9206b16..baaf647 100644 --- a/yaml/model.go +++ b/yaml/model.go @@ -67,22 +67,16 @@ type Expected struct { CustomChecks []CustomCheck `yaml:"custom-checks"` } -type JavaGeneratorParams struct { - OtelJmxConfig string `yaml:"otel-jmx-config"` - DisableDataSaver bool `yaml:"disable-data-saver"` -} - type Matrix struct { Name string `yaml:"name"` DockerCompose *DockerCompose `yaml:"docker-compose"` } type DockerCompose struct { - Generator string `yaml:"generator"` - Files []string `yaml:"files"` - Environment []string `yaml:"env"` - Resources []string `yaml:"resources"` - JavaGeneratorParams JavaGeneratorParams `yaml:"java-generator-params"` + Generator string `yaml:"generator"` + Files []string `yaml:"files"` + Environment []string `yaml:"env"` + Resources []string `yaml:"resources"` } type Input struct { From aff3c1ec24ed62fe91a0ec1ada7579fd58f65820 Mon Sep 17 00:00:00 2001 From: Gregor Zeitlinger Date: Tue, 17 Dec 2024 13:32:38 +0100 Subject: [PATCH 02/10] remove unused code --- yaml/docker-compose-lgtm-template.yml | 45 ------------------------- yaml/generator.go | 47 ++++----------------------- yaml/model.go | 9 +---- 3 files changed, 8 insertions(+), 93 deletions(-) delete mode 100644 yaml/docker-compose-lgtm-template.yml diff --git a/yaml/docker-compose-lgtm-template.yml b/yaml/docker-compose-lgtm-template.yml deleted file mode 100644 index 7630003..0000000 --- a/yaml/docker-compose-lgtm-template.yml +++ /dev/null @@ -1,45 +0,0 @@ -version: "3.9" -services: - grafana: - image: grafana/grafana:10.0.5 - # environment: - # - GF_AUTH_DISABLE_LOGIN_FORM=true - # - GF_AUTH_ANONYMOUS_ENABLED=true - # - GF_AUTH_ANONYMOUS_ORG_NAME=OATs - # - GF_AUTH_ANONYMOUS_ORG_ROLE=admin - volumes: - - "{{ .ConfigDir }}/grafana-datasources.yaml:/etc/grafana/provisioning/datasources/grafana-datasources.yaml" - - "{{ .ConfigDir }}/grafana-dashboards.yaml:/etc/grafana/provisioning/dashboards/grafana-dashboards.yaml" - - "{{ .Dashboard }}:/etc/grafana/grafana-test-dashboard.json" - ports: - - "{{ .GrafanaHTTPPort }}:3000" - prometheus: - image: prom/prometheus:v2.47.0 - command: - - --web.enable-remote-write-receiver - - --enable-feature=exemplar-storage - - --enable-feature=native-histograms - - --config.file=/etc/prometheus/prometheus.yml - ports: - - "{{ .PrometheusHTTPPort }}:9090" - tempo: - image: grafana/tempo:2.2.3 - volumes: - - "{{ .ConfigDir }}/tempo-config.yaml:/config.yaml" - command: - - --config.file=/config.yaml - ports: - - "{{ .TempoHTTPPort }}:3200" - - loki: - image: grafana/loki:2.9.0 - ports: - - "{{ .LokiHTTPPort }}:3100" - collector: - image: otel/opentelemetry-collector-contrib:0.85.0 - volumes: - - "{{ .ConfigDir }}/otelcol-config.yaml:/config.yaml" - command: - - --config=file:/config.yaml - # we currently don't support this in our dashboards and grafana agent doesn't understand it yet - - --feature-gates=-pkg.translator.prometheus.NormalizeName diff --git a/yaml/generator.go b/yaml/generator.go index fab53a1..4d82255 100644 --- a/yaml/generator.go +++ b/yaml/generator.go @@ -27,35 +27,7 @@ func (c *TestCase) CreateDockerComposeFile() string { } func (c *TestCase) getContent(compose *DockerCompose) []byte { - if compose.Generator != "" { - return c.generateDockerComposeFile() - } else { - // TODO: allow for template vars on docker-compose files, similar to generator - var buf []byte - for _, filename := range compose.Files { - var err error - buf, err = joinComposeFiles(buf, readComposeFile(compose, filename)) - Expect(err).ToNot(HaveOccurred()) - } - return buf - } -} - -func readComposeFile(compose *DockerCompose, file string) []byte { - b, err := os.ReadFile(file) - Expect(err).ToNot(HaveOccurred()) - return replaceRefs(compose, b) -} - -func replaceRefs(compose *DockerCompose, bytes []byte) []byte { - baseDir := filepath.Dir(compose.Files[0]) // TODO: more direct way of getting baseDir? - lines := strings.Split(string(bytes), "\n") - for i, line := range lines { - for _, resource := range compose.Resources { - lines[i] = strings.ReplaceAll(line, "./"+resource, filepath.Join(baseDir, resource)) - } - } - return []byte(strings.Join(lines, "\n")) + return c.generateDockerComposeFile() } func (c *TestCase) generateDockerComposeFile() []byte { @@ -70,7 +42,12 @@ func (c *TestCase) generateDockerComposeFile() []byte { configDir, err := filepath.Abs("configs") Expect(err).ToNot(HaveOccurred()) - name, vars := c.getTemplateVars() + generator := c.Definition.DockerCompose.Generator + if generator == "" { + generator = "docker-lgtm" + } + name := filepath.FromSlash("./docker-compose-" + generator + "-template.yml") + vars := map[string]any{} vars["Dashboard"] = filepath.ToSlash(dashboard) vars["ConfigDir"] = filepath.ToSlash(configDir) vars["ApplicationPort"] = c.PortConfig.ApplicationPort @@ -133,16 +110,6 @@ func (c *TestCase) generateDockerComposeFile() []byte { return content } -func (c *TestCase) getTemplateVars() (string, map[string]any) { - generator := c.Definition.DockerCompose.Generator - switch generator { - case "java": - return c.javaTemplateVars() - default: - return filepath.FromSlash("./docker-compose-" + generator + "-template.yml"), map[string]any{} - } -} - func joinComposeFiles(template []byte, addition []byte) ([]byte, error) { base := map[string]any{} add := map[string]any{} diff --git a/yaml/model.go b/yaml/model.go index baaf647..e911ed2 100644 --- a/yaml/model.go +++ b/yaml/model.go @@ -73,10 +73,9 @@ type Matrix struct { } type DockerCompose struct { - Generator string `yaml:"generator"` + Generator string `yaml:"generator"` // deprecated: only used by beyla Files []string `yaml:"files"` Environment []string `yaml:"env"` - Resources []string `yaml:"resources"` } type Input struct { @@ -256,12 +255,6 @@ func validateDockerCompose(d *DockerCompose, dir string) { for i, filename := range d.Files { d.Files[i] = filepath.Join(dir, filename) Expect(d.Files[i]).To(BeARegularFile()) - for _, resource := range d.Resources { - Expect(filepath.Join(filepath.Dir(d.Files[i]), resource)).To(BeAnExistingFile()) - } } - } else { - Expect(d.Generator).ToNot(BeEmpty(), "generator needed if no file is specified") - Expect(d.Resources).To(BeEmpty(), "resources requires file") } } From 61120a6f248df7a8b1febb32aa1bf0233dca0341 Mon Sep 17 00:00:00 2001 From: Gregor Zeitlinger Date: Tue, 17 Dec 2024 14:14:21 +0100 Subject: [PATCH 03/10] new readme --- README.md | 191 +++++++++++++++++++++++++------ yaml/README.md | 172 +--------------------------- yaml/testdata/oats-merged.yaml | 1 - yaml/testdata/oats-template.yaml | 1 - 4 files changed, 161 insertions(+), 204 deletions(-) diff --git a/README.md b/README.md index be38886..2b0d916 100644 --- a/README.md +++ b/README.md @@ -1,48 +1,175 @@ -## OpenTelemetry Acceptance Tests (OATs) +# OpenTelemetry Acceptance Tests (OATs) -### Goals +OpenTelemetry Acceptance Tests (OATs), or OATs for short, is a test framework for OpenTelemetry. -1. Flexibility to support qualification of changes to the [OpenTelemetry Collector][], and [Tempo][] -1. Ability to support OpenTelemetry SDK functionality such as [sampling][] -1. Highlight the use of [Ginkgo][], and [Gomega][] -1. Have a cute name +- Declarative tests written in YAML +- Supported signals: traces, logs, metrics +- Full round-trip testing: from the application to the observability stack + - Data is stored in the LGTM stack ([Loki], [Grafana], [Tempo], [Prometheus], [OpenTelemetry Collector]) + - Data is queried using LogQL, PromQL, and TraceQL + - All data is sent to the observability stack via OTLP - so OATS can also be used with other observability stacks +- End-to-end testing + - Docker Compose with the [docker-otel-lgtm] image + - Kubernetes with the [docker-otel-lgtm] and [k3d] -[Tempo]: https://github.com/grafana/tempo -[OpenTelemetry Collector]: https://github.com/open-telemetry/opentelemetry-collector -[Prometheus]: https://github.com/prometheus/prometheus -[dockertest]: https://github.com/ory/dockertest -[sampling]: https://opentelemetry.io/docs/instrumentation/go/sampling/ -[Ginkgo]: https://onsi.github.io/ginkgo/ -[Gomega]: https://onsi.github.io/gomega/ +Under the hood, OATs uses [Ginkgo] and [Gomega] to run the tests. + +## Getting Started + +> You can use the test cases in [prom_client_java](https://github.com/prometheus/client_java/tree/main/examples/example-exporter-opentelemetry/oats-tests) as a reference. +> The [GitHub action](https://github.com/prometheus/client_java/blob/main/.github/workflows/acceptance-tests.yml) +> uses a [script](https://github.com/prometheus/client_java/blob/main/scripts/run-acceptance-tests.sh) to run the tests. + +1. Create a folder `oats-tests` for the following files +2. Create `Dockerfile` to build the application you want to test + ```Dockerfile + FROM eclipse-temurin:21-jre + COPY target/example-exporter-opentelemetry.jar ./app.jar + ENTRYPOINT [ "java", "-jar", "./app.jar" ] + ``` +3. Create `docker-compose.yaml` to start the application and any dependencies + ```yaml + version: '3.4' + + services: + java: + build: + dockerfile: Dockerfile + environment: + OTEL_SERVICE_NAME: "rolldice" + OTEL_EXPORTER_OTLP_ENDPOINT: http://lgtm:4318 + OTEL_EXPORTER_OTLP_PROTOCOL: http/protobuf + OTEL_METRIC_EXPORT_INTERVAL: "5000" # so we don't have to wait 60s for metrics + ``` +4. Create `oats.yaml` with the test cases + ```yaml + # OATS is an acceptance testing framework for OpenTelemetry - https://github.com/grafana/oats + docker-compose: + files: + - ./docker-compose.yaml + expected: + metrics: + - promql: 'uptime_seconds_total{}' + value: '>= 0' + ``` +5. `cd /path/to/oats/yaml` +6. `go install github.com/onsi/ginkgo/v2/ginkgo` +7. `TESTCASE_BASE_PATH=/path/to/oats-tests ginkgo -v` + +## Test Case Syntax -### Getting Started +> You can use any file name that matches `oats*.yaml` (e.g. `oats-test.yaml`), that doesn't end in `-template.yaml`. +> `oats-template.yaml` is reserved for template files, which are used in the "include" section. -1. Install [Go][] -1. Install [Docker][] ([Podman][] also works provided it is listening on the expected Docker Unix socket) -1. Clone the repository -1. Ensure that `${GOBIN}` is on your `${PATH}` -1. From within the repository directory, install [Ginkgo][] +The syntax is a bit similar to https://github.com/kubeshop/tracetest +This is an example: + +```yaml +include: + - ../oats-template.yaml +docker-compose: + file: ../docker-compose.yaml +input: + - url: http://localhost:8080/stock +interval: 500ms +expected: + traces: + - traceql: '{ name =~ "SELECT .*product"}' + spans: + - name: 'regex:SELECT .*' + attributes: + db.system: h2 + logs: + - logql: '{exporter = "OTLP"}' + contains: + - 'hello LGTM' + metrics: + - promql: 'db_client_connections_max{pool_name="HikariPool-1"}' + value: "== 10" + dashboards: + - path: ../jdbc-dashboard.json + panels: + - title: Connection pool waiting requests + value: "== 0" + - title: Connection pool utilization + value: "> 0" ``` -go install github.com/onsi/ginkgo/v2/ginkgo + +You have to provide the root path of the directory where your test cases are located to ginkgo +via the environment variable `TESTCASE_BASE_PATH`. + +## Docker Compose + +Describes the docker-compose file(s) to use for the test. +The files typically defines the instrumented application you want to test and optionally some dependencies, +e.g. a database server to send requests to. +You don't need (and should have) to define the observability stack (e.g. prometheus, grafana, etc.), +because this is provided by the test framework (and may test different versions of the observability stack, +e.g. otel collector and grafana agent). + +This docker-compose file is relative to the `oats.yaml` file. + +## Kubernetes + +A local kubernetes cluster can be used to test the application in a kubernetes environment rather than in docker-compose. +This is useful to test the application in a more realistic environment - and when you want to test Kubernetes specific features. + +Describes the kubernetes manifest(s) to use for the test. + +```yaml +kubernetes: + dir: k8s + app-service: dice + app-docker-file: Dockerfile + app-docker-context: .. + app-docker-tag: dice:1.1-SNAPSHOT + app-docker-port: 8080 ``` -1. Run the specs +## Matrix of test cases +Matrix tests are useful to test different configurations of the same application, +e.g. with different settings of the otel collector or different flags in the application. + +```yaml +matrix: + - name: new + docker-compose: + - name: old-jvm-metrics + docker-compose: +input: + - path: /stock ``` -ginkgo -r (or ginkgo ./...) + +## Debugging + +If you want to run a single test case, you can use the `--focus` option: + +```sh +TESTCASE_BASE_PATH=/path/to/project ginkgo -v --focus="jdbc" ``` -1. Browse the [example][] +You can increase the timeout, which is useful if you want to inspect the telemetry data manually +in grafana at http://localhost:3000 -[Go]: https://go.dev/ -[Docker]: https://www.docker.com/ -[Podman]: https://podman.io/ -[example]: examples/dockertest/send_simple_trace_test.go +```sh +TESTCASE_TIMEOUT=1h TESTCASE_BASE_PATH=/path/to/project ginkgo -v +``` -### Writing Specs +You can keep the container running without executing the tests - which is useful to debug in grafana manually: + +```sh +TESTCASE_MANUAL_DEBUG=true TESTCASE_BASE_PATH=/path/to/project ginkgo -v +``` + +[Ginkgo]: https://onsi.github.io/ginkgo/ +[Gomega]: https://onsi.github.io/gomega/ +[Tempo]: https://github.com/grafana/tempo +[OpenTelemetry Collector]: https://opentelemetry.io/docs/collector/ +[Prometheus]: https://prometheus.io/ +[Grafana]: https://grafana.com/ +[Loki]: https://github.com/grafana/loki +[docker-otel-lgtm]: https://github.com/grafana/docker-otel-lgtm/ +[k3d]: https://k3d.io/ -1. Decide whether to use the `testhelpers/observability` package, individual packages such as - `testhelpers/tempo`, or only support externally provisioned endpoints -1. Write the specs using [Ginkgo][], and [Gomega][] -1. Profit diff --git a/yaml/README.md b/yaml/README.md index b65d5e2..07b4707 100644 --- a/yaml/README.md +++ b/yaml/README.md @@ -1,171 +1,3 @@ -# Declarative Yaml tests +# Declarative YAML tests -You can use declarative yaml tests in `oats.yaml` files: - -> You can use any file name that matches `oats*.yaml` (e.g. `oats-test.yaml`), that doesn't end in `-template.yaml`. -> `oats-template.yaml` is reserved for template files, which are used in the "include" section. - -The syntax is a bit similar to https://github.com/kubeshop/tracetest - -This is an example: - -```yaml -include: - - ../oats-template.yaml -docker-compose: - generator: java - file: ../docker-compose.yaml - resources: - - kafka -input: - - url: http://localhost:8080/stock -interval: 500ms -expected: - traces: - - traceql: '{ name =~ "SELECT .*product"}' - spans: - - name: 'regex:SELECT .*' - attributes: - db.system: h2 - logs: - - logql: '{exporter = "OTLP"}' - contains: - - 'hello LGTM' - metrics: - - promql: 'db_client_connections_max{pool_name="HikariPool-1"}' - value: "== 10" - dashboards: - - path: ../jdbc-dashboard.json - panels: - - title: Connection pool waiting requests - value: "== 0" - - title: Connection pool utilization - value: "> 0" -``` - -You have to provide the root path of the directory where your test cases are located to ginkgo -via the environment variable `TESTCASE_BASE_PATH`. - -## Docker Compose - -Describes the docker-compose file(s) to use for the test. -The files typically defines the instrumented application you want to test and optionally some dependencies, -e.g. a database server to send requests to. -You don't need (and should have) to define the observability stack (e.g. prometheus, grafana, etc.), -because this is provided by the test framework (and may test different versions of the observability stack, -e.g. otel collector and grafana agent). - -This docker-compose file is relative to the `oats.yaml` file. -If you're referencing other configuration files, you can use the `resources` field to specify them. - -### Generators - -Generators can be used to generate a docker-compose file from a template as a way to avoid repetition. - -Currently, the only defined generator is `java` which generates a docker-compose file for the java distribution -examples. -Using an undefined generator name (e.g.) `name` will result in using the file `docker-compose-name-template.yml` -and performing template variable substitution, with the vars as seen in this excerpt of generateDockerComposeFile() in generator.go: -``` - vars["Dashboard"] = filepath.ToSlash(dashboard) - vars["ConfigDir"] = filepath.ToSlash(configDir) - vars["ApplicationPort"] = c.PortConfig.ApplicationPort - vars["GrafanaHTTPPort"] = c.PortConfig.GrafanaHTTPPort - vars["PrometheusHTTPPort"] = c.PortConfig.PrometheusHTTPPort - vars["LokiHTTPPort"] = c.PortConfig.LokiHTTPPort - vars["TempoHTTPPort"] = c.PortConfig.TempoHTTPPort -``` -Additional variables could be added for more specific generators as needed. (e.g. add new case in getTemplateVars() that adds more vars.) - -When a generator is used, template variable interpolation will also occur on all docker-compose file(s). - -## Kubernetes - -A local kubernetes cluster can be used to test the application in a kubernetes environment rather than in docker-compose. -This is useful to test the application in a more realistic environment - and when you want to test Kubernetes specific features. - -Describes the kubernetes manifest(s) to use for the test. - -```yaml -kubernetes: - dir: k8s - app-service: dice - app-docker-file: Dockerfile - app-docker-context: .. - app-docker-tag: dice:1.1-SNAPSHOT - app-docker-port: 8080 -``` - -## Matrix of test cases - -Matrix tests are useful to test different configurations of the same application, -e.g. with different settings of the otel collector or different flags in the application. - -```yaml -matrix: - - name: new - docker-compose: - generator: java - - name: old-jvm-metrics - docker-compose: - generator: java - java-generator-params: - old-jvm-metrics: true - disable-data-saver: true -input: - - path: /stock -``` - -## Starting the Tests - -The java distribution is used as an example here, but you can use any other example. - -```sh -TESTCASE_BASE_PATH=/path/to/grafana-opentelemetry-java/examples ginkgo -v -r -``` - -If you want to run a single test case, you can use the `--focus` option: - -```sh -TESTCASE_BASE_PATH=/path/to/grafana-opentelemetry-java/examples ginkgo -v -r --focus="jdbc" -``` - -You can increase the timeout, which is useful if you want to inspect the telemetry data manually -in grafana at http://localhost:3000 - -```sh -TESTCASE_TIMEOUT=1h TESTCASE_BASE_PATH=/path/to/grafana-opentelemetry-java/examples ginkgo -v -r -``` - -You can also run the tests in parallel: - -```sh -TESTCASE_BASE_PATH=/path/to/grafana-opentelemetry-java/examples ginkgo -v -r -p -``` - -You can keep the container running without executing the tests - which is useful to debug in grafana manually: - -```sh -TESTCASE_MANUAL_DEBUG=true TESTCASE_BASE_PATH=/path/to/grafana-opentelemetry-java/examples ginkgo -v -r -``` - -### Java specific options - -If you don't want to build the java examples, you can use the `TESTCASE_SKIP_BUILD` environment variable: - -```sh -TESTCASE_SKIP_BUILD=true TESTCASE_BASE_PATH=/path/to/grafana-opentelemetry-java/examples ginkgo -v -r -``` - -If you want to attach a debugger to the java application, you can use the `TESTCASE_JVM_DEBUG` environment variable: - -```sh -TESTCASE_JVM_DEBUG=true TESTCASE_BASE_PATH=/path/to/grafana-opentelemetry-java/examples ginkgo -v -r -``` - -If you want to enable all instrumentations (including the ones that are disabled by default), you can use the `TESTCASE_INCLUDE_ALL_INSTRUMENTATIONS` environment variable: - -```sh -TESTCASE_INCLUDE_ALL_INSTRUMENTATIONS=true TESTCASE_BASE_PATH=/path/to/grafana-opentelemetry-java/examples ginkgo -v -r -``` -You can then attach a debugger to the java application at port 5005. +See [OpenTelemetry Acceptance Tests (OATs)](../README.md) for more information. diff --git a/yaml/testdata/oats-merged.yaml b/yaml/testdata/oats-merged.yaml index 226c33c..7499618 100644 --- a/yaml/testdata/oats-merged.yaml +++ b/yaml/testdata/oats-merged.yaml @@ -1,5 +1,4 @@ docker-compose: - generator: java input: - url: http://localhost:8080/stock expected: diff --git a/yaml/testdata/oats-template.yaml b/yaml/testdata/oats-template.yaml index 039aef3..54f5c1a 100644 --- a/yaml/testdata/oats-template.yaml +++ b/yaml/testdata/oats-template.yaml @@ -1,5 +1,4 @@ docker-compose: - generator: java input: - url: http://localhost:8080/stock expected: From 3537a156cd49b8aea555de6aef5a26a11d1393d9 Mon Sep 17 00:00:00 2001 From: Gregor Zeitlinger Date: Tue, 17 Dec 2024 14:32:58 +0100 Subject: [PATCH 04/10] remove unused code --- yaml/model.go | 38 +++++++++++-------------------- yaml/runner.go | 58 +++++++++++++----------------------------------- yaml/testcase.go | 12 ---------- 3 files changed, 29 insertions(+), 79 deletions(-) diff --git a/yaml/model.go b/yaml/model.go index e911ed2..ab599cd 100644 --- a/yaml/model.go +++ b/yaml/model.go @@ -16,9 +16,8 @@ import ( ) type ExpectedDashboardPanel struct { - Title string `yaml:"title"` - Value string `yaml:"value"` - MatrixCondition string `yaml:"matrix-condition"` + Title string `yaml:"title"` + Value string `yaml:"value"` } type ExpectedDashboard struct { @@ -27,9 +26,8 @@ type ExpectedDashboard struct { } type ExpectedMetrics struct { - PromQL string `yaml:"promql"` - Value string `yaml:"value"` - MatrixCondition string `yaml:"matrix-condition"` + PromQL string `yaml:"promql"` + Value string `yaml:"value"` } type ExpectedSpan struct { @@ -46,13 +44,11 @@ type ExpectedLogs struct { Attributes map[string]string `yaml:"attributes"` AttributeRegexp map[string]string `yaml:"attribute-regexp"` NoExtraAttributes bool `yaml:"no-extra-attributes"` - MatrixCondition string `yaml:"matrix-condition"` } type ExpectedTraces struct { - TraceQL string `yaml:"traceql"` - Spans []ExpectedSpan `yaml:"spans"` - MatrixCondition string `yaml:"matrix-condition"` + TraceQL string `yaml:"traceql"` + Spans []ExpectedSpan `yaml:"spans"` } type CustomCheck struct { @@ -67,11 +63,6 @@ type Expected struct { CustomChecks []CustomCheck `yaml:"custom-checks"` } -type Matrix struct { - Name string `yaml:"name"` - DockerCompose *DockerCompose `yaml:"docker-compose"` -} - type DockerCompose struct { Generator string `yaml:"generator"` // deprecated: only used by beyla Files []string `yaml:"files"` @@ -87,7 +78,6 @@ type TestCaseDefinition struct { Include []string `yaml:"include"` DockerCompose *DockerCompose `yaml:"docker-compose"` Kubernetes *kubernetes.Kubernetes `yaml:"kubernetes"` - Matrix []Matrix `yaml:"matrix"` Input []Input `yaml:"input"` Interval time.Duration `yaml:"interval"` Expected Expected `yaml:"expected"` @@ -101,7 +91,6 @@ func (d *TestCaseDefinition) Merge(other TestCaseDefinition) { d.Expected.Metrics = append(d.Expected.Metrics, other.Expected.Metrics...) d.Expected.Dashboards = append(d.Expected.Dashboards, other.Expected.Dashboards...) d.Expected.CustomChecks = append(d.Expected.CustomChecks, other.Expected.CustomChecks...) - d.Matrix = append(d.Matrix, other.Matrix...) if d.DockerCompose == nil { d.DockerCompose = other.DockerCompose } @@ -122,14 +111,13 @@ type PortConfig struct { } type TestCase struct { - Name string - MatrixTestCaseName string - Dir string - OutputDir string - Definition TestCaseDefinition - PortConfig *PortConfig - Dashboard *TestDashboard - Timeout time.Duration + Name string + Dir string + OutputDir string + Definition TestCaseDefinition + PortConfig *PortConfig + Dashboard *TestDashboard + Timeout time.Duration } type QueryLogger struct { diff --git a/yaml/runner.go b/yaml/runner.go index f9d40c7..58670ba 100644 --- a/yaml/runner.go +++ b/yaml/runner.go @@ -13,8 +13,6 @@ import ( "strconv" "time" - "github.com/grafana/regexp" - "github.com/grafana/oats/testhelpers/compose" "github.com/grafana/oats/testhelpers/requests" . "github.com/onsi/ginkgo/v2" @@ -79,47 +77,39 @@ func RunTestCase(c *TestCase) { // (depending on OTEL_METRIC_EXPORT_INTERVAL). for _, log := range expected.Logs { l := log - if r.MatchesMatrixCondition(l.MatrixCondition, l.LogQL) { - It(fmt.Sprintf("should have '%s' in loki", l.LogQL), func() { - r.eventually(func() { - AssertLoki(r, l) - }) + It(fmt.Sprintf("should have '%s' in loki", l.LogQL), func() { + r.eventually(func() { + AssertLoki(r, l) }) - } + }) } for _, trace := range expected.Traces { t := trace - if r.MatchesMatrixCondition(t.MatrixCondition, t.TraceQL) { - It(fmt.Sprintf("should have '%s' in tempo", t.TraceQL), func() { - r.eventually(func() { - AssertTempo(r, t) - }) + It(fmt.Sprintf("should have '%s' in tempo", t.TraceQL), func() { + r.eventually(func() { + AssertTempo(r, t) }) - } + }) } for _, dashboard := range expected.Dashboards { dashboardAssert := NewDashboardAssert(dashboard) for i, panel := range dashboard.Panels { iCopy := i p := panel - if r.MatchesMatrixCondition(p.MatrixCondition, p.Title) { - It(fmt.Sprintf("dashboard panel '%s'", p.Title), func() { - r.eventually(func() { - dashboardAssert.AssertDashboard(r, iCopy) - }) + It(fmt.Sprintf("dashboard panel '%s'", p.Title), func() { + r.eventually(func() { + dashboardAssert.AssertDashboard(r, iCopy) }) - } + }) } } for _, metric := range expected.Metrics { m := metric - if r.MatchesMatrixCondition(m.MatrixCondition, m.PromQL) { - It(fmt.Sprintf("should have '%s' in prometheus", m.PromQL), func() { - r.eventually(func() { - AssertProm(r, m.PromQL, m.Value) - }) + It(fmt.Sprintf("should have '%s' in prometheus", m.PromQL), func() { + r.eventually(func() { + AssertProm(r, m.PromQL, m.Value) }) - } + }) } for _, customCheck := range expected.CustomChecks { c := customCheck @@ -232,19 +222,3 @@ func (r *runner) eventually(asserter func()) { a() } } - -func (r *runner) MatchesMatrixCondition(matrixCondition string, subject string) bool { - if matrixCondition == "" { - return true - } - name := r.testCase.MatrixTestCaseName - if name == "" { - r.queryLogger.LogQueryResult("matrix condition %v ignored we're not in a matrix test\n", matrixCondition) - return true - } - if regexp.MustCompile(matrixCondition).MatchString(name) { - return true - } - fmt.Printf("matrix condition not matched - ignoring assertion: %v/%v/%v\n", r.testCase.Name, name, subject) - return false -} diff --git a/yaml/testcase.go b/yaml/testcase.go index 0647fc9..7734193 100644 --- a/yaml/testcase.go +++ b/yaml/testcase.go @@ -2,7 +2,6 @@ package yaml import ( "errors" - "fmt" "os" "path/filepath" "regexp" @@ -78,17 +77,6 @@ func collectTestCases(base string, duration time.Duration, evaluateIgnoreFile bo if err != nil { return err } - if testCase.Definition.Matrix != nil { - for _, matrix := range testCase.Definition.Matrix { - newCase := testCase - newCase.Definition = testCase.Definition - newCase.Definition.DockerCompose = matrix.DockerCompose - newCase.Name = fmt.Sprintf("%s-%s", testCase.Name, matrix.Name) - newCase.MatrixTestCaseName = matrix.Name - cases = append(cases, &newCase) - } - return nil - } cases = append(cases, &testCase) return nil }) From 22b08e6d704bd549c80ff134ebacfeaf968782de Mon Sep 17 00:00:00 2001 From: Gregor Zeitlinger Date: Tue, 17 Dec 2024 14:46:05 +0100 Subject: [PATCH 05/10] new readme --- README.md | 72 ++++++++++++++++++++++++++++++++++++++++--------------- 1 file changed, 53 insertions(+), 19 deletions(-) diff --git a/README.md b/README.md index 2b0d916..bffab3c 100644 --- a/README.md +++ b/README.md @@ -72,7 +72,7 @@ docker-compose: file: ../docker-compose.yaml input: - url: http://localhost:8080/stock -interval: 500ms +interval: 500ms # interval between requests to the input URL expected: traces: - traceql: '{ name =~ "SELECT .*product"}' @@ -87,7 +87,7 @@ expected: metrics: - promql: 'db_client_connections_max{pool_name="HikariPool-1"}' value: "== 10" - dashboards: + dashboards: # Grafana dashboards - path: ../jdbc-dashboard.json panels: - title: Connection pool waiting requests @@ -96,8 +96,57 @@ expected: value: "> 0" ``` -You have to provide the root path of the directory where your test cases are located to ginkgo -via the environment variable `TESTCASE_BASE_PATH`. +### Query traces + +Each entry in the `traces` array is a test case for traces. + +```yaml +expected: + traces: + - traceql: '{ name =~ "SELECT .*product"}' + spans: + - name: 'regex:SELECT .*' # regex match + attributes: + db.system: h2 + allow-duplicates: true # allow multiple spans with the same attributes +``` + +### Query logs + +Each entry in the `logs` array is a test case for logs. + +```yaml +expected: + logs: + - logql: '{exporter = "OTLP"}' + contains: + - 'hello LGTM' + attributes: + service_name: rolldice + attribute-regexp: + container_id: ".*" + no-extra-attributes: true # fail if there are extra attributes + - logql: '{service_name="rolldice"} |~ `Anonymous player is rolling the dice.*`' + equals: 'Anonymous player is rolling the dice' + - logql: '{service_name="rolldice"} |~ `Anonymous player is rolling the dice.*`' + regexp: 'Anonymous player is .*' +``` + +### Query metrics + +```yaml +expected: + metrics: + - promql: 'db_client_connections_max{pool_name="HikariPool-1"}' + value: "== 10" + dashboards: # Useful if you populate Grafana dashboards from JSON + - path: ../jdbc-dashboard.json + panels: + - title: Connection pool waiting requests + value: "== 0" + - title: Connection pool utilization + value: "> 0" +``` ## Docker Compose @@ -127,21 +176,6 @@ kubernetes: app-docker-port: 8080 ``` -## Matrix of test cases - -Matrix tests are useful to test different configurations of the same application, -e.g. with different settings of the otel collector or different flags in the application. - -```yaml -matrix: - - name: new - docker-compose: - - name: old-jvm-metrics - docker-compose: -input: - - path: /stock -``` - ## Debugging If you want to run a single test case, you can use the `--focus` option: From 9e855e1dffeb3919736677ac58a83da4d7c6c6c9 Mon Sep 17 00:00:00 2001 From: Gregor Zeitlinger Date: Tue, 17 Dec 2024 14:47:30 +0100 Subject: [PATCH 06/10] new readme --- README.md | 2 -- 1 file changed, 2 deletions(-) diff --git a/README.md b/README.md index bffab3c..2ea64ed 100644 --- a/README.md +++ b/README.md @@ -119,8 +119,6 @@ Each entry in the `logs` array is a test case for logs. expected: logs: - logql: '{exporter = "OTLP"}' - contains: - - 'hello LGTM' attributes: service_name: rolldice attribute-regexp: From 4559f929dedc0be6c526715dc7880a8a536755d3 Mon Sep 17 00:00:00 2001 From: Gregor Zeitlinger Date: Tue, 17 Dec 2024 14:47:57 +0100 Subject: [PATCH 07/10] new readme --- README.md | 5 ++--- 1 file changed, 2 insertions(+), 3 deletions(-) diff --git a/README.md b/README.md index 2ea64ed..a9930af 100644 --- a/README.md +++ b/README.md @@ -118,14 +118,13 @@ Each entry in the `logs` array is a test case for logs. ```yaml expected: logs: - - logql: '{exporter = "OTLP"}' + - logql: '{service_name="rolldice"} |~ `Anonymous player is rolling the dice.*`' + equals: 'Anonymous player is rolling the dice' attributes: service_name: rolldice attribute-regexp: container_id: ".*" no-extra-attributes: true # fail if there are extra attributes - - logql: '{service_name="rolldice"} |~ `Anonymous player is rolling the dice.*`' - equals: 'Anonymous player is rolling the dice' - logql: '{service_name="rolldice"} |~ `Anonymous player is rolling the dice.*`' regexp: 'Anonymous player is .*' ``` From 7734375de66b25e1100d10a3b7cafef29c81cf7a Mon Sep 17 00:00:00 2001 From: Matthew Hensley Date: Tue, 17 Dec 2024 09:48:34 -0500 Subject: [PATCH 08/10] OATS -> OATs --- README.md | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/README.md b/README.md index a9930af..fe6a527 100644 --- a/README.md +++ b/README.md @@ -7,7 +7,7 @@ OpenTelemetry Acceptance Tests (OATs), or OATs for short, is a test framework fo - Full round-trip testing: from the application to the observability stack - Data is stored in the LGTM stack ([Loki], [Grafana], [Tempo], [Prometheus], [OpenTelemetry Collector]) - Data is queried using LogQL, PromQL, and TraceQL - - All data is sent to the observability stack via OTLP - so OATS can also be used with other observability stacks + - All data is sent to the observability stack via OTLP - so OATs can also be used with other observability stacks - End-to-end testing - Docker Compose with the [docker-otel-lgtm] image - Kubernetes with the [docker-otel-lgtm] and [k3d] @@ -43,7 +43,7 @@ Under the hood, OATs uses [Ginkgo] and [Gomega] to run the tests. ``` 4. Create `oats.yaml` with the test cases ```yaml - # OATS is an acceptance testing framework for OpenTelemetry - https://github.com/grafana/oats + # OATs is an acceptance testing framework for OpenTelemetry - https://github.com/grafana/oats docker-compose: files: - ./docker-compose.yaml From 9f7fb118cf6a366465f18617cbc89ca11aef8fe2 Mon Sep 17 00:00:00 2001 From: Matthew Hensley Date: Tue, 17 Dec 2024 09:54:39 -0500 Subject: [PATCH 09/10] proper nouns --- README.md | 10 +++++----- 1 file changed, 5 insertions(+), 5 deletions(-) diff --git a/README.md b/README.md index fe6a527..04485c6 100644 --- a/README.md +++ b/README.md @@ -152,16 +152,16 @@ The files typically defines the instrumented application you want to test and op e.g. a database server to send requests to. You don't need (and should have) to define the observability stack (e.g. prometheus, grafana, etc.), because this is provided by the test framework (and may test different versions of the observability stack, -e.g. otel collector and grafana agent). +e.g. OTel Collector and Grafana Alloy). This docker-compose file is relative to the `oats.yaml` file. ## Kubernetes -A local kubernetes cluster can be used to test the application in a kubernetes environment rather than in docker-compose. +A local Kubernetes cluster can be used to test the application in a Kubernetes environment rather than in docker-compose. This is useful to test the application in a more realistic environment - and when you want to test Kubernetes specific features. -Describes the kubernetes manifest(s) to use for the test. +Describes the Kubernetes manifest(s) to use for the test. ```yaml kubernetes: @@ -182,13 +182,13 @@ TESTCASE_BASE_PATH=/path/to/project ginkgo -v --focus="jdbc" ``` You can increase the timeout, which is useful if you want to inspect the telemetry data manually -in grafana at http://localhost:3000 +in Grafana at http://localhost:3000 ```sh TESTCASE_TIMEOUT=1h TESTCASE_BASE_PATH=/path/to/project ginkgo -v ``` -You can keep the container running without executing the tests - which is useful to debug in grafana manually: +You can keep the container running without executing the tests - which is useful to debug in Grafana manually: ```sh TESTCASE_MANUAL_DEBUG=true TESTCASE_BASE_PATH=/path/to/project ginkgo -v From 73f119ff5ac2926d1296c8dabaabcc0eb8275cd7 Mon Sep 17 00:00:00 2001 From: Matthew Hensley Date: Tue, 17 Dec 2024 09:55:00 -0500 Subject: [PATCH 10/10] edits --- README.md | 6 +++--- 1 file changed, 3 insertions(+), 3 deletions(-) diff --git a/README.md b/README.md index 04485c6..a737759 100644 --- a/README.md +++ b/README.md @@ -59,7 +59,7 @@ Under the hood, OATs uses [Ginkgo] and [Gomega] to run the tests. ## Test Case Syntax > You can use any file name that matches `oats*.yaml` (e.g. `oats-test.yaml`), that doesn't end in `-template.yaml`. -> `oats-template.yaml` is reserved for template files, which are used in the "include" section. +> `oats-template.yaml` is reserved for template files, which are used in the `include` section. The syntax is a bit similar to https://github.com/kubeshop/tracetest @@ -148,9 +148,9 @@ expected: ## Docker Compose Describes the docker-compose file(s) to use for the test. -The files typically defines the instrumented application you want to test and optionally some dependencies, +The files typically define the instrumented application you want to test and optionally some dependencies, e.g. a database server to send requests to. -You don't need (and should have) to define the observability stack (e.g. prometheus, grafana, etc.), +You don't need (and shouldn't have) to define the observability stack (e.g. Prometheus, Grafana, etc.), because this is provided by the test framework (and may test different versions of the observability stack, e.g. OTel Collector and Grafana Alloy).