Add transformation test make target and docs (#1722)

Signed-off-by: braydonk <[email protected]>

.gitignore

@@ -1,6 +1,14 @@
+# Ignore a file actually called Makefile. The presence of this file interferes
+# with our Linux package building processes. To make a local symlink called
+# "Makefile" for development convenience, use this command:
+# make -f tasks.mak makefile_symlink
 Makefile
 
 # Ignore generated credentials from google-github-actions/auth
 gha-creds-*.json
 
-.vscode/launch.json
+# This is a folder used to hold local builds of fluent-bit and otelopscol
+# when running transformation tests locally.
+dist/
+
+.vscode/launch.json

builds/agent_wrapper.sh

@@ -1,4 +1,18 @@
 #!/bin/bash
+# Copyright 2024 Google LLC
+#
+# Licensed under the Apache License, Version 2.0 (the "License");
+# you may not use this file except in compliance with the License.
+# You may obtain a copy of the License at
+#
+#     http://www.apache.org/licenses/LICENSE-2.0
+#
+# Unless required by applicable law or agreed to in writing, software
+# distributed under the License is distributed on an "AS IS" BASIS,
+# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+# See the License for the specific language governing permissions and
+# limitations under the License.
+
 set -x -e
 DESTDIR=$1
 mkdir -p "$DESTDIR/opt/google-cloud-ops-agent/libexec"

builds/fluent_bit.sh

@@ -1,4 +1,18 @@
 #!/bin/bash
+# Copyright 2024 Google LLC
+#
+# Licensed under the Apache License, Version 2.0 (the "License");
+# you may not use this file except in compliance with the License.
+# You may obtain a copy of the License at
+#
+#     http://www.apache.org/licenses/LICENSE-2.0
+#
+# Unless required by applicable law or agreed to in writing, software
+# distributed under the License is distributed on an "AS IS" BASIS,
+# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+# See the License for the specific language governing permissions and
+# limitations under the License.
+
 set -x -e
 DESTDIR=$1
 mkdir -p $DESTDIR

builds/ops_agent_diagnostics.sh

@@ -1,4 +1,18 @@
 #!/bin/bash
+# Copyright 2024 Google LLC
+#
+# Licensed under the Apache License, Version 2.0 (the "License");
+# you may not use this file except in compliance with the License.
+# You may obtain a copy of the License at
+#
+#     http://www.apache.org/licenses/LICENSE-2.0
+#
+# Unless required by applicable law or agreed to in writing, software
+# distributed under the License is distributed on an "AS IS" BASIS,
+# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+# See the License for the specific language governing permissions and
+# limitations under the License.
+
 set -x -e
 DESTDIR=$1
 mkdir -p "$DESTDIR/opt/google-cloud-ops-agent/libexec"

builds/otel.sh

@@ -1,26 +1,51 @@
 #!/bin/bash
+# Copyright 2024 Google LLC
+#
+# Licensed under the Apache License, Version 2.0 (the "License");
+# you may not use this file except in compliance with the License.
+# You may obtain a copy of the License at
+#
+#     http://www.apache.org/licenses/LICENSE-2.0
+#
+# Unless required by applicable law or agreed to in writing, software
+# distributed under the License is distributed on an "AS IS" BASIS,
+# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+# See the License for the specific language governing permissions and
+# limitations under the License.
+
 set -x -e
-DESTDIR=$1
+
 otel_dir=/opt/google-cloud-ops-agent/subagents/opentelemetry-collector
-DESTDIR="${DESTDIR}${otel_dir}"
-
-cd submodules/opentelemetry-java-contrib
-mkdir -p "$DESTDIR"
-./gradlew --no-daemon -Djdk.lang.Process.launchMechanism=vfork :jmx-metrics:build
-cp jmx-metrics/build/libs/opentelemetry-jmx-metrics-*-alpha-SNAPSHOT.jar "$DESTDIR/opentelemetry-java-contrib-jmx-metrics.jar"
-
-# Rename LICENSE file because it causes issues with file hash consistency due to an unknown
-# issue with the debuild/rpmbuild processes. Something is unzipping the jar in a case-insensitive
-# environment and having a conflict between the LICENSE file and license/ directory, leading to a changed jar file
-mkdir ./META-INF
-unzip -j "$DESTDIR/opentelemetry-java-contrib-jmx-metrics.jar" "META-INF/LICENSE" -d ./META-INF
-zip -d "$DESTDIR/opentelemetry-java-contrib-jmx-metrics.jar" "META-INF/LICENSE"
-mv ./META-INF/LICENSE ./META-INF/LICENSE.renamed
-zip -u "$DESTDIR/opentelemetry-java-contrib-jmx-metrics.jar" "META-INF/LICENSE.renamed"
-
-cd ../opentelemetry-operations-collector
-# Using array assignment to drop the filename from the sha256sum output
-JAR_SHA_256=($(sha256sum "$DESTDIR/opentelemetry-java-contrib-jmx-metrics.jar"))
-go build -tags=gpu -buildvcs=false -o "$DESTDIR/otelopscol" \
--ldflags "-X github.com/open-telemetry/opentelemetry-collector-contrib/receiver/jmxreceiver.MetricsGathererHash=$JAR_SHA_256" \
-./cmd/otelopscol
+DESTDIR="${1}${otel_dir}"
+
+mkdir -p $DESTDIR
+
+LDFLAGS=""
+
+# If this is not a local build, then we build otel java and add it to
+# the ldflags.
+if [ "$SKIP_OTEL_JAVA" != "true" ]; then
+    cd submodules/opentelemetry-java-contrib
+
+    mkdir -p "$DESTDIR"
+    ./gradlew --no-daemon -Djdk.lang.Process.launchMechanism=vfork :jmx-metrics:build
+    cp jmx-metrics/build/libs/opentelemetry-jmx-metrics-*-alpha-SNAPSHOT.jar "$DESTDIR/opentelemetry-java-contrib-jmx-metrics.jar"
+
+    # Rename LICENSE file because it causes issues with file hash consistency due to an unknown
+    # issue with the debuild/rpmbuild processes. Something is unzipping the jar in a case-insensitive
+    # environment and having a conflict between the LICENSE file and license/ directory, leading to a changed jar file
+    mkdir ./META-INF
+    unzip -j "$DESTDIR/opentelemetry-java-contrib-jmx-metrics.jar" "META-INF/LICENSE" -d ./META-INF
+    zip -d "$DESTDIR/opentelemetry-java-contrib-jmx-metrics.jar" "META-INF/LICENSE"
+    mv ./META-INF/LICENSE ./META-INF/LICENSE.renamed
+    zip -u "$DESTDIR/opentelemetry-java-contrib-jmx-metrics.jar" "META-INF/LICENSE.renamed"
+
+    # Using array assignment to drop the filename from the sha256sum output
+    JAR_SHA_256=($(sha256sum "$DESTDIR/opentelemetry-java-contrib-jmx-metrics.jar"))
+    LDFLAGS="-X github.com/open-telemetry/opentelemetry-collector-contrib/receiver/jmxreceiver.MetricsGathererHash=$JAR_SHA_256"
+
+    cd ../..
+fi
+
+cd submodules/opentelemetry-operations-collector
+go build -tags=gpu -buildvcs=false -o "$DESTDIR/otelopscol" -ldflags "$LDFLAGS" ./cmd/otelopscol

builds/systemd.sh

@@ -1,4 +1,18 @@
 #!/bin/bash
+# Copyright 2024 Google LLC
+#
+# Licensed under the Apache License, Version 2.0 (the "License");
+# you may not use this file except in compliance with the License.
+# You may obtain a copy of the License at
+#
+#     http://www.apache.org/licenses/LICENSE-2.0
+#
+# Unless required by applicable law or agreed to in writing, software
+# distributed under the License is distributed on an "AS IS" BASIS,
+# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+# See the License for the specific language governing permissions and
+# limitations under the License.
+
 set -x -e
 DESTDIR=$1
 

cloudbuild/new-distro-detector/run.sh

@@ -1,4 +1,18 @@
 #!/bin/bash
+# Copyright 2024 Google LLC
+#
+# Licensed under the Apache License, Version 2.0 (the "License");
+# you may not use this file except in compliance with the License.
+# You may obtain a copy of the License at
+#
+#     http://www.apache.org/licenses/LICENSE-2.0
+#
+# Unless required by applicable law or agreed to in writing, software
+# distributed under the License is distributed on an "AS IS" BASIS,
+# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+# See the License for the specific language governing permissions and
+# limitations under the License.
+
 
 # See go/sdi-new-distro-detector for details on how this script is being used.
 # A short summary is that it is run periodically by GCB to file bugs when

cmd/agent_wrapper/main_test.go

@@ -1,3 +1,17 @@
+// Copyright 2024 Google LLC
+//
+// Licensed under the Apache License, Version 2.0 (the "License");
+// you may not use this file except in compliance with the License.
+// You may obtain a copy of the License at
+//
+//     http://www.apache.org/licenses/LICENSE-2.0
+//
+// Unless required by applicable law or agreed to in writing, software
+// distributed under the License is distributed on an "AS IS" BASIS,
+// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+// See the License for the specific language governing permissions and
+// limitations under the License.
+
 package main
 
 import (

confgenerator/feature_tracking_internal_test.go

@@ -1,3 +1,17 @@
+// Copyright 2024 Google LLC
+//
+// Licensed under the Apache License, Version 2.0 (the "License");
+// you may not use this file except in compliance with the License.
+// You may obtain a copy of the License at
+//
+//     http://www.apache.org/licenses/LICENSE-2.0
+//
+// Unless required by applicable law or agreed to in writing, software
+// distributed under the License is distributed on an "AS IS" BASIS,
+// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+// See the License for the specific language governing permissions and
+// limitations under the License.
+
 package confgenerator
 
 import (

dev-docs/makefile.md

@@ -6,14 +6,18 @@ The Makefile `tasks.mak` is a Makefile for the project that is used to quickly p
 
 We are unable to use the actual `Makefile` name in this repo; because we build the Ops Agent into a deb package, `debhelper` tries to autodetect `make` as the build system in most of its `dh_auto` steps. We do not want `debhelper` to pick up the Makfile and try picking up different targets to build and test it.
 
+## Disclaimer
+
+`tasks.mak` is provided without any guarantees or warranty on its targets. It is meant purely for developer convenience, and it is advised not to make any dependency on the targets since they are subject to change at any time.
+
 ## Usage
 
 For convenience it helps to make a symlink from `Makefile` to `tasks.mak` . There is a target to do this, to run it:
 ```
 make -f tasks.mak makefile_symlink
 ```
 
-Now you can call the targets simply with the `make` command without needing to specify the `tasks.mak` file. The file `Makefile` is gitignored, since we don't want to push it to the repo for the reasons stated in the previous section. 
+Now you can call the targets simply with the `make` command without needing to specify the `tasks.mak` file. The file `Makefile` is gitignored, since we don't want to push it to the repo for the reasons stated in the previous section.
 
 To run something from the Makefile, you will need to give `make` the `-f` (file) flag with a path to `tasks.mak`. For example, to run the `test` target:
 ```
@@ -35,8 +39,4 @@ A new target should be added to `tasks.mak` if:
 * There's a new package with Unit Tests
 * There is something that developers may need to run often, especially if it's a complicated command
 * There are new tasks to run in CI
-* There is a new development tool to be installed and used 
-
-## Disclaimer
-
-`tasks.mak` is provided without any guarantees or warranty on its targets. It is meant purely for developer convenience, and it is advised not to make any dependency on the targets since they are subject to change at any time.
+* There is a new development tool to be installed and used

dev-docs/transformation-tests.md

@@ -0,0 +1,51 @@
+# Transformation Tests
+
+The transformation tests are tests that will run Fluent Bit and OpenTelemetry Operations Collector (otelopscol) directly in your environment and verify the results of various logging operations by reading their output from stdout and stderr. This is to check the correctness of log parsing expectations in a more lightweight way than full integration tests.
+
+## Makefile Usage
+
+NOTE: It is assumed that you have followed the [makefile setup instructions](./makefile.md#usage).
+
+For your convenience, there is a Makefile target to run transformation tests:
+```bash
+make transformation_test
+```
+You can also update the transformation test goldens:
+```bash
+make transformation_test_update
+```
+These targets are part of the `precommit` and `precommit_update` targets respectively.
+
+If you are on a new branch that updates Fluent Bit or otelopscol and are locally verifying the transformation tests, you can force the agents to rebuild by deleting the `dist` folder before running `make transfomration_test`; the target will rebuild the agents if they are not present.
+
+Read on if you are interested in how the transformation tests actually work.
+
+## Usage
+
+The transformation tests take paths to Fluent Bit and otelopscol binaries to execute directly and compare expected output. To run the tests, first you will need to build both binaries. Use the build scripts to build each agent and install them into the `dist` folder:
+```bash
+sh ./builds/fluent_bit.sh $(pwd)/dist
+SKIP_OTEL_JAVA=true sh ./builds/otel.sh $(pwd)/dist
+```
+After running each script, you will have a `dist` folder that contains both agents within the full install directory. (Note that these instructions will not build OTel Java; if that is needed, do not pass `SKIP_OTEL_JAVA`).
+
+The paths to each can passed into the test using environment variables or flags.
+```bash
+go test ./transformation_test \
+-flb=${PWD}/dist/opt/google-cloud-ops-agent/subagents/fluent-bit/bin/fluent-bit \
+-otelopscol=${PWD}/dist/opt/google-cloud-ops-agent/subagents/opentelemetry-collector/otelopscol
+```
+or
+```bash
+FLB=${PWD}/dist/opt/google-cloud-ops-agent/subagents/fluent-bit/bin/fluent-bit \
+OTELOPSCOL=${PWD}/dist/opt/google-cloud-ops-agent/subagents/opentelemetry-collector/otelopscol \
+go test ./transformation_test
+```
+
+If you have updated the fluent-bit or otelopscol submodules and need to update the golden testdata, you can add the `-update` flag to the test.
+```bash
+FLB=${PWD}/dist/opt/google-cloud-ops-agent/subagents/fluent-bit/bin/fluent-bit \
+OTELOPSCOL=${PWD}/dist/opt/google-cloud-ops-agent/subagents/opentelemetry-collector/otelopscol \
+go test ./transformation_test -update
+```
+(Note: the `-update` flag belongs to the test, not to `go test`. The flag must go after the specified package, not before.)

dockerfiles/compile_test.go

@@ -1,3 +1,17 @@
+// Copyright 2024 Google LLC
+//
+// Licensed under the Apache License, Version 2.0 (the "License");
+// you may not use this file except in compliance with the License.
+// You may obtain a copy of the License at
+//
+//     http://www.apache.org/licenses/LICENSE-2.0
+//
+// Unless required by applicable law or agreed to in writing, software
+// distributed under the License is distributed on an "AS IS" BASIS,
+// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+// See the License for the specific language governing permissions and
+// limitations under the License.
+
 package main
 
 import (

integration_test/gce/zone_picker.go

@@ -1,3 +1,17 @@
+// Copyright 2024 Google LLC
+//
+// Licensed under the Apache License, Version 2.0 (the "License");
+// you may not use this file except in compliance with the License.
+// You may obtain a copy of the License at
+//
+//     http://www.apache.org/licenses/LICENSE-2.0
+//
+// Unless required by applicable law or agreed to in writing, software
+// distributed under the License is distributed on an "AS IS" BASIS,
+// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+// See the License for the specific language governing permissions and
+// limitations under the License.
+
 package gce
 
 import (

integration_test/mirror_content.sh

@@ -1,4 +1,18 @@
 #!/bin/bash
+# Copyright 2024 Google LLC
+#
+# Licensed under the Apache License, Version 2.0 (the "License");
+# you may not use this file except in compliance with the License.
+# You may obtain a copy of the License at
+#
+#     http://www.apache.org/licenses/LICENSE-2.0
+#
+# Unless required by applicable law or agreed to in writing, software
+# distributed under the License is distributed on an "AS IS" BASIS,
+# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+# See the License for the specific language governing permissions and
+# limitations under the License.
+
 
 USAGE="Usage: ./mirror_content.sh [URL]...
 

integration_test/third_party_apps_test/applications/couchbase/features.yaml

@@ -1,3 +1,17 @@
+# Copyright 2024 Google LLC
+#
+# Licensed under the Apache License, Version 2.0 (the "License");
+# you may not use this file except in compliance with the License.
+# You may obtain a copy of the License at
+#
+#     http://www.apache.org/licenses/LICENSE-2.0
+#
+# Unless required by applicable law or agreed to in writing, software
+# distributed under the License is distributed on an "AS IS" BASIS,
+# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+# See the License for the specific language governing permissions and
+# limitations under the License.
+
 features:
 - feature: receivers:couchbase
   module: metrics

integration_test/third_party_apps_test/applications/couchdb/features.yaml

@@ -1,3 +1,17 @@
+# Copyright 2024 Google LLC
+#
+# Licensed under the Apache License, Version 2.0 (the "License");
+# you may not use this file except in compliance with the License.
+# You may obtain a copy of the License at
+#
+#     http://www.apache.org/licenses/LICENSE-2.0
+#
+# Unless required by applicable law or agreed to in writing, software
+# distributed under the License is distributed on an "AS IS" BASIS,
+# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+# See the License for the specific language governing permissions and
+# limitations under the License.
+
 features:
 - feature: receivers:couchdb
   module: metrics