trace_agent_integration_tests: tentatively fix flakiness

The `trace-agent` integration tests have been repeatedly failing on
timeouts, for instance:
- https://gitlab.ddbuild.io/DataDog/datadog-agent/-/jobs/1261246921
- https://gitlab.ddbuild.io/DataDog/datadog-agent/-/jobs/1261237699
- https://gitlab.ddbuild.io/DataDog/datadog-agent/-/jobs/1261226844

There are at least 2 culprits:
1. rebuilding the `trace-agent` binary from scratch for every test run.
   When running with the race detector (`-race` flag), each build takes
   time,
2. module cache contention when building, see #43207 for why.

This change implements the TODO at agent.go:94 by:
- ceating a temporary directory in `TestMain` (`BuildCachedBinaries`),
- building `trace-agent` and `secret-script` binaries once in that
  directory, with `GOPRIVATE=*` and `GOPROXY=off` to prevent concurrent
  module cache access,
- creating symlinks to the cached binaries in per-test subdirectories,
- cleaning up the entire temp directory tree after all tests complete.

This is expected to reduce test execution time, especially with race
detection enabled, and hopefully preventtimeouts in `TestTraces`.

Author: rdesgroppes

cmd/trace-agent/test/agent.go

@@ -45,6 +45,8 @@ var ErrNotInstalled = errors.New("agent: trace-agent not found in $PATH")
 // SecretBackendBinary secret binary name
 var SecretBackendBinary = "secret-script.test"
 
+var tmpDir string
+
 type grpcServer struct {
 	pb.UnimplementedAgentSecureServer
 }
@@ -63,41 +65,74 @@ type agentRunner struct {
 	authToken            string
 }
 
-func newAgentRunner(ddAddr string, verbose bool, buildSecretBackend bool) (*agentRunner, error) {
-	bindir, err := os.MkdirTemp("", "trace-agent-integration-tests")
+// BuildCachedBinaries builds the trace-agent and secret-script binaries once and caches them.
+// Returns a cleanup function that should be called to remove the temporary directory.
+// This should be called once from TestMain before running tests.
+func BuildCachedBinaries(verbose bool) (func(), error) {
+	var err error
+	tmpDir, err = os.MkdirTemp("", "trace-agent-integration-tests")
 	if err != nil {
 		return nil, err
 	}
-	binpath := filepath.Join(bindir, "trace-agent")
+	cleanup := func() {
+		_ = os.RemoveAll(tmpDir)
+	}
+	binpath := filepath.Join(tmpDir, "trace-agent")
 	if verbose {
 		log.Printf("agent: installing in %s...", binpath)
 	}
-	// TODO(gbbr): find a way to re-use the same binary within a whole run
-	// instead of creating new ones on each test creating a new runner.
 	o, err := exec.Command("go", "build", "-tags", "otlp", "-o", binpath, "github.com/DataDog/datadog-agent/cmd/trace-agent").CombinedOutput()
 	if err != nil {
 		if verbose {
 			log.Printf("error installing trace-agent: %v", err)
 			log.Print(string(o))
 		}
+		cleanup()
 		return nil, ErrNotInstalled
 	}
 
-	if buildSecretBackend {
-		binSecrets := filepath.Join(bindir, SecretBackendBinary)
-		o, err := exec.Command("go", "build", "-o", binSecrets, "./testdata/secretscript.go").CombinedOutput()
+	binSecrets := filepath.Join(tmpDir, SecretBackendBinary)
+	o, err = exec.Command("go", "build", "-o", binSecrets, "./testdata/secretscript.go").CombinedOutput()
+	if err != nil {
+		if verbose {
+			log.Printf("error installing secret-script: %v", err)
+			log.Print(string(o))
+		}
+		cleanup()
+		return nil, ErrNotInstalled
+	}
 
-		if err != nil {
-			if verbose {
-				log.Printf("error installing secret-script: %v", err)
-				log.Print(string(o))
-			}
-			return nil, ErrNotInstalled
+	if err := os.Chmod(binSecrets, 0700); err != nil {
+		if verbose {
+			log.Printf("error changing permissions secret-script: %v", err)
+		}
+		cleanup()
+		return nil, ErrNotInstalled
+	}
+	return cleanup, nil
+}
+
+func newAgentRunner(ddAddr string, verbose bool, buildSecretBackend bool) (*agentRunner, error) {
+	bindir, err := os.MkdirTemp(tmpDir, "runner-")
+	if err != nil {
+		return nil, err
+	}
+	binpath := filepath.Join(bindir, "trace-agent")
+	if verbose {
+		log.Printf("agent: installing in %s...", binpath)
+	}
+	if err := os.Symlink(filepath.Join(tmpDir, "trace-agent"), binpath); err != nil {
+		if verbose {
+			log.Printf("error installing trace-agent: %v", err)
 		}
+		return nil, ErrNotInstalled
+	}
 
-		if err := os.Chmod(binSecrets, 0700); err != nil {
+	if buildSecretBackend {
+		binSecrets := filepath.Join(bindir, SecretBackendBinary)
+		if err := os.Symlink(filepath.Join(tmpDir, SecretBackendBinary), binSecrets); err != nil {
 			if verbose {
-				log.Printf("error changing permissions secret-script: %v", err)
+				log.Printf("error installing secret-script: %v", err)
 			}
 			return nil, ErrNotInstalled
 		}

cmd/trace-agent/test/testsuite/hostname_test.go

@@ -6,6 +6,7 @@
 package testsuite
 
 import (
+	"flag"
 	"log"
 	"os"
 	"testing"
@@ -21,6 +22,16 @@ func TestMain(m *testing.M) {
 		log.Println("--- SKIP: to run tests in this package, set the INTEGRATION environment variable")
 		os.Exit(0)
 	}
+	// Set environment variables to prevent go build commands from accessing
+	// the module cache concurrently, which can cause timeouts.
+	os.Setenv("GOPRIVATE", "*")
+	os.Setenv("GOPROXY", "off")
+	flag.Parse()
+	cleanup, err := test.BuildCachedBinaries(testing.Verbose())
+	if err != nil {
+		log.Fatalf("failed to build cached binaries: %v", err)
+	}
+	defer cleanup()
 	os.Exit(m.Run())
 }