diff --git a/.github/workflows/ci.yml b/.github/workflows/ci.yml index a0dcebad..8c2cfaec 100644 --- a/.github/workflows/ci.yml +++ b/.github/workflows/ci.yml @@ -86,18 +86,60 @@ jobs: - name: Expose GitHub tokens for caching uses: crazy-max/ghaction-github-runtime@b3a9207c0e1ef41f4cf215303c976869d0c2c1c4 # v3.0.0 + - name: Setup jaeger + run: | + set -e + docker run -d --net=host --restart=always --name jaeger -e COLLECTOR_OTLP_ENABLED=true jaegertracing/all-in-one:1.62.0 + docker0_ip="$(ip -f inet addr show docker0 | grep -Po 'inet \K[\d.]+')" + echo "OTEL_EXPORTER_OTLP_ENDPOINT=http://${docker0_ip}:4318" >> "${GITHUB_ENV}" + echo "OTEL_SERVICE_NAME=dalec-integration-test" >> "${GITHUB_ENV}" + + tmp="$(mktemp)" + echo "Environment=\"OTEL_EXPORTER_OTLP_ENDPOINT=http://${docker0_ip}:4318\"" > "${tmp}" + sudo mkdir -p /etc/systemd/system/docker.service.d + sudo mkdir -p /etc/systemd/system/containerd.service.d + sudo cp "${tmp}" /etc/systemd/system/docker.service.d/otlp.conf + sudo cp "${tmp}" /etc/systemd/system/containerd.service.d/otlp.conf + + sudo systemctl daemon-reload + sudo systemctl restart containerd + sudo systemctl restart docker + # Tests currently require buildkit v0.12.0 or higher # The version of buildkit builtin to moby currently (v24) is too old # So we need to setup a custom builder. - name: Set up builder uses: docker/setup-buildx-action@c47758b77c9736f4b2ef4073d4d51994fabfe349 # v3.7.1 + with: + driver-opts: | + network=host + env.OTLP_EXPORTER_OPTELP_ENDPOINT=http://127.0.0.1:4318 + env.OTEL_SERVICE_NAME=buildkitd + - name: download deps run: go mod download + - name: Precache base images + run: go run ./cmd/ci-precache - name: Run integration tests - run: go test -v -json ./test | go run ./cmd/test2json2gha + run: go test -p 16 -v -json ./test | go run ./cmd/test2json2gha --slow 5s - name: dump logs if: failure() run: sudo journalctl -u docker + - name: Get traces + if: always() + run: | + set -ex + mkdir -p /tmp/reports + curl -sSLf localhost:16686/api/traces?service=${OTEL_SERVICE_NAME} > /tmp/reports/jaeger-tests.json + curl -sSLf localhost:16686/api/traces?service=containerd > /tmp/reports/jaeger-containerd.json + curl -sSLf localhost:16686/api/traces?service=buildkitd > /tmp/reports/jaeger-buildkitd.json + - name: Upload reports + if: always() + uses: actions/upload-artifact@v4 + with: + name: integration-test-reports + path: /tmp/reports/* + retention-days: 1 unit: diff --git a/cmd/ci-precache/main.go b/cmd/ci-precache/main.go new file mode 100644 index 00000000..1dfe04b2 --- /dev/null +++ b/cmd/ci-precache/main.go @@ -0,0 +1,219 @@ +package main + +import ( + "context" + "encoding/json" + "fmt" + "os" + "os/signal" + "syscall" + "time" + + "github.com/Azure/dalec/frontend/azlinux" + "github.com/Azure/dalec/frontend/jammy" + "github.com/Azure/dalec/frontend/windows" + "github.com/Azure/dalec/test/testenv" + "github.com/moby/buildkit/client" + "github.com/moby/buildkit/client/llb" + "github.com/moby/buildkit/frontend/dockerui" + gwclient "github.com/moby/buildkit/frontend/gateway/client" + "github.com/moby/buildkit/identity" + "github.com/moby/buildkit/solver/pb" + "github.com/moby/buildkit/util/bklog" + "github.com/moby/buildkit/util/progress/progressui" + "github.com/moby/patternmatcher/ignorefile" + "github.com/pkg/errors" + "github.com/tonistiigi/fsutil" + "golang.org/x/sync/errgroup" +) + +func main() { + ctx, done := signal.NotifyContext(context.Background(), os.Interrupt) + go func() { + <-ctx.Done() + // The context was cancelled due to interupt + // This _should_ trigger builds to cancel naturally and exit the program, + // but in some cases it may not (due to timing, bugs in buildkit, uninteruptable operations, etc.). + // Cancel our signal handler so the normal handler takes over from here. + // This allows subsequent interupts to use the default behavior (exit the program) + done() + + <-time.After(30 * time.Second) + fmt.Fprintln(os.Stderr, "Timeout waiting for builds to cancel after interupt") + os.Exit(int(syscall.SIGINT)) + }() + + bklog.G(ctx).Logger.SetOutput(os.Stderr) + + if err := run(ctx); err != nil { + fmt.Fprintln(os.Stderr, err) + os.Exit(1) + } +} + +func run(ctx context.Context) (retErr error) { + env := testenv.New() + + c, err := env.Buildkit(ctx) + if err != nil { + return err + } + defer c.Close() + + ch := make(chan *client.SolveStatus) + d, err := progressui.NewDisplay(os.Stderr, progressui.AutoMode) + if err != nil { + return err + } + + chErr := make(chan error, 1) + go func() { + warnings, err := d.UpdateFrom(ctx, ch) + for _, w := range warnings { + bklog.G(ctx).Warn(string(w.Short)) + } + chErr <- err + }() + + defer func() { + e := <-chErr + if retErr == nil { + retErr = e + } + }() + + localFS, err := fsutil.NewFS(".") + if err != nil { + return err + } + + ignoreF, err := localFS.Open(".dockerignore") + if err != nil { + return err + } + defer ignoreF.Close() + excludes, err := ignorefile.ReadAll(ignoreF) + if err != nil { + return err + } + + localFS, err = fsutil.NewFilterFS(localFS, &fsutil.FilterOpt{ + ExcludePatterns: excludes, + }) + if err != nil { + return err + } + + so := client.SolveOpt{ + LocalMounts: map[string]fsutil.FS{ + ".": localFS, + }, + } + _, err = c.Build(ctx, so, "", build, ch) + return err +} + +func build(ctx context.Context, client gwclient.Client) (*gwclient.Result, error) { + res, err := buildFrontend(ctx, client) + if err != nil { + return nil, err + } + + ref, err := res.SingleRef() + if err != nil { + return nil, err + } + + st, err := ref.ToState() + if err != nil { + return nil, err + } + + frontend, err := st.Marshal(ctx) + if err != nil { + return nil, err + } + + workers := []string{ + azlinux.Mariner2TargetKey, + azlinux.AzLinux3TargetKey, + windows.DefaultTargetKey, + jammy.DefaultTargetKey, + } + + eg, ctx := errgroup.WithContext(ctx) + + metaDt, err := json.Marshal(res.Metadata) + if err != nil { + return nil, err + } + + nullDef, err := nullDockerfile.Marshal(ctx) + if err != nil { + return nil, err + } + + id := identity.NewID() + for _, t := range workers { + eg.Go(func() error { + sr := gwclient.SolveRequest{ + Evaluate: true, + Frontend: "gateway.v0", + FrontendOpt: map[string]string{ + "source": id, + "target": t + "/worker", + "context:" + id: "input:" + id, + "input-metadata:" + id: string(metaDt), + }, + FrontendInputs: map[string]*pb.Definition{ + id: frontend.ToPB(), + dockerui.DefaultLocalNameDockerfile: nullDef.ToPB(), + }, + } + + _, err := client.Solve(ctx, sr) + if err != nil { + return fmt.Errorf("target %q: %w", t, err) + } + return nil + }) + } + if err := eg.Wait(); err != nil { + return nil, err + } + + return gwclient.NewResult(), nil +} + +var nullDockerfile = llb.Scratch().File( + llb.Mkfile("Dockerfile", 0o644, []byte("null")), +) + +func buildFrontend(ctx context.Context, client gwclient.Client) (*gwclient.Result, error) { + bctx, err := llb.Local(".").Marshal(ctx) + if err != nil { + return nil, err + } + + dt, err := os.ReadFile("Dockerfile") + if err != nil { + return nil, errors.Wrap(err, "error reading Dockerfile") + } + + dockerfile, err := llb.Scratch().File( + llb.Mkfile(dockerui.DefaultLocalNameDockerfile, 0o644, dt), + ).Marshal(ctx) + if err != nil { + return nil, err + } + + sr := gwclient.SolveRequest{ + Frontend: "dockerfile.v0", + FrontendInputs: map[string]*pb.Definition{ + dockerui.DefaultLocalNameContext: bctx.ToPB(), + dockerui.DefaultLocalNameDockerfile: dockerfile.ToPB(), + }, + } + + return client.Solve(ctx, sr) +} diff --git a/cmd/test2json2gha/event.go b/cmd/test2json2gha/event.go new file mode 100644 index 00000000..e52128cf --- /dev/null +++ b/cmd/test2json2gha/event.go @@ -0,0 +1,56 @@ +package main + +import ( + "os" + "time" + + "github.com/pkg/errors" +) + +// TestEvent is the go test2json event data structure we receive from `go test` +// This is defined in https://pkg.go.dev/cmd/test2json#hdr-Output_Format +type TestEvent struct { + Time time.Time + Action string + Package string + Test string + Elapsed float64 // seconds + Output string +} + +// TestResult is where we collect all the data about a test +type TestResult struct { + output *os.File + failed bool + pkg string + name string + elapsed float64 + skipped bool +} + +func (r *TestResult) Close() { + r.output.Close() +} + +func handlEvent(te *TestEvent, tr *TestResult) error { + if te.Output != "" { + _, err := tr.output.Write([]byte(te.Output)) + if err != nil { + return errors.Wrap(err, "error collecting test event output") + } + } + + tr.pkg = te.Package + tr.name = te.Test + if te.Elapsed > 0 { + tr.elapsed = te.Elapsed + } + + if te.Action == "fail" { + tr.failed = true + } + if te.Action == "skip" { + tr.skipped = true + } + return nil +} diff --git a/cmd/test2json2gha/main.go b/cmd/test2json2gha/main.go index b4679535..1badfba8 100644 --- a/cmd/test2json2gha/main.go +++ b/cmd/test2json2gha/main.go @@ -4,6 +4,7 @@ import ( "bufio" "bytes" "encoding/json" + "flag" "fmt" "io" "log/slog" @@ -15,6 +16,7 @@ import ( "time" "github.com/pkg/errors" + "github.com/vearutop/dynhist-go" ) func main() { @@ -23,6 +25,11 @@ func main() { panic(err) } + var slowThreshold time.Duration + flag.DurationVar(&slowThreshold, "slow", 500*time.Millisecond, "Threshold to mark test as slow") + + flag.Parse() + logger := slog.New(slog.NewTextHandler(os.Stderr, nil)) slog.SetDefault(logger) @@ -38,7 +45,7 @@ func main() { cleanup := func() { os.RemoveAll(tmp) } - anyFail, err := do(os.Stdin, os.Stdout, mod) + anyFail, err := do(os.Stdin, os.Stdout, mod, slowThreshold) if err != nil { fmt.Fprintf(os.Stderr, "%+v", err) cleanup() @@ -53,31 +60,7 @@ func main() { } } -// TestEvent is the go test2json event data structure we receive from `go test` -// This is defined in https://pkg.go.dev/cmd/test2json#hdr-Output_Format -type TestEvent struct { - Time time.Time - Action string - Package string - Test string - Elapsed float64 // seconds - Output string -} - -// TestResult is where we collect all the data about a test -type TestResult struct { - output *os.File - failed bool - pkg string - name string - elapsed float64 -} - -func (r *TestResult) Close() { - r.output.Close() -} - -func do(in io.Reader, out io.Writer, modName string) (bool, error) { +func do(in io.Reader, out io.Writer, modName string, slowThreshold time.Duration) (bool, error) { dec := json.NewDecoder(in) te := &TestEvent{} @@ -128,14 +111,36 @@ func do(in io.Reader, out io.Writer, modName string) (bool, error) { } buf := bufio.NewWriter(out) - var anyFail bool + + summaryF := getSummaryFile() + defer summaryF.Close() + + var failCount, skipCount int + var elapsed float64 + + failBuf := bytes.NewBuffer(nil) + hist := &dynhist.Collector{ + PrintSum: true, + WeightFunc: dynhist.ExpWidth(1.2, 0.9), + BucketsLimit: 10, + } + + slowBuf := bytes.NewBuffer(nil) + slow := slowThreshold.Seconds() for _, tr := range outs { + if tr.skipped { + skipCount++ + } + if tr.failed { - anyFail = true + failCount++ } - if err := writeResult(tr, buf, modName); err != nil { + hist.Add(tr.elapsed) + elapsed += tr.elapsed + + if err := writeResult(tr, buf, failBuf, slowBuf, slow, modName); err != nil { slog.Error("Error writing result", "error", err) continue } @@ -145,28 +150,42 @@ func do(in io.Reader, out io.Writer, modName string) (bool, error) { } } - return anyFail, nil -} + fmt.Fprintln(summaryF, "## Test metrics") + separator := strings.Repeat(" ", 4) + fmt.Fprintln(summaryF, mdBold("Skipped:"), skipCount, separator, mdBold("Failed:"), failCount, separator, mdBold("Total:"), len(outs), separator, mdBold("Elapsed:"), fmt.Sprintf("%.3fs", elapsed)) -func handlEvent(te *TestEvent, tr *TestResult) error { - if te.Output != "" { - _, err := tr.output.Write([]byte(te.Output)) - if err != nil { - return errors.Wrap(err, "error collecting test event output") - } + fmt.Fprintln(summaryF, mdPreformat(hist.String())) + + if failBuf.Len() > 0 { + fmt.Fprintln(summaryF, "## Failures") + fmt.Fprintln(summaryF, failBuf.String()) + } + + if slowBuf.Len() > 0 { + fmt.Fprintln(summaryF, "## Slow Tests") + fmt.Fprintln(summaryF, slowBuf.String()) } - tr.pkg = te.Package - tr.name = te.Test - tr.elapsed = te.Elapsed + return failCount > 0, nil +} + +func (c *nopWriteCloser) Close() error { return nil } - if te.Action == "fail" { - tr.failed = true +func getSummaryFile() io.WriteCloser { + v := os.Getenv("GITHUB_STEP_SUMMARY") + if v == "" { + return &nopWriteCloser{io.Discard} } - return nil + + f, err := os.OpenFile(v, os.O_WRONLY|os.O_APPEND, 0) + if err != nil { + slog.Error("Error opening step summary file", "error", err) + return &nopWriteCloser{io.Discard} + } + return f } -func writeResult(tr *TestResult, out io.Writer, modName string) error { +func writeResult(tr *TestResult, out, failBuf, slowBuf io.Writer, slow float64, modName string) error { if tr.name == "" { return nil } @@ -190,31 +209,44 @@ func writeResult(tr *TestResult, out io.Writer, modName string) error { prefix = "\u274c " } - dur := time.Duration(tr.elapsed * float64(time.Second)) - fmt.Fprintln(out, "::group::"+prefix+group, dur) + fmt.Fprintf(out, "::group::%s %.3fs\n", prefix+group, tr.elapsed) defer func() { fmt.Fprintln(out, "::endgroup::") }() - dt, err := io.ReadAll(tr.output) - if err != nil { - return fmt.Errorf("error reading test output: %w", err) + var rdr io.Reader = tr.output + + if tr.elapsed > slow { + buf := bytes.NewBuffer(nil) + rdr = io.TeeReader(rdr, buf) + defer func() { + if buf.Len() > 0 { + fmt.Fprintln(slowBuf, mdLog(fmt.Sprintf("%s %.3fs", tr.name, tr.elapsed), buf)) + } + }() } if !tr.failed { - if _, err := out.Write(dt); err != nil { + if _, err := io.Copy(out, rdr); err != nil { return fmt.Errorf("error writing test output to output stream: %w", err) } return nil } - scanner := bufio.NewScanner(bytes.NewReader(dt)) + failLog := bytes.NewBuffer(nil) + rdr = io.TeeReader(rdr, failLog) + defer func() { + fmt.Fprintln(failBuf, mdLog(tr.name+fmt.Sprintf(" %3.fs", tr.elapsed), failLog)) + }() + + scanner := bufio.NewScanner(rdr) var ( file, line string ) buf := bytes.NewBuffer(nil) + for scanner.Scan() { txt := scanner.Text() f, l, ok := getTestOutputLoc(txt) @@ -235,6 +267,7 @@ func writeResult(tr *TestResult, out io.Writer, modName string) error { file = filepath.Join(pkg, file) fmt.Fprintf(out, "::error file=%s,line=%s::%s\n", file, line, buf) + return nil } diff --git a/cmd/test2json2gha/main_test.go b/cmd/test2json2gha/main_test.go new file mode 100644 index 00000000..27af1fbd --- /dev/null +++ b/cmd/test2json2gha/main_test.go @@ -0,0 +1,26 @@ +package main + +import ( + "testing" + "time" +) + +func TestFail(t *testing.T) { + t.Log("This is for checking what a test failure looks like") + t.Fatal("This is a failure") +} + +func TestAnotherFail(t *testing.T) { + t.Log("This is for checking what a test failure looks like") + t.Fatal("This is yet another failure!") +} + +func TestSlow(t *testing.T) { + time.Sleep(10 * time.Second) + t.Log("This is a slow test!") +} + +func TestAnothrSlow(t *testing.T) { + time.Sleep(5 * time.Second) + t.Log("This is yet another slow test!") +} diff --git a/cmd/test2json2gha/markdown.go b/cmd/test2json2gha/markdown.go new file mode 100644 index 00000000..4f85e8e5 --- /dev/null +++ b/cmd/test2json2gha/markdown.go @@ -0,0 +1,34 @@ +package main + +import ( + "fmt" + "io" + "strings" +) + +func mdBold(v string) string { + return "**" + v + "**" +} + +func mdDetails(s string) string { + return fmt.Sprintf("\n
\n%s\n
\n", s) +} + +func mdSummary(s string) string { + return "" + s + "\n" +} + +func mdPreformat(s string) string { + return fmt.Sprintf("\n```\n%s\n```\n", s) +} + +type nopWriteCloser struct { + io.Writer +} + +func mdLog(head string, content fmt.Stringer) string { + sb := &strings.Builder{} + sb.WriteString(mdSummary(head)) + sb.WriteString(mdPreformat(content.String())) + return mdDetails(sb.String()) +} diff --git a/go.mod b/go.mod index 217a7fc4..0b66f678 100644 --- a/go.mod +++ b/go.mod @@ -18,6 +18,7 @@ require ( github.com/sirupsen/logrus v1.9.3 github.com/stretchr/testify v1.9.0 github.com/tonistiigi/fsutil v0.0.0-20240424095704-91a3fc46842c + github.com/vearutop/dynhist-go v1.2.3 go.opentelemetry.io/otel v1.21.0 go.opentelemetry.io/otel/sdk v1.21.0 golang.org/x/exp v0.0.0-20240909161429-701f63a606c0 diff --git a/go.sum b/go.sum index cdbcf8a9..aa6b9758 100644 --- a/go.sum +++ b/go.sum @@ -17,6 +17,8 @@ github.com/anchore/go-struct-converter v0.0.0-20221118182256-c68fdcfa2092 h1:aM1 github.com/anchore/go-struct-converter v0.0.0-20221118182256-c68fdcfa2092/go.mod h1:rYqSE9HbjzpHTI74vwPvae4ZVYZd1lue2ta6xHPdblA= github.com/bahlo/generic-list-go v0.2.0 h1:5sz/EEAK+ls5wF+NeqDpk5+iNdMDXrh3z3nPnH1Wvgk= github.com/bahlo/generic-list-go v0.2.0/go.mod h1:2KvAjgMlE5NNynlg/5iLrrCCZ2+5xWbdbCW3pNTGyYg= +github.com/bool64/dev v0.2.28 h1:6ayDfrB/jnNr2iQAZHI+uT3Qi6rErSbJYQs1y8rSrwM= +github.com/bool64/dev v0.2.28/go.mod h1:iJbh1y/HkunEPhgebWRNcs8wfGq7sjvJ6W5iabL8ACg= github.com/buger/jsonparser v1.1.1 h1:2PnMjfWD7wBILjqQbt530v576A/cAbQvEW9gGIpYMUs= github.com/buger/jsonparser v1.1.1/go.mod h1:6RYKKt7H4d4+iWqouImQ9R2FZql3VbhNgx27UK13J/0= github.com/cenkalti/backoff/v4 v4.2.1 h1:y4OZtCnogmCPw98Zjyt5a6+QwPLGkiQsYW5oUqylYbM= @@ -192,6 +194,8 @@ github.com/tonistiigi/vt100 v0.0.0-20240514184818-90bafcd6abab h1:H6aJ0yKQ0gF49Q github.com/tonistiigi/vt100 v0.0.0-20240514184818-90bafcd6abab/go.mod h1:ulncasL3N9uLrVann0m+CDlJKWsIAP34MPcOJF6VRvc= github.com/vbatts/tar-split v0.11.5 h1:3bHCTIheBm1qFTcgh9oPu+nNBtX+XJIupG/vacinCts= github.com/vbatts/tar-split v0.11.5/go.mod h1:yZbwRsSeGjusneWgA781EKej9HF8vme8okylkAeNKLk= +github.com/vearutop/dynhist-go v1.2.3 h1:EIMWszSDm6b7zmqySgx8zW2qNctE3IXUJggGlDFwJBE= +github.com/vearutop/dynhist-go v1.2.3/go.mod h1:liiiYiwAi8ixC3DbkxooEhASTF6ysJSXy+piCrBtxEg= github.com/wk8/go-ordered-map/v2 v2.1.8 h1:5h/BUHu93oj4gIdvHHHGsScSTMijfx5PeYkE/fJgbpc= github.com/wk8/go-ordered-map/v2 v2.1.8/go.mod h1:5nJHM5DyteebpVlHnWMV0rPz6Zp7+xBAnxjb1X5vnTw= github.com/yuin/goldmark v1.1.27/go.mod h1:3hX8gzYuyVAZsxl0MRgGTJEmQBFcNTphYh9decYSb74= diff --git a/test/azlinux_test.go b/test/azlinux_test.go index 7a0b3314..71f7e7a4 100644 --- a/test/azlinux_test.go +++ b/test/azlinux_test.go @@ -217,6 +217,8 @@ type OSRelease struct { func testLinuxDistro(ctx context.Context, t *testing.T, testConfig testLinuxConfig) { t.Run("Fail when non-zero exit code during build", func(t *testing.T) { t.Parallel() + ctx := startTestSpan(ctx, t) + spec := dalec.Spec{ Name: "test-build-commands-fail", Version: "0.0.1", @@ -247,6 +249,9 @@ func testLinuxDistro(ctx context.Context, t *testing.T, testConfig testLinuxConf }) t.Run("container", func(t *testing.T) { + t.Parallel() + ctx := startTestSpan(baseCtx, t) + const src2Patch3File = "patch3" src2Patch3Content := []byte(` diff --git a/file3 b/file3 @@ -523,6 +528,8 @@ echo "$BAR" > bar.txt t.Run("test systemd unit single", func(t *testing.T) { t.Parallel() + ctx := startTestSpan(baseCtx, t) + spec := &dalec.Spec{ Name: "test-systemd-unit", Description: "Test systemd unit", @@ -646,6 +653,8 @@ WantedBy=multi-user.target t.Run("test systemd unit multiple components", func(t *testing.T) { t.Parallel() + ctx := startTestSpan(baseCtx, t) + spec := &dalec.Spec{ Name: "test-systemd-unit", Description: "Test systemd unit", @@ -760,6 +769,8 @@ Environment="FOO_ARGS=--some-foo-args" t.Run("test systemd with only config dropin", func(t *testing.T) { t.Parallel() + ctx := startTestSpan(baseCtx, t) + spec := &dalec.Spec{ Name: "test-systemd-unit", Description: "Test systemd unit", @@ -871,6 +882,8 @@ Environment="KUBELET_KUBECONFIG_ARGS=--bootstrap-kubeconfig=/etc/kubernetes/boot t.Run("test directory creation", func(t *testing.T) { t.Parallel() + ctx := startTestSpan(ctx, t) + spec := &dalec.Spec{ Name: "test-directory-creation", Version: "0.0.1", @@ -937,6 +950,8 @@ Environment="KUBELET_KUBECONFIG_ARGS=--bootstrap-kubeconfig=/etc/kubernetes/boot t.Run("test data file installation", func(t *testing.T) { t.Parallel() + ctx := startTestSpan(baseCtx, t) + spec := &dalec.Spec{ Name: "test-data-file-installation", Version: "0.0.1", @@ -1029,6 +1044,8 @@ Environment="KUBELET_KUBECONFIG_ARGS=--bootstrap-kubeconfig=/etc/kubernetes/boot t.Run("test libexec file installation", func(t *testing.T) { t.Parallel() + ctx := startTestSpan(baseCtx, t) + spec := &dalec.Spec{ Name: "libexec-test", Version: "0.0.1", @@ -1133,6 +1150,8 @@ Environment="KUBELET_KUBECONFIG_ARGS=--bootstrap-kubeconfig=/etc/kubernetes/boot t.Run("test config files handled", func(t *testing.T) { t.Parallel() + ctx := startTestSpan(baseCtx, t) + spec := &dalec.Spec{ Name: "test-config-files-work", Version: "0.0.1", @@ -1192,6 +1211,8 @@ Environment="KUBELET_KUBECONFIG_ARGS=--bootstrap-kubeconfig=/etc/kubernetes/boot t.Run("docs and headers and licenses are handled correctly", func(t *testing.T) { t.Parallel() + ctx := startTestSpan(baseCtx, t) + spec := &dalec.Spec{ Name: "test-docs-handled", Version: "0.0.1", @@ -1364,9 +1385,7 @@ Environment="KUBELET_KUBECONFIG_ARGS=--bootstrap-kubeconfig=/etc/kubernetes/boot }) t.Run("custom repo", func(t *testing.T) { - t.Parallel() - ctx := startTestSpan(baseCtx, t) testCustomRepo(ctx, t, testConfig.Worker, testConfig.Target) }) @@ -1606,6 +1625,7 @@ func testPinnedBuildDeps(ctx context.Context, t *testing.T, cfg testLinuxConfig) t.Run(tt.name, func(t *testing.T) { t.Parallel() + ctx := startTestSpan(baseCtx, t) testEnv.RunTest(ctx, t, func(ctx context.Context, gwc gwclient.Client) { worker := getWorker(ctx, t, gwc) diff --git a/test/fixtures/phony.go b/test/fixtures/phony.go index ee60f234..2ec2b537 100644 --- a/test/fixtures/phony.go +++ b/test/fixtures/phony.go @@ -13,8 +13,8 @@ import ( ) var ( - goModCache = llb.AddMount("/go/pkg/mod", llb.Scratch(), llb.AsPersistentCacheDir("dalec-go-mod-cache", llb.CacheMountShared)) - goBuildCache = llb.AddMount("/root/.cache/go-build", llb.Scratch(), llb.AsPersistentCacheDir("dalec-go-build-cache", llb.CacheMountShared)) + goModCache = llb.AddMount("/go/pkg/mod", llb.Scratch(), llb.AsPersistentCacheDir("/go/pkg/mod", llb.CacheMountShared)) + goBuildCache = llb.AddMount("/root/.cache/go-build", llb.Scratch(), llb.AsPersistentCacheDir("/root/.cache/go-build", llb.CacheMountShared)) ) func PhonyFrontend(ctx context.Context, gwc gwclient.Client) (*gwclient.Result, error) { diff --git a/test/fs_test.go b/test/fs_test.go index a777506b..c6911978 100644 --- a/test/fs_test.go +++ b/test/fs_test.go @@ -16,9 +16,12 @@ import ( ) func TestStateWrapper_ReadAt(t *testing.T) { + t.Parallel() + ctx := startTestSpan(baseCtx, t) + st := llb.Scratch().File(llb.Mkfile("/foo", 0644, []byte("hello world"))) - testEnv.RunTest(baseCtx, t, func(ctx context.Context, gwc gwclient.Client) { + testEnv.RunTest(ctx, t, func(ctx context.Context, gwc gwclient.Client) { rfs, err := bkfs.FromState(ctx, &st, gwc) assert.Nil(t, err) @@ -45,8 +48,11 @@ func TestStateWrapper_ReadAt(t *testing.T) { } func TestStateWrapper_OpenInvalidPath(t *testing.T) { + t.Parallel() + ctx := startTestSpan(baseCtx, t) + st := llb.Scratch().File(llb.Mkfile("/bar", 0644, []byte("hello world"))) - testEnv.RunTest(baseCtx, t, func(ctx context.Context, gwc gwclient.Client) { + testEnv.RunTest(ctx, t, func(ctx context.Context, gwc gwclient.Client) { rfs, err := bkfs.FromState(ctx, &st, gwc) assert.Nil(t, err) @@ -61,10 +67,13 @@ func TestStateWrapper_OpenInvalidPath(t *testing.T) { } func TestStateWrapper_Open(t *testing.T) { + t.Parallel() + ctx := startTestSpan(baseCtx, t) + st := llb.Scratch(). File(llb.Mkfile("/foo", 0644, []byte("hello world"))) - testEnv.RunTest(baseCtx, t, func(ctx context.Context, gwc gwclient.Client) { + testEnv.RunTest(ctx, t, func(ctx context.Context, gwc gwclient.Client) { fs, err := bkfs.FromState(ctx, &st, gwc) assert.Nil(t, err) @@ -84,8 +93,11 @@ func TestStateWrapper_Open(t *testing.T) { } func TestStateWrapper_Stat(t *testing.T) { + t.Parallel() + ctx := startTestSpan(baseCtx, t) + st := llb.Scratch().File(llb.Mkfile("/foo", 0755, []byte("contents"))) - testEnv.RunTest(baseCtx, t, func(ctx context.Context, gwc gwclient.Client) { + testEnv.RunTest(ctx, t, func(ctx context.Context, gwc gwclient.Client) { rfs, err := bkfs.FromState(ctx, &st, gwc) assert.Nil(t, err) @@ -103,6 +115,9 @@ func TestStateWrapper_Stat(t *testing.T) { } func TestStateWrapper_ReadDir(t *testing.T) { + t.Parallel() + ctx := startTestSpan(baseCtx, t) + st := llb.Scratch().File(llb.Mkdir("/bar", 0644)). File(llb.Mkfile("/bar/foo", 0644, []byte("file contents"))). File(llb.Mkdir("/bar/baz", 0644)) @@ -124,7 +139,7 @@ func TestStateWrapper_ReadDir(t *testing.T) { }, } - testEnv.RunTest(baseCtx, t, func(ctx context.Context, gwc gwclient.Client) { + testEnv.RunTest(ctx, t, func(ctx context.Context, gwc gwclient.Client) { rfs, err := bkfs.FromState(ctx, &st, gwc) assert.Nil(t, err) @@ -148,6 +163,9 @@ func TestStateWrapper_ReadDir(t *testing.T) { } func TestStateWrapper_Walk(t *testing.T) { + t.Parallel() + ctx := startTestSpan(baseCtx, t) + // create a simple test file structure like so: /* dir/ @@ -216,7 +234,7 @@ func TestStateWrapper_Walk(t *testing.T) { }, } - testEnv.RunTest(baseCtx, t, func(ctx context.Context, gwc gwclient.Client) { + testEnv.RunTest(ctx, t, func(ctx context.Context, gwc gwclient.Client) { rfs, err := bkfs.FromState(ctx, &st, gwc) assert.Nil(t, err) totalCalls := 0 @@ -259,6 +277,9 @@ func TestStateWrapper_Walk(t *testing.T) { } func TestStateWrapper_ReadPartial(t *testing.T) { + t.Parallel() + ctx := startTestSpan(baseCtx, t) + contents := []byte(` This is a multline @@ -266,7 +287,7 @@ func TestStateWrapper_ReadPartial(t *testing.T) { `) st := llb.Scratch().File(llb.Mkfile("/foo", 0644, contents)) - testEnv.RunTest(baseCtx, t, func(ctx context.Context, gwc gwclient.Client) { + testEnv.RunTest(ctx, t, func(ctx context.Context, gwc gwclient.Client) { rfs, err := bkfs.FromState(ctx, &st, gwc) assert.Nil(t, err) @@ -308,6 +329,9 @@ func TestStateWrapper_ReadPartial(t *testing.T) { } func TestStateWrapper_ReadAll(t *testing.T) { + t.Parallel() + ctx := startTestSpan(baseCtx, t) + // purposefully exceed initial length of io.ReadAll buffer (512) b := make([]byte, 520) for i := 0; i < 520; i++ { @@ -316,7 +340,7 @@ func TestStateWrapper_ReadAll(t *testing.T) { st := llb.Scratch().File(llb.Mkfile("/file", 0644, b)) - testEnv.RunTest(baseCtx, t, func(ctx context.Context, gwc gwclient.Client) { + testEnv.RunTest(ctx, t, func(ctx context.Context, gwc gwclient.Client) { rfs, err := bkfs.FromState(ctx, &st, gwc) assert.Nil(t, err) diff --git a/test/handlers_test.go b/test/handlers_test.go index c20d7aa7..93439e67 100644 --- a/test/handlers_test.go +++ b/test/handlers_test.go @@ -21,6 +21,8 @@ import ( // TestHandlerTargetForwarding tests that targets are forwarded to the correct frontend. // We do this by registering a phony frontend and then forwarding a target to it and checking the outputs. func TestHandlerTargetForwarding(t *testing.T) { + t.Parallel() + runTest := func(t *testing.T, f testenv.TestFunc) { t.Helper() ctx := startTestSpan(baseCtx, t) @@ -129,10 +131,12 @@ func TestHandlerTargetForwarding(t *testing.T) { func TestHandlerSubrequestResolve(t *testing.T) { t.Parallel() + ctx := startTestSpan(baseCtx, t) - testPlatforms := func(t *testing.T, pls ...string) func(t *testing.T) { + testPlatforms := func(ctx context.Context, pls ...string) func(t *testing.T) { return func(t *testing.T) { t.Parallel() + startTestSpan(ctx, t) runTest(t, func(ctx context.Context, gwc gwclient.Client) { spec := &dalec.Spec{ @@ -190,7 +194,7 @@ func TestHandlerSubrequestResolve(t *testing.T) { } } - t.Run("no platform", testPlatforms(t)) - t.Run("single platform", testPlatforms(t, "linux/amd64")) - t.Run("multi-platform", testPlatforms(t, "linux/amd64", "linux/arm64")) + t.Run("no platform", testPlatforms(ctx)) + t.Run("single platform", testPlatforms(ctx, "linux/amd64")) + t.Run("multi-platform", testPlatforms(ctx, "linux/amd64", "linux/arm64")) } diff --git a/test/signing_test.go b/test/signing_test.go index fcf19bdd..6212997c 100644 --- a/test/signing_test.go +++ b/test/signing_test.go @@ -50,6 +50,9 @@ func newSimpleSpec() *dalec.Spec { func linuxSigningTests(ctx context.Context, testConfig testLinuxConfig) func(*testing.T) { return func(t *testing.T) { + t.Parallel() + ctx := startTestSpan(baseCtx, t) + newSigningSpec := func() *dalec.Spec { spec := newSimpleSpec() spec.PackageConfig = &dalec.PackageConfig{ @@ -133,6 +136,8 @@ func linuxSigningTests(ctx context.Context, testConfig testLinuxConfig) func(*te }) t.Run("with path build arg and build context", func(t *testing.T) { + t.Parallel() + spec := newSigningSpec() spec.PackageConfig.Signer = nil @@ -153,6 +158,8 @@ signer: }) t.Run("path build arg takes precedence over spec config", func(t *testing.T) { + t.Parallel() + spec := newSigningSpec() spec.PackageConfig.Signer.Frontend.Image = "notexist" @@ -191,6 +198,8 @@ signer: }) t.Run("with path build arg and build context", func(t *testing.T) { + t.Parallel() + spec := newSigningSpec() spec.PackageConfig.Signer = nil @@ -211,6 +220,8 @@ signer: }) t.Run("with no build context and config path build arg", func(t *testing.T) { + t.Parallel() + spec := newSigningSpec() spec.PackageConfig.Signer = nil @@ -231,6 +242,8 @@ signer: }) t.Run("local context with config path takes precedence over spec", func(t *testing.T) { + t.Parallel() + spec := newSigningSpec() spec.PackageConfig.Signer.Frontend.Image = "notexist" @@ -362,6 +375,7 @@ signer: } func windowsSigningTests(t *testing.T) { + t.Parallel() t.Run("target spec config", func(t *testing.T) { t.Parallel() runTest(t, func(ctx context.Context, gwc gwclient.Client) { diff --git a/test/source_test.go b/test/source_test.go index 0094ea4e..6b8c0c3a 100644 --- a/test/source_test.go +++ b/test/source_test.go @@ -65,16 +65,24 @@ func TestSourceCmd(t *testing.T) { } } - testEnv.RunTest(ctx, t, func(ctx context.Context, gwc gwclient.Client) { - spec := testSpec() - req := newSolveRequest(withBuildTarget("debug/sources"), withSpec(ctx, t, spec)) - res := solveT(ctx, t, gwc, req) + t.Run("base", func(t *testing.T) { + t.Parallel() + ctx := startTestSpan(ctx, t) + + testEnv.RunTest(ctx, t, func(ctx context.Context, gwc gwclient.Client) { + spec := testSpec() + req := newSolveRequest(withBuildTarget("debug/sources"), withSpec(ctx, t, spec)) + res := solveT(ctx, t, gwc, req) - checkFile(ctx, t, filepath.Join(sourceName, "foo"), res, []byte("foo bar\n")) - checkFile(ctx, t, filepath.Join(sourceName, "hello"), res, []byte("hello\n")) + checkFile(ctx, t, filepath.Join(sourceName, "foo"), res, []byte("foo bar\n")) + checkFile(ctx, t, filepath.Join(sourceName, "hello"), res, []byte("hello\n")) + }) }) t.Run("with mounted file", func(t *testing.T) { + t.Parallel() + ctx := startTestSpan(ctx, t) + testEnv.RunTest(ctx, t, func(ctx context.Context, gwc gwclient.Client) { spec := testSpec() spec.Sources[sourceName].DockerImage.Cmd.Steps = []*dalec.BuildStep{ @@ -139,6 +147,7 @@ func TestSourceBuild(t *testing.T) { } t.Run("inline", func(t *testing.T) { + t.Parallel() fileSrc := func() dalec.Source { return dalec.Source{ Inline: &dalec.SourceInline{ @@ -165,16 +174,19 @@ func TestSourceBuild(t *testing.T) { } t.Run("unspecified build file path", func(t *testing.T) { + t.Parallel() doBuildTest(t, "file", newBuildSpec("", fileSrc)) doBuildTest(t, "dir", newBuildSpec("", dirSrc("Dockerfile"))) }) t.Run("Dockerfile as build file path", func(t *testing.T) { + t.Parallel() doBuildTest(t, "file", newBuildSpec("Dockerfile", fileSrc)) doBuildTest(t, "dir", newBuildSpec("Dockerfile", dirSrc("Dockerfile"))) }) t.Run("non-standard build file path", func(t *testing.T) { + t.Parallel() doBuildTest(t, "file", newBuildSpec("foo", fileSrc)) doBuildTest(t, "dir", newBuildSpec("foo", dirSrc("foo"))) }) @@ -371,6 +383,7 @@ index ea874f5..ba38f84 100644 }) t.Run("with patch", func(t *testing.T) { + t.Parallel() t.Run("file", func(t *testing.T) { t.Parallel() testEnv.RunTest(baseCtx, t, func(ctx context.Context, gwc gwclient.Client) { diff --git a/test/testenv/builld.go b/test/testenv/build.go similarity index 95% rename from test/testenv/builld.go rename to test/testenv/build.go index a0ce5945..4cd5ae55 100644 --- a/test/testenv/builld.go +++ b/test/testenv/build.go @@ -1,6 +1,7 @@ package testenv import ( + "bufio" "context" "encoding/json" "io" @@ -154,35 +155,24 @@ func displaySolveStatus(ctx context.Context, t *testing.T) (chan *client.SolveSt } defer f.Close() - if !t.Failed() { - return - } - - sz, _ := f.Seek(0, io.SeekEnd) _, err = f.Seek(0, io.SeekStart) if err != nil { t.Log(err) return } - _, err = io.CopyN(&testWriter{t}, f, sz) - if err != nil { + + scanner := bufio.NewScanner(f) + for scanner.Scan() { + t.Log(scanner.Text()) + } + if err := scanner.Err(); err != nil { t.Log(err) - return } }() return ch, done } -type testWriter struct { - t *testing.T -} - -func (t *testWriter) Write(p []byte) (n int, err error) { - t.t.Log(string(p)) - return len(p), nil -} - // withProjectRoot adds the current project root as the build context for the solve request. func withProjectRoot(t *testing.T, opts *client.SolveOpt) { t.Helper() diff --git a/test/testenv/buildx.go b/test/testenv/buildx.go index 8d038beb..45681d62 100644 --- a/test/testenv/buildx.go +++ b/test/testenv/buildx.go @@ -290,7 +290,7 @@ func (b *BuildxEnv) RunTest(ctx context.Context, t *testing.T, f TestFunc, opts var so client.SolveOpt withProjectRoot(t, &so) - withGHCache(t, &so) + // withGHCache(t, &so) withResolveLocal(&so) _, err = c.Build(ctx, so, "", func(ctx context.Context, gwc gwclient.Client) (*gwclient.Result, error) { diff --git a/test/var_passthrough_test.go b/test/var_passthrough_test.go index 9aca4a37..67e08c2f 100644 --- a/test/var_passthrough_test.go +++ b/test/var_passthrough_test.go @@ -38,6 +38,8 @@ func getBuildPlatform(ctx context.Context, t *testing.T) *ocispecs.Platform { } func TestPassthroughVars(t *testing.T) { + t.Parallel() + runTest := func(t *testing.T, f testenv.TestFunc) { t.Helper() ctx := startTestSpan(baseCtx, t) diff --git a/test/windows_test.go b/test/windows_test.go index 0813f033..e7cea462 100644 --- a/test/windows_test.go +++ b/test/windows_test.go @@ -147,6 +147,8 @@ func testWindows(ctx context.Context, t *testing.T, cfg targetConfig) { }) t.Run("container", func(t *testing.T) { + t.Parallel() + spec := dalec.Spec{ Name: "test-container-build", Version: "0.0.1",