From bfe8d3d97ea63b8fc54cdab036fce0a046442678 Mon Sep 17 00:00:00 2001 From: Brian Goff Date: Mon, 8 Jul 2024 22:23:49 +0000 Subject: [PATCH 1/6] azlinux/mariner: Use provided platform for outputs Before this change, the mariner/azl targets were ignoring the client provided platform for the produced aritifacts. With this change the platform is set on images so it will rely on binfmt_misc to execute those images correctly. There's probably some optimizations that can be made here to run certain tasks on the native platform, some thoughts: - Use native (host-arch) golang to download go modules - Use native (host-arch) tdnf to download/install non-native packages onto the target build environment. This does *not* add support for cross compilation (run x86 code to generate, e.g., arm64 code). Signed-off-by: Brian Goff --- .github/workflows/ci.yml | 8 ++ frontend/azlinux/handle_container.go | 4 +- frontend/azlinux/handle_depsonly.go | 2 +- frontend/azlinux/handle_rpm.go | 2 +- frontend/debug/handler.go | 1 - frontend/gateway.go | 8 +- frontend/mux.go | 8 +- helpers.go | 10 ++ test/azlinux_test.go | 147 ++++++++++++++++++++++++++- test/helpers_test.go | 72 ++++++++++++- test/testenv/buildx.go | 27 +++++ 11 files changed, 271 insertions(+), 18 deletions(-) diff --git a/.github/workflows/ci.yml b/.github/workflows/ci.yml index 5f38832f..3c465479 100644 --- a/.github/workflows/ci.yml +++ b/.github/workflows/ci.yml @@ -31,6 +31,12 @@ on: - go.mod - go.sum +env: + # Used in tests to determine if certain tests should be skipped. + # Setting this ensures that they are *not* skipped and instead make sure CI + # is setup to be able to properly run all tests. + DALEC_CI: "1" + permissions: contents: read @@ -93,6 +99,8 @@ jobs: uses: docker/setup-buildx-action@988b5a0280414f521da01fcc63a27aeeb4b104db # v3.6.1 - name: download deps run: go mod download + - name: Setup QEMU + run: docker run --rm --privileged multiarch/qemu-user-static --reset -p yes - name: Run integaration tests run: go test -v -json ./test | go run ./cmd/test2json2gha - name: dump logs diff --git a/frontend/azlinux/handle_container.go b/frontend/azlinux/handle_container.go index e0863346..29861828 100644 --- a/frontend/azlinux/handle_container.go +++ b/frontend/azlinux/handle_container.go @@ -25,7 +25,7 @@ func handleContainer(w worker) gwclient.BuildFunc { pg := dalec.ProgressGroup("Building " + targetKey + " container: " + spec.Name) - rpmDir, err := specToRpmLLB(ctx, w, client, spec, sOpt, targetKey, pg) + rpmDir, err := specToRpmLLB(ctx, w, client, spec, sOpt, targetKey, pg, dalec.WithPlatform(platform)) if err != nil { return nil, nil, fmt.Errorf("error creating rpm: %w", err) } @@ -35,7 +35,7 @@ func handleContainer(w worker) gwclient.BuildFunc { return nil, nil, err } - st, err := specToContainerLLB(w, spec, targetKey, rpmDir, rpms, sOpt, pg) + st, err := specToContainerLLB(w, spec, targetKey, rpmDir, rpms, sOpt, pg, dalec.WithPlatform(platform)) if err != nil { return nil, nil, err } diff --git a/frontend/azlinux/handle_depsonly.go b/frontend/azlinux/handle_depsonly.go index 068d516f..93c2856d 100644 --- a/frontend/azlinux/handle_depsonly.go +++ b/frontend/azlinux/handle_depsonly.go @@ -21,7 +21,7 @@ func handleDepsOnly(w worker) gwclient.BuildFunc { return nil, nil, err } - baseImg, err := w.Base(sOpt, pg) + baseImg, err := w.Base(sOpt, pg, dalec.WithPlatform(platform)) if err != nil { return nil, nil, err } diff --git a/frontend/azlinux/handle_rpm.go b/frontend/azlinux/handle_rpm.go index 87e89f54..f54ead6c 100644 --- a/frontend/azlinux/handle_rpm.go +++ b/frontend/azlinux/handle_rpm.go @@ -26,7 +26,7 @@ func handleRPM(w worker) gwclient.BuildFunc { return nil, nil, err } - st, err := specToRpmLLB(ctx, w, client, spec, sOpt, targetKey, pg) + st, err := specToRpmLLB(ctx, w, client, spec, sOpt, targetKey, pg, dalec.WithPlatform(platform)) if err != nil { return nil, nil, err } diff --git a/frontend/debug/handler.go b/frontend/debug/handler.go index cc64333d..23958f84 100644 --- a/frontend/debug/handler.go +++ b/frontend/debug/handler.go @@ -25,6 +25,5 @@ func Handle(ctx context.Context, client gwclient.Client) (*gwclient.Result, erro Name: "gomods", Description: "Outputs all the gomodule dependencies for the spec", }) - return r.Handle(ctx, client) } diff --git a/frontend/gateway.go b/frontend/gateway.go index c8bf5c0b..e6e0d0d1 100644 --- a/frontend/gateway.go +++ b/frontend/gateway.go @@ -17,10 +17,12 @@ import ( ) const ( - requestIDKey = "requestid" - dalecSubrequstForwardBuild = "dalec.forward.build" + // KeyRequestID is a key used in buildkit to performa subrequest + // This is exposed for convenience only. + KeyRequestID = "requestid" - gatewayFrontend = "gateway.v0" + dalecSubrequstForwardBuild = "dalec.forward.build" + gatewayFrontend = "gateway.v0" ) func getDockerfile(ctx context.Context, client gwclient.Client, build *dalec.SourceBuild, defPb *pb.Definition) ([]byte, error) { diff --git a/frontend/mux.go b/frontend/mux.go index 8df85341..9950515c 100644 --- a/frontend/mux.go +++ b/frontend/mux.go @@ -117,7 +117,7 @@ func (m *BuildMux) describe() (*gwclient.Result, error) { } func (m *BuildMux) handleSubrequest(ctx context.Context, client gwclient.Client, opts map[string]string) (*gwclient.Result, bool, error) { - switch opts[requestIDKey] { + switch opts[KeyRequestID] { case "": return nil, false, nil case subrequests.RequestSubrequestsDescribe: @@ -135,7 +135,7 @@ func (m *BuildMux) handleSubrequest(ctx context.Context, client gwclient.Client, res, err := handleDefaultPlatform() return res, true, err default: - return nil, false, errors.Errorf("unsupported subrequest %q", opts[requestIDKey]) + return nil, false, errors.Errorf("unsupported subrequest %q", opts[KeyRequestID]) } } @@ -369,7 +369,7 @@ func (m *BuildMux) Handle(ctx context.Context, client gwclient.Client) (_ *gwcli WithFields(logrus.Fields{ "handlers": maps.Keys(m.handlers), "target": opts[keyTarget], - "requestid": opts[requestIDKey], + "requestid": opts[KeyRequestID], "targetKey": GetTargetKey(client), })) @@ -403,7 +403,7 @@ func (m *BuildMux) Handle(ctx context.Context, client gwclient.Client) (_ *gwcli // If this request was a request to list targets, we need to modify the response a bit // Otherwise we can just return the result as is. - if opts[requestIDKey] == bktargets.SubrequestsTargetsDefinition.Name { + if opts[KeyRequestID] == bktargets.SubrequestsTargetsDefinition.Name { return m.fixupListResult(matched, res) } return res, nil diff --git a/helpers.go b/helpers.go index 0d4832e3..5cca02e1 100644 --- a/helpers.go +++ b/helpers.go @@ -12,6 +12,7 @@ import ( "github.com/moby/buildkit/client/llb" "github.com/moby/buildkit/identity" + ocispecs "github.com/opencontainers/image-spec/specs-go/v1" ) var disableDiffMerge atomic.Bool @@ -413,3 +414,12 @@ func (s *Spec) GetPackageDeps(target string) *PackageDependencies { } return s.Dependencies } + +// WithPlatform sets the platform in the constraints opts +// This is similar to [llb.Platform] except this takes a pointer so you don't +// need to worry about dereferencing a potentially nil pointer. +func WithPlatform(p *ocispecs.Platform) llb.ConstraintsOpt { + return constraintsOptFunc(func(c *llb.Constraints) { + c.Platform = p + }) +} diff --git a/test/azlinux_test.go b/test/azlinux_test.go index f0e44372..597ebf11 100644 --- a/test/azlinux_test.go +++ b/test/azlinux_test.go @@ -1,7 +1,9 @@ package test import ( + "bytes" "context" + "debug/elf" "errors" "fmt" "os" @@ -10,9 +12,16 @@ import ( "github.com/Azure/dalec" "github.com/Azure/dalec/frontend/azlinux" + "github.com/containerd/platforms" "github.com/moby/buildkit/client/llb" gwclient "github.com/moby/buildkit/frontend/gateway/client" moby_buildkit_v1_frontend "github.com/moby/buildkit/frontend/gateway/pb" + ocispecs "github.com/opencontainers/image-spec/specs-go/v1" +) + +var ( + linuxAmd64 = ocispecs.Platform{OS: "linux", Architecture: "amd64"} + linuxArm64 = ocispecs.Platform{OS: "linux", Architecture: "arm64"} ) var azlinuxConstraints = constraintsSymbols{ @@ -53,6 +62,7 @@ func TestMariner2(t *testing.T) { ID: "mariner", VersionID: "2.0", }, + SupportedPlatforms: platforms.Any(linuxAmd64, linuxArm64), }) } @@ -83,6 +93,7 @@ func TestAzlinux3(t *testing.T) { ID: "azurelinux", VersionID: "3.0", }, + SupportedPlatforms: platforms.Any(linuxAmd64, linuxArm64), }) } @@ -154,8 +165,9 @@ type testLinuxConfig struct { Units string Targets string } - Worker workerConfig - Release OSRelease + Worker workerConfig + Release OSRelease + SupportedPlatforms platforms.Matcher } type OSRelease struct { @@ -633,6 +645,7 @@ WantedBy=multi-user.target Dir: &dalec.SourceInlineDir{ Files: map[string]*dalec.SourceInlineFile{ + "foo.service": { Contents: ` [Unit] @@ -1174,6 +1187,11 @@ Environment="KUBELET_KUBECONFIG_ARGS=--bootstrap-kubeconfig=/etc/kubernetes/boot }) }) + t.Run("platform", func(t *testing.T) { + ctx := startTestSpan(ctx, t) + testPlatforms(ctx, t, testConfig) + }) + t.Run("custom worker", func(t *testing.T) { t.Parallel() ctx := startTestSpan(baseCtx, t) @@ -1276,6 +1294,131 @@ func testCustomLinuxWorker(ctx context.Context, t *testing.T, targetCfg targetCo // Unfortunately it seems like there is an issue with the gateway client passing // in source policies. }) + +} + +func testPlatforms(ctx context.Context, t *testing.T, testConfig testLinuxConfig) { + if testConfig.SupportedPlatforms == nil { + t.Skip("List of supported platforms not provided for distro") + } + t.Run("build against different platform", func(t *testing.T) { + t.Parallel() + + ls, err := testEnv.Platforms(ctx) + if err != nil { + t.Fatal(err) + } + if len(ls) <= 1 { + t.Skipf("builder does not support multiple platforms: %s", platformsAsStringer(ls)) + } + + testEnv.RunTest(ctx, t, func(ctx context.Context, client gwclient.Client) { + p := readDefaultPlatform(ctx, t, client) + + matcher := platforms.OnlyStrict(p) + var testPlatform *ocispecs.Platform + for _, p2 := range ls { + // Get the first platform that is not the host platform that matches a supported distro platform + if !matcher.Match(p2) && testConfig.SupportedPlatforms.Match(p2) { + testPlatform = &p2 + break + } + } + + if testPlatform == nil { + msg := "could not find a platform suitable for testing, host platform: %s, available: %s" + ps := platformStringer(p) + workerPlatforms := platformsAsStringer(ls) + if os.Getenv("DALEC_CI") != "" { + t.Fatalf(msg, ps, workerPlatforms) + } + t.Skipf(msg, ps, workerPlatforms) + } + + spec := &dalec.Spec{ + Name: "test-platforms", + Version: "0.0.1", + Revision: "1", + Description: "Testing building on platform different from host platform", + License: "MIT", + Dependencies: &dalec.PackageDependencies{ + Build: map[string]dalec.PackageConstraints{ + "golang": {}, + }, + }, + Sources: map[string]dalec.Source{ + "src": { + Inline: &dalec.SourceInline{ + Dir: &dalec.SourceInlineDir{ + Files: map[string]*dalec.SourceInlineFile{ + "go.mod": { + Contents: "module test\n\ngo 1.21.6", + }, + "main.go": { + Contents: "package main\n\nfunc main() {}\n", + }, + }, + }, + }, + }, + }, + Build: dalec.ArtifactBuild{ + Steps: []dalec.BuildStep{ + {Command: "cd src; go build -o /tmp/test"}, + }, + }, + Artifacts: dalec.Artifacts{ + Binaries: map[string]dalec.ArtifactConfig{ + "/tmp/test": {}, + }, + }, + } + + tp := *testPlatform + req := newSolveRequest(withPlatform(tp), withSpec(ctx, t, spec), withBuildTarget(testConfig.Target.Container)) + res := solveT(ctx, t, client, req) + + imgPlatforms := readResultPlatforms(t, res) + if len(imgPlatforms) != 1 { + t.Fatal("expected image output to contain 1 platform") + } + + if !platforms.OnlyStrict(tp).Match(imgPlatforms[0]) { + t.Errorf("Expected image platform %q, got: %q", platformStringer(tp), platformStringer(imgPlatforms[0])) + } + + ref, err := res.SingleRef() + if err != nil { + t.Fatal(err) + } + if ref == nil { + t.Fatal("got empty reference -- most likely an empty (scratch) state was returned") + } + + // Read the ELF header so we can determine what the target architecture is. + dt, err := ref.ReadFile(ctx, gwclient.ReadRequest{ + Filename: "/usr/bin/test", + }) + if err != nil { + t.Fatal(err) + } + + f, err := elf.NewFile(bytes.NewReader(dt)) + if err != nil { + t.Fatal(err) + } + + check := ocispecs.Platform{ + OS: "linux", + } + elfToPlatform(f, &check) + + if !platforms.OnlyStrict(*testPlatform).Match(check) { + t.Fatalf("output binary has unexpected platform, expected: %s, got: %s", platformStringer(*testPlatform), platformStringer(check)) + } + }) + }) + } func testPinnedBuildDeps(ctx context.Context, t *testing.T, cfg testLinuxConfig) { diff --git a/test/helpers_test.go b/test/helpers_test.go index 94549841..50ec95e7 100644 --- a/test/helpers_test.go +++ b/test/helpers_test.go @@ -3,11 +3,14 @@ package test import ( "bytes" "context" + "debug/elf" + "encoding/binary" "encoding/json" "fmt" "io/fs" "path/filepath" "slices" + "strings" "testing" "github.com/Azure/dalec" @@ -20,6 +23,7 @@ import ( gwclient "github.com/moby/buildkit/frontend/gateway/client" "github.com/moby/buildkit/frontend/subrequests/targets" "github.com/moby/buildkit/solver/pb" + ocispecs "github.com/opencontainers/image-spec/specs-go/v1" "github.com/tonistiigi/fsutil/types" "go.opentelemetry.io/otel" "go.opentelemetry.io/otel/codes" @@ -197,6 +201,9 @@ func newSolveRequest(opts ...srOpt) gwclient.SolveRequest { func withPlatform(platform platforms.Platform) srOpt { return func(cfg *newSolveRequestConfig) { + if cfg.req.FrontendOpt == nil { + cfg.req.FrontendOpt = make(map[string]string) + } cfg.req.FrontendOpt["platform"] = platforms.Format(platform) } } @@ -221,21 +228,21 @@ func withSpec(ctx context.Context, t *testing.T, spec *dalec.Spec) srOpt { } } -func withBuildTarget(target string) srOpt { +func withSubrequest(id string) srOpt { return func(cfg *newSolveRequestConfig) { if cfg.req.FrontendOpt == nil { cfg.req.FrontendOpt = make(map[string]string) } - cfg.req.FrontendOpt["target"] = target + cfg.req.FrontendOpt["requestid"] = id } } -func withSubrequest(id string) srOpt { +func withBuildTarget(target string) srOpt { return func(cfg *newSolveRequestConfig) { if cfg.req.FrontendOpt == nil { cfg.req.FrontendOpt = make(map[string]string) } - cfg.req.FrontendOpt["requestid"] = id + cfg.req.FrontendOpt["target"] = target } } @@ -328,3 +335,60 @@ func readDefaultPlatform(ctx context.Context, t *testing.T, gwc gwclient.Client) assert.NilError(t, err) return p } + +func elfToPlatform(f *elf.File, target *ocispecs.Platform) { + switch f.Machine { + case elf.EM_X86_64: + target.Architecture = "amd64" + case elf.EM_ARM: + target.Architecture = "arm" + // TODO: subarch? + case elf.EM_AARCH64: + target.Architecture = "arm64" + case elf.EM_PPC64: + if f.ByteOrder == binary.LittleEndian { + target.Architecture = "ppc64le" + } + target.Architecture = "ppc64" + case elf.EM_S390: + target.Architecture = "s390x" + case elf.EM_RISCV: + target.Architecture = "riscv64" + } +} + +type platformsAsStringer []ocispecs.Platform + +func (ls platformsAsStringer) String() string { + collect := make([]string, 0, len(ls)) + for _, p := range ls { + collect = append(collect, platforms.Format(p)) + } + + return strings.Join(collect, ", ") +} + +func readResultPlatforms(t *testing.T, res *gwclient.Result) []ocispecs.Platform { + dt, ok := res.Metadata[exptypes.ExporterPlatformsKey] + if !ok { + return nil + } + + var pls exptypes.Platforms + if err := json.Unmarshal(dt, &pls); err != nil { + t.Fatal(err) + } + + out := make([]ocispecs.Platform, 0, len(pls.Platforms)) + for _, p := range pls.Platforms { + out = append(out, p.Platform) + } + + return out +} + +type platformStringer ocispecs.Platform + +func (p platformStringer) String() string { + return platforms.Format(ocispecs.Platform(p)) +} diff --git a/test/testenv/buildx.go b/test/testenv/buildx.go index 8d038beb..3c2aec10 100644 --- a/test/testenv/buildx.go +++ b/test/testenv/buildx.go @@ -15,9 +15,11 @@ import ( "sync" "testing" + "github.com/containerd/platforms" "github.com/moby/buildkit/client" gwclient "github.com/moby/buildkit/frontend/gateway/client" "github.com/moby/buildkit/solver/pb" + ocispecs "github.com/opencontainers/image-spec/specs-go/v1" pkgerrors "github.com/pkg/errors" ) @@ -215,6 +217,31 @@ func (b *BuildxEnv) Buildkit(ctx context.Context) (*client.Client, error) { panic("unreachable: if you see this then this is a bug in the testenv bootstrap code") } +func (b *BuildxEnv) Platforms(ctx context.Context) ([]ocispecs.Platform, error) { + client, err := b.Buildkit(ctx) + if err != nil { + return nil, err + } + + workers, err := client.ListWorkers(ctx) + if err != nil { + return nil, err + } + + v := make(map[string]ocispecs.Platform) + for _, w := range workers { + for _, p := range w.Platforms { + v[platforms.Format(p)] = p + } + } + + out := make([]ocispecs.Platform, 0, len(v)) + for _, p := range v { + out = append(out, p) + } + return out, nil +} + type FrontendSpec struct { ID string Build gwclient.BuildFunc From 7e93646f8a62e4b4f6230b80035184708692c0e8 Mon Sep 17 00:00:00 2001 From: Brian Goff Date: Thu, 18 Jul 2024 16:31:16 -0700 Subject: [PATCH 2/6] Use native platform to fetch sources, generate buildroot, etc When building for a non-native platform (typically under qemu), use the native platform for most operations that do not need platform specific things, such as applying patches, tarring up sources, and fetching go module deps. Signed-off-by: Brian Goff --- frontend/azlinux/handle_container.go | 2 +- frontend/azlinux/handle_depsonly.go | 2 +- frontend/azlinux/handle_rpm.go | 73 +++++++++++++++++++++------- frontend/azlinux/handler.go | 10 ++-- frontend/rpm/handle_buildroot.go | 6 ++- frontend/rpm/handle_sources.go | 4 +- 6 files changed, 69 insertions(+), 28 deletions(-) diff --git a/frontend/azlinux/handle_container.go b/frontend/azlinux/handle_container.go index 29861828..4284774d 100644 --- a/frontend/azlinux/handle_container.go +++ b/frontend/azlinux/handle_container.go @@ -25,7 +25,7 @@ func handleContainer(w worker) gwclient.BuildFunc { pg := dalec.ProgressGroup("Building " + targetKey + " container: " + spec.Name) - rpmDir, err := specToRpmLLB(ctx, w, client, spec, sOpt, targetKey, pg, dalec.WithPlatform(platform)) + rpmDir, err := buildOutputRPM(ctx, w, client, spec, sOpt, targetKey, platform, pg) if err != nil { return nil, nil, fmt.Errorf("error creating rpm: %w", err) } diff --git a/frontend/azlinux/handle_depsonly.go b/frontend/azlinux/handle_depsonly.go index 93c2856d..a7499f8b 100644 --- a/frontend/azlinux/handle_depsonly.go +++ b/frontend/azlinux/handle_depsonly.go @@ -36,7 +36,7 @@ func handleDepsOnly(w worker) gwclient.BuildFunc { return nil, nil, err } - st, err := specToContainerLLB(w, spec, targetKey, rpmDir, files, sOpt, pg) + st, err := specToContainerLLB(w, spec, targetKey, rpmDir, files, sOpt, pg, dalec.WithPlatform(platform)) if err != nil { return nil, nil, err } diff --git a/frontend/azlinux/handle_rpm.go b/frontend/azlinux/handle_rpm.go index f54ead6c..8cbe1767 100644 --- a/frontend/azlinux/handle_rpm.go +++ b/frontend/azlinux/handle_rpm.go @@ -11,6 +11,7 @@ import ( "github.com/moby/buildkit/client/llb" gwclient "github.com/moby/buildkit/frontend/gateway/client" ocispecs "github.com/opencontainers/image-spec/specs-go/v1" + "github.com/pkg/errors" ) func handleRPM(w worker) gwclient.BuildFunc { @@ -26,7 +27,7 @@ func handleRPM(w worker) gwclient.BuildFunc { return nil, nil, err } - st, err := specToRpmLLB(ctx, w, client, spec, sOpt, targetKey, pg, dalec.WithPlatform(platform)) + st, err := buildOutputRPM(ctx, w, client, spec, sOpt, targetKey, platform, pg) if err != nil { return nil, nil, err } @@ -52,8 +53,10 @@ func handleRPM(w worker) gwclient.BuildFunc { } } +type installFunc func(dalec.SourceOpts) (llb.RunOption, error) + // Creates and installs an rpm meta-package that requires the passed in deps as runtime-dependencies -func installBuildDepsPackage(target string, packageName string, w worker, deps map[string]dalec.PackageConstraints, installOpts ...installOpt) installFunc { +func installBuildDepsPackage(target string, packageName string, w worker, sOpt dalec.SourceOpts, deps map[string]dalec.PackageConstraints, platform *ocispecs.Platform, installOpts ...installOpt) installFunc { // depsOnly is a simple dalec spec that only includes build dependencies and their constraints depsOnly := dalec.Spec{ Name: fmt.Sprintf("%s-build-dependencies", packageName), @@ -66,11 +69,11 @@ func installBuildDepsPackage(target string, packageName string, w worker, deps m }, } - return func(ctx context.Context, client gwclient.Client, sOpt dalec.SourceOpts) (llb.RunOption, error) { + return func(Opt dalec.SourceOpts) (llb.RunOption, error) { pg := dalec.ProgressGroup("Building container for build dependencies") // create an RPM with just the build dependencies, using our same base worker - rpmDir, err := specToRpmLLB(ctx, w, client, &depsOnly, sOpt, target, pg) + rpmDir, err := createRPM(w, sOpt, &depsOnly, target, platform, pg) if err != nil { return nil, err } @@ -91,20 +94,15 @@ func installBuildDepsPackage(target string, packageName string, w worker, deps m } } -func installBuildDeps(ctx context.Context, w worker, client gwclient.Client, spec *dalec.Spec, targetKey string, opts ...llb.ConstraintsOpt) (llb.StateOption, error) { +func installBuildDeps(w worker, sOpt dalec.SourceOpts, spec *dalec.Spec, targetKey string, platform *ocispecs.Platform, opts ...llb.ConstraintsOpt) (llb.StateOption, error) { deps := spec.GetBuildDeps(targetKey) if len(deps) == 0 { return func(in llb.State) llb.State { return in }, nil } - sOpt, err := frontend.SourceOptFromClient(ctx, client) - if err != nil { - return nil, err - } - opts = append(opts, dalec.ProgressGroup("Install build deps")) - installOpt, err := installBuildDepsPackage(targetKey, spec.Name, w, deps, installWithConstraints(opts))(ctx, client, sOpt) + installOpt, err := installBuildDepsPackage(targetKey, spec.Name, w, sOpt, deps, platform, installWithConstraints(opts))(sOpt) if err != nil { return nil, err } @@ -114,24 +112,65 @@ func installBuildDeps(ctx context.Context, w worker, client gwclient.Client, spe }, nil } -func specToRpmLLB(ctx context.Context, w worker, client gwclient.Client, spec *dalec.Spec, sOpt dalec.SourceOpts, targetKey string, opts ...llb.ConstraintsOpt) (llb.State, error) { - base, err := w.Base(sOpt, opts...) +func rpmWorker(w worker, sOpt dalec.SourceOpts, spec *dalec.Spec, targetKey string, platform *ocispecs.Platform, opts ...llb.ConstraintsOpt) (llb.State, error) { + base, err := w.Base(sOpt, append(opts, dalec.WithPlatform(platform))...) if err != nil { return llb.Scratch(), err } - installOpt, err := installBuildDeps(ctx, w, client, spec, targetKey, opts...) + installDeps, err := installBuildDeps(w, sOpt, spec, targetKey, platform, opts...) if err != nil { return llb.Scratch(), err } - base = base.With(installOpt) - br, err := rpm.SpecToBuildrootLLB(base, spec, sOpt, targetKey, opts...) + base = base.With(installDeps) + return base, nil +} + +func createBuildroot(w worker, sOpt dalec.SourceOpts, spec *dalec.Spec, targetKey string, opts ...llb.ConstraintsOpt) (llb.State, error) { + opts = append(opts, dalec.ProgressGroup("Prepare rpm build root: "+spec.Name)) + + // Always generate the build root using the native platform + // There is nothing it does that should require the requested target platform + native, err := w.Base(sOpt, opts...) if err != nil { return llb.Scratch(), err } + + if spec.HasGomods() { + // Since the spec has go mods in it, we need to make sure we have go + // installed in the image. + install, err := installBuildDeps(w, sOpt, spec, targetKey, nil, opts...) + if err != nil { + return llb.Scratch(), err + } + + native = native.With(install) + } + + return rpm.SpecToBuildrootLLB(native, spec, sOpt, targetKey, opts...) +} + +func createRPM(w worker, sOpt dalec.SourceOpts, spec *dalec.Spec, targetKey string, platform *ocispecs.Platform, opts ...llb.ConstraintsOpt) (llb.State, error) { + br, err := createBuildroot(w, sOpt, spec, targetKey, opts...) + if err != nil { + return llb.Scratch(), errors.Wrap(err, "error creating rpm build root") + } + + base, err := rpmWorker(w, sOpt, spec, targetKey, platform, opts...) + if err != nil { + return llb.Scratch(), nil + } + specPath := filepath.Join("SPECS", spec.Name, spec.Name+".spec") - st := rpm.Build(br, base, specPath, opts...) + opts = append(opts, dalec.ProgressGroup("Create RPM: "+spec.Name)) + return rpm.Build(br, base, specPath, opts...), nil +} +func buildOutputRPM(ctx context.Context, w worker, client gwclient.Client, spec *dalec.Spec, sOpt dalec.SourceOpts, targetKey string, platform *ocispecs.Platform, opts ...llb.ConstraintsOpt) (llb.State, error) { + st, err := createRPM(w, sOpt, spec, targetKey, platform, opts...) + if err != nil { + return llb.Scratch(), err + } return frontend.MaybeSign(ctx, client, st, spec, targetKey, sOpt) } diff --git a/frontend/azlinux/handler.go b/frontend/azlinux/handler.go index dc2498f7..b29963b5 100644 --- a/frontend/azlinux/handler.go +++ b/frontend/azlinux/handler.go @@ -18,8 +18,6 @@ const ( tdnfCacheDir = "/var/cache/tdnf" ) -type installFunc func(context.Context, gwclient.Client, dalec.SourceOpts) (llb.RunOption, error) - type worker interface { Base(sOpt dalec.SourceOpts, opts ...llb.ConstraintsOpt) (llb.State, error) Install(pkgs []string, opts ...installOpt) llb.RunOption @@ -62,12 +60,12 @@ func handleDebug(w worker) gwclient.BuildFunc { if err != nil { return nil, err } - return rpm.HandleDebug(getSpecWorker(ctx, w, client, sOpt))(ctx, client) + return rpm.HandleDebug(getSpecWorker(ctx, w, sOpt))(ctx, client) } } -func getSpecWorker(ctx context.Context, w worker, client gwclient.Client, sOpt dalec.SourceOpts) rpm.WorkerFunc { - return func(resolver llb.ImageMetaResolver, spec *dalec.Spec, targetKey string, opts ...llb.ConstraintsOpt) (llb.State, error) { +func getSpecWorker(ctx context.Context, w worker, sOpt dalec.SourceOpts) rpm.WorkerFunc { + return func(resolver llb.ImageMetaResolver, spec *dalec.Spec, targetKey string, platform *ocispecs.Platform, opts ...llb.ConstraintsOpt) (llb.State, error) { st, err := w.Base(sOpt, opts...) if err != nil { return llb.Scratch(), err @@ -83,7 +81,7 @@ func getSpecWorker(ctx context.Context, w worker, client gwclient.Client, sOpt d return llb.Scratch(), errors.New("spec contains go modules but does not have golang in build deps") } - installOpt, err := installBuildDeps(ctx, w, client, spec, targetKey, opts...) + installOpt, err := installBuildDeps(w, sOpt, spec, targetKey, platform, opts...) if err != nil { return llb.Scratch(), err } diff --git a/frontend/rpm/handle_buildroot.go b/frontend/rpm/handle_buildroot.go index 8dee177b..c47b91ea 100644 --- a/frontend/rpm/handle_buildroot.go +++ b/frontend/rpm/handle_buildroot.go @@ -11,7 +11,7 @@ import ( ocispecs "github.com/opencontainers/image-spec/specs-go/v1" ) -type WorkerFunc func(resolver llb.ImageMetaResolver, spec *dalec.Spec, targetKey string, opts ...llb.ConstraintsOpt) (llb.State, error) +type WorkerFunc func(resolver llb.ImageMetaResolver, spec *dalec.Spec, targetKey string, platform *ocispecs.Platform, opts ...llb.ConstraintsOpt) (llb.State, error) func HandleBuildroot(wf WorkerFunc) gwclient.BuildFunc { return func(ctx context.Context, client gwclient.Client) (*gwclient.Result, error) { @@ -21,7 +21,9 @@ func HandleBuildroot(wf WorkerFunc) gwclient.BuildFunc { return nil, nil, err } - worker, err := wf(sOpt.Resolver, spec, targetKey) + // Note, we are not passing platform down here because everything should + // be able to work regardless of platform, so prefer the native platform. + worker, err := wf(sOpt.Resolver, spec, targetKey, platform) if err != nil { return nil, nil, err } diff --git a/frontend/rpm/handle_sources.go b/frontend/rpm/handle_sources.go index d46ddacc..18116697 100644 --- a/frontend/rpm/handle_sources.go +++ b/frontend/rpm/handle_sources.go @@ -21,11 +21,13 @@ func HandleSources(wf WorkerFunc) gwclient.BuildFunc { return nil, nil, err } - worker, err := wf(sOpt.Resolver, spec, targetKey) + worker, err := wf(sOpt.Resolver, spec, targetKey, platform) if err != nil { return nil, nil, err } + // Note, we are not passing platform down here because everything should + // be able to work regardless of platform, so prefer the native platform. sources, err := Dalec2SourcesLLB(worker, spec, sOpt) if err != nil { return nil, nil, err From 6ed351c6ab5bfc9c10956422191ad3d713288e91 Mon Sep 17 00:00:00 2001 From: Brian Goff Date: Thu, 29 Aug 2024 10:12:06 -0700 Subject: [PATCH 3/6] azlinux: make worker target build correct platform Signed-off-by: Brian Goff --- frontend/azlinux/handler.go | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/frontend/azlinux/handler.go b/frontend/azlinux/handler.go index b29963b5..cea35c85 100644 --- a/frontend/azlinux/handler.go +++ b/frontend/azlinux/handler.go @@ -101,7 +101,7 @@ func handleBaseImg(w worker) gwclient.BuildFunc { return nil, nil, err } - st, err := w.Base(sOpt) + st, err := w.Base(sOpt, dalec.WithPlatform(platform)) if err != nil { return nil, nil, err } From c4c36e9c22269b37fdf85e304147698f0b2d3f4c Mon Sep 17 00:00:00 2001 From: Brian Goff Date: Thu, 29 Aug 2024 10:13:46 -0700 Subject: [PATCH 4/6] azlinux: Use native go when building non-native platform This is an optimization to allow using the native go compiler to cross compile to the desired target. This is transparent to the user. Signed-off-by: Brian Goff --- frontend/azlinux/handle_rpm.go | 56 +++++++++++++++++++++++++++++++++- frontend/rpm/rpmbuild.go | 7 ++++- 2 files changed, 61 insertions(+), 2 deletions(-) diff --git a/frontend/azlinux/handle_rpm.go b/frontend/azlinux/handle_rpm.go index 8cbe1767..2ed253e7 100644 --- a/frontend/azlinux/handle_rpm.go +++ b/frontend/azlinux/handle_rpm.go @@ -4,10 +4,12 @@ import ( "context" "fmt" "path/filepath" + "strings" "github.com/Azure/dalec" "github.com/Azure/dalec/frontend" "github.com/Azure/dalec/frontend/rpm" + "github.com/containerd/platforms" "github.com/moby/buildkit/client/llb" gwclient "github.com/moby/buildkit/frontend/gateway/client" ocispecs "github.com/opencontainers/image-spec/specs-go/v1" @@ -151,6 +153,46 @@ func createBuildroot(w worker, sOpt dalec.SourceOpts, spec *dalec.Spec, targetKe return rpm.SpecToBuildrootLLB(native, spec, sOpt, targetKey, opts...) } +func nativeGoMount(native llb.State, p *ocispecs.Platform) llb.RunOption { + const ( + gorootPath = "/usr/lib/golang" + goBinPath = "/usr/bin/go" + internalBinPath = "/tmp/internal/dalec/bin" + ) + + runOpts := []llb.RunOption{ + llb.AddMount(gorootPath, native, llb.SourcePath(gorootPath), llb.Readonly), + llb.AddEnv("GOARCH", p.Architecture), + dalec.RunOptFunc(func(ei *llb.ExecInfo) { + if p.Variant != "" { + switch p.Architecture { + case "arm": + // GOARM cannot have the `v` prefix that would be in the platform struct + llb.AddEnv("GOARM", strings.TrimPrefix(p.Variant, "v")).SetRunOption(ei) + case "amd64": + // Unlike GOARM, GOAMD64 must have the `v` prefix (Which should be + // present in the platform struct) + llb.AddEnv("GOAMD64", p.Variant).SetRunOption(ei) + default: + // go does not support any other special sub-architectures currently. + } + } + }), + } + + return dalec.WithRunOptions(runOpts...) +} + +func hasGolangBuildDep(spec *dalec.Spec, targetKey string) bool { + deps := spec.GetBuildDeps(targetKey) + for pkg := range deps { + if pkg == "golang" || pkg == "msft-golang" { + return true + } + } + return false +} + func createRPM(w worker, sOpt dalec.SourceOpts, spec *dalec.Spec, targetKey string, platform *ocispecs.Platform, opts ...llb.ConstraintsOpt) (llb.State, error) { br, err := createBuildroot(w, sOpt, spec, targetKey, opts...) if err != nil { @@ -162,9 +204,21 @@ func createRPM(w worker, sOpt dalec.SourceOpts, spec *dalec.Spec, targetKey stri return llb.Scratch(), nil } + var runOpts []llb.RunOption + if platform != nil { + if platforms.Only(platforms.DefaultSpec()).Match(*platform) && hasGolangBuildDep(spec, targetKey) { + native, err := rpmWorker(w, sOpt, spec, targetKey, nil, opts...) + if err != nil { + return llb.Scratch(), err + } + + runOpts = append(runOpts, nativeGoMount(native, platform)) + } + } + specPath := filepath.Join("SPECS", spec.Name, spec.Name+".spec") opts = append(opts, dalec.ProgressGroup("Create RPM: "+spec.Name)) - return rpm.Build(br, base, specPath, opts...), nil + return rpm.Build(br, base, specPath, runOpts, opts...), nil } func buildOutputRPM(ctx context.Context, w worker, client gwclient.Client, spec *dalec.Spec, sOpt dalec.SourceOpts, targetKey string, platform *ocispecs.Platform, opts ...llb.ConstraintsOpt) (llb.State, error) { diff --git a/frontend/rpm/rpmbuild.go b/frontend/rpm/rpmbuild.go index e46e8f9d..290aad15 100644 --- a/frontend/rpm/rpmbuild.go +++ b/frontend/rpm/rpmbuild.go @@ -16,7 +16,7 @@ import ( // It is expected to have rpmbuild and any other necessary build dependencies installed // // `specPath` is the path to the spec file to build relative to `topDir` -func Build(topDir, workerImg llb.State, specPath string, opts ...llb.ConstraintsOpt) llb.State { +func Build(topDir, workerImg llb.State, specPath string, runOpts []llb.RunOption, opts ...llb.ConstraintsOpt) llb.State { opts = append(opts, dalec.ProgressGroup("Build RPM")) return workerImg.Run( // some notes on these args: @@ -35,6 +35,11 @@ func Build(topDir, workerImg llb.State, specPath string, opts ...llb.Constraints llb.Dir("/build/top"), llb.Network(llb.NetModeNone), dalec.WithConstraints(opts...), + dalec.RunOptFunc(func(ei *llb.ExecInfo) { + for _, opt := range runOpts { + opt.SetRunOption(ei) + } + }), ). AddMount("/build/out", llb.Scratch()) } From 6c9e983c8f846bbe805b77c284d9dd785241da58 Mon Sep 17 00:00:00 2001 From: Brian Goff Date: Thu, 29 Aug 2024 16:46:15 -0700 Subject: [PATCH 5/6] azlinux: Use native dnf to install packages When cross compiling, use dnf from the native platform to install packages into the target arch rootfs. This helps speed up package install a bit since dnf won't need to run under emulation. Signed-off-by: Brian Goff --- frontend/azlinux/azlinux3.go | 12 ++-- frontend/azlinux/handle_rpm.go | 100 ++++++++++++++++++--------------- frontend/azlinux/handler.go | 4 -- frontend/azlinux/install.go | 52 +++++++++++++++-- frontend/azlinux/mariner2.go | 12 ++-- 5 files changed, 111 insertions(+), 69 deletions(-) diff --git a/frontend/azlinux/azlinux3.go b/frontend/azlinux/azlinux3.go index c09b074a..54665b7d 100644 --- a/frontend/azlinux/azlinux3.go +++ b/frontend/azlinux/azlinux3.go @@ -3,7 +3,6 @@ package azlinux import ( "context" "encoding/json" - "path/filepath" "github.com/Azure/dalec" "github.com/moby/buildkit/client/llb" @@ -13,8 +12,7 @@ import ( ) const ( - AzLinux3TargetKey = "azlinux3" - tdnfCacheNameAzlinux3 = "azlinux3-tdnf-cache" + AzLinux3TargetKey = "azlinux3" // Azlinux3Ref is the image ref used for the base worker image Azlinux3Ref = "mcr.microsoft.com/azurelinux/base/core:3.0" @@ -49,6 +47,8 @@ func (w azlinux3) Base(sOpt dalec.SourceOpts, opts ...llb.ConstraintsOpt) (llb.S img := llb.Image(Azlinux3Ref, llb.WithMetaResolver(sOpt.Resolver), dalec.WithConstraints(opts...)) return img.Run( + w.Install([]string{"dnf"}, installWithConstraints(opts), tdnfOnly), + ).Run( w.Install([]string{"rpm-build", "mariner-rpm-macros", "build-essential", "ca-certificates"}, installWithConstraints(opts)), dalec.WithConstraints(opts...), ).Root(), nil @@ -57,7 +57,7 @@ func (w azlinux3) Base(sOpt dalec.SourceOpts, opts ...llb.ConstraintsOpt) (llb.S func (w azlinux3) Install(pkgs []string, opts ...installOpt) llb.RunOption { var cfg installConfig setInstallOptions(&cfg, opts) - return dalec.WithRunOptions(tdnfInstall(&cfg, "3.0", pkgs), w.tdnfCacheMount(cfg.root)) + return dalec.WithRunOptions(dnfInstall(&cfg, "3.0", pkgs, AzLinux3TargetKey)) } func (w azlinux3) BasePackages() []string { @@ -91,7 +91,3 @@ func (azlinux3) WorkerImageConfig(ctx context.Context, resolver llb.ImageMetaRes return &cfg, nil } - -func (azlinux3) tdnfCacheMount(root string) llb.RunOption { - return llb.AddMount(filepath.Join(root, tdnfCacheDir), llb.Scratch(), llb.AsPersistentCacheDir(tdnfCacheNameAzlinux3, llb.CacheMountLocked)) -} diff --git a/frontend/azlinux/handle_rpm.go b/frontend/azlinux/handle_rpm.go index 2ed253e7..d475f674 100644 --- a/frontend/azlinux/handle_rpm.go +++ b/frontend/azlinux/handle_rpm.go @@ -55,13 +55,31 @@ func handleRPM(w worker) gwclient.BuildFunc { } } -type installFunc func(dalec.SourceOpts) (llb.RunOption, error) +func platformFuzzyMatches(p *ocispecs.Platform) bool { + if p == nil { + return true + } + + // Note, this is intentionally not doing a strict match here + // (e.g. [platforms.OnlyStrict]) + // This is used to see if we can get some optimizations when building for a + // non-native platformm and in most cases the [platforms.Only] vector handles + // things like building armv7 on an arm64 machine, which should be able to run + // natively. + return platforms.Only(platforms.DefaultSpec()).Match(*p) +} + +func installBuildDeps(w worker, sOpt dalec.SourceOpts, spec *dalec.Spec, targetKey string, platform *ocispecs.Platform, opts ...llb.ConstraintsOpt) (llb.StateOption, error) { + deps := spec.GetBuildDeps(targetKey) + if len(deps) == 0 { + return func(in llb.State) llb.State { return in }, nil + } + + opts = append(opts, dalec.ProgressGroup("Install build deps")) -// Creates and installs an rpm meta-package that requires the passed in deps as runtime-dependencies -func installBuildDepsPackage(target string, packageName string, w worker, sOpt dalec.SourceOpts, deps map[string]dalec.PackageConstraints, platform *ocispecs.Platform, installOpts ...installOpt) installFunc { // depsOnly is a simple dalec spec that only includes build dependencies and their constraints depsOnly := dalec.Spec{ - Name: fmt.Sprintf("%s-build-dependencies", packageName), + Name: spec.Name + "-build-dependencies", Description: "Provides build dependencies for mariner2 and azlinux3", Version: "1.0", License: "Apache 2.0", @@ -71,46 +89,42 @@ func installBuildDepsPackage(target string, packageName string, w worker, sOpt d }, } - return func(Opt dalec.SourceOpts) (llb.RunOption, error) { - pg := dalec.ProgressGroup("Building container for build dependencies") + // create an RPM with just the build dependencies, using our same base worker + rpmDir, err := createRPM(w, sOpt, &depsOnly, targetKey, platform, opts...) + if err != nil { + return nil, err + } + + rpmMountDir := "/tmp/rpms" + pkg := []string{"/tmp/rpms/*/*.rpm"} - // create an RPM with just the build dependencies, using our same base worker - rpmDir, err := createRPM(w, sOpt, &depsOnly, target, platform, pg) + if !platformFuzzyMatches(platform) { + base, err := w.Base(sOpt, opts...) if err != nil { return nil, err } - var opts []llb.ConstraintsOpt - opts = append(opts, dalec.ProgressGroup("Install build deps")) - - rpmMountDir := "/tmp/rpms" - - installOpts = append([]installOpt{ - noGPGCheck, - withMounts(llb.AddMount(rpmMountDir, rpmDir, llb.SourcePath("/RPMS"))), - installWithConstraints(opts), - }, installOpts...) - - // install the built RPMs into the worker itself - return w.Install([]string{"/tmp/rpms/*/*.rpm"}, installOpts...), nil - } -} - -func installBuildDeps(w worker, sOpt dalec.SourceOpts, spec *dalec.Spec, targetKey string, platform *ocispecs.Platform, opts ...llb.ConstraintsOpt) (llb.StateOption, error) { - deps := spec.GetBuildDeps(targetKey) - if len(deps) == 0 { - return func(in llb.State) llb.State { return in }, nil - } - - opts = append(opts, dalec.ProgressGroup("Install build deps")) - - installOpt, err := installBuildDepsPackage(targetKey, spec.Name, w, sOpt, deps, platform, installWithConstraints(opts))(sOpt) - if err != nil { - return nil, err + return func(in llb.State) llb.State { + return base.Run( + w.Install( + pkg, + withMounts(llb.AddMount(rpmMountDir, rpmDir, llb.SourcePath("/RPMS"))), + atRoot("/tmp/rootfs"), + withPlatform(platform), + ), + ).AddMount("/tmp/rootfs", in) + }, nil } return func(in llb.State) llb.State { - return in.Run(installOpt, dalec.WithConstraints(opts...)).Root() + return in.Run( + w.Install( + []string{"/tmp/rpms/*/*.rpm"}, + withMounts(llb.AddMount(rpmMountDir, rpmDir, llb.SourcePath("/RPMS"))), + installWithConstraints(opts), + ), + dalec.WithConstraints(opts...), + ).Root() }, nil } @@ -205,15 +219,13 @@ func createRPM(w worker, sOpt dalec.SourceOpts, spec *dalec.Spec, targetKey stri } var runOpts []llb.RunOption - if platform != nil { - if platforms.Only(platforms.DefaultSpec()).Match(*platform) && hasGolangBuildDep(spec, targetKey) { - native, err := rpmWorker(w, sOpt, spec, targetKey, nil, opts...) - if err != nil { - return llb.Scratch(), err - } - - runOpts = append(runOpts, nativeGoMount(native, platform)) + if !platformFuzzyMatches(platform) && hasGolangBuildDep(spec, targetKey) { + native, err := rpmWorker(w, sOpt, spec, targetKey, nil, opts...) + if err != nil { + return llb.Scratch(), err } + + runOpts = append(runOpts, nativeGoMount(native, platform)) } specPath := filepath.Join("SPECS", spec.Name, spec.Name+".spec") diff --git a/frontend/azlinux/handler.go b/frontend/azlinux/handler.go index cea35c85..68fc0e70 100644 --- a/frontend/azlinux/handler.go +++ b/frontend/azlinux/handler.go @@ -14,10 +14,6 @@ import ( ocispecs "github.com/opencontainers/image-spec/specs-go/v1" ) -const ( - tdnfCacheDir = "/var/cache/tdnf" -) - type worker interface { Base(sOpt dalec.SourceOpts, opts ...llb.ConstraintsOpt) (llb.State, error) Install(pkgs []string, opts ...installOpt) llb.RunOption diff --git a/frontend/azlinux/install.go b/frontend/azlinux/install.go index c933e833..25a3046a 100644 --- a/frontend/azlinux/install.go +++ b/frontend/azlinux/install.go @@ -7,6 +7,7 @@ import ( "github.com/Azure/dalec" "github.com/moby/buildkit/client/llb" + ocispecs "github.com/opencontainers/image-spec/specs-go/v1" ) type installConfig struct { @@ -23,7 +24,14 @@ type installConfig struct { // Additional mounts to add to the tdnf install command (useful if installing RPMS which are mounted to a local directory) mounts []llb.RunOption + // Instructs the installer to install packages for the specified platform + platform *ocispecs.Platform + constraints []llb.ConstraintsOpt + + // This forces the use of tdnf + // Note this will almost certainly not work when platform is set. + tdnfOnly bool } type installOpt func(*installConfig) @@ -32,6 +40,10 @@ func noGPGCheck(cfg *installConfig) { cfg.noGPGCheck = true } +func tdnfOnly(cfg *installConfig) { + cfg.tdnfOnly = true +} + func withMounts(opts ...llb.RunOption) installOpt { return func(cfg *installConfig) { cfg.mounts = append(cfg.mounts, opts...) @@ -48,13 +60,19 @@ func atRoot(root string) installOpt { } } +func withPlatform(p *ocispecs.Platform) installOpt { + return func(cfg *installConfig) { + cfg.platform = p + } +} + func installWithConstraints(opts []llb.ConstraintsOpt) installOpt { return func(cfg *installConfig) { cfg.constraints = opts } } -func tdnfInstallFlags(cfg *installConfig) string { +func dnfInstallFlags(cfg *installConfig) string { var cmdOpts string if cfg.noGPGCheck { @@ -63,12 +81,30 @@ func tdnfInstallFlags(cfg *installConfig) string { if cfg.root != "" { cmdOpts += " --installroot=" + cfg.root - cmdOpts += " --setopt=reposdir=/etc/yum.repos.d" + cmdOpts += " --setopt reposdir=/etc/yum.repos.d" + } + + if cfg.platform != nil { + // cmdOpts += " --ignorearch=true" + cmdOpts += " --forcearch=" + ociArchToOS(cfg.platform) } return cmdOpts } +func ociArchToOS(p *ocispecs.Platform) string { + switch p.Architecture { + case "amd64": + return "x86_64" + case "arm64": + return "aarch64" + // azlinux only supports amd64 and arm64 + // We shouldn't need any other arches. + default: + return p.Architecture + } +} + func setInstallOptions(cfg *installConfig, opts []installOpt) { for _, o := range opts { o(cfg) @@ -111,9 +147,14 @@ rm -rf `+rpmdbDir+` const manifestSh = "manifest.sh" -func tdnfInstall(cfg *installConfig, relVer string, pkgs []string) llb.RunOption { - cmdFlags := tdnfInstallFlags(cfg) - cmdArgs := fmt.Sprintf("set -ex; tdnf install -y --refresh --releasever=%s %s %s", relVer, cmdFlags, strings.Join(pkgs, " ")) +func dnfInstall(cfg *installConfig, relVer string, pkgs []string, cachePrefix string) llb.RunOption { + cmdFlags := dnfInstallFlags(cfg) + + cmd := "dnf" + if cfg.tdnfOnly { + cmd = "tdnf" + } + cmdArgs := fmt.Sprintf("set -ex; %s install -y --refresh --releasever=%s %s %s", cmd, relVer, cmdFlags, strings.Join(pkgs, " ")) var runOpts []llb.RunOption @@ -128,6 +169,7 @@ func tdnfInstall(cfg *installConfig, relVer string, pkgs []string) llb.RunOption runOpts = append(runOpts, dalec.ShArgs(cmdArgs)) runOpts = append(runOpts, cfg.mounts...) + runOpts = append(runOpts, llb.AddMount("/var/cache/"+cmd, llb.Scratch(), llb.AsPersistentCacheDir(cachePrefix+"-"+cmd+"-"+"cache", llb.CacheMountLocked))) return dalec.WithRunOptions(runOpts...) } diff --git a/frontend/azlinux/mariner2.go b/frontend/azlinux/mariner2.go index 078d1033..0c267695 100644 --- a/frontend/azlinux/mariner2.go +++ b/frontend/azlinux/mariner2.go @@ -3,7 +3,6 @@ package azlinux import ( "context" "encoding/json" - "path/filepath" "github.com/Azure/dalec" "github.com/moby/buildkit/client/llb" @@ -13,8 +12,7 @@ import ( ) const ( - Mariner2TargetKey = "mariner2" - tdnfCacheNameMariner2 = "mariner2-tdnf-cache" + Mariner2TargetKey = "mariner2" Mariner2Ref = "mcr.microsoft.com/cbl-mariner/base/core:2.0" Mariner2WorkerContextName = "dalec-mariner2-worker" @@ -46,6 +44,8 @@ func (w mariner2) Base(sOpt dalec.SourceOpts, opts ...llb.ConstraintsOpt) (llb.S } return base.Run( + w.Install([]string{"dnf"}, installWithConstraints(opts), tdnfOnly), + ).Run( w.Install([]string{"rpm-build", "mariner-rpm-macros", "build-essential", "ca-certificates"}, installWithConstraints(opts)), dalec.WithConstraints(opts...), ).Root(), nil @@ -54,7 +54,7 @@ func (w mariner2) Base(sOpt dalec.SourceOpts, opts ...llb.ConstraintsOpt) (llb.S func (w mariner2) Install(pkgs []string, opts ...installOpt) llb.RunOption { var cfg installConfig setInstallOptions(&cfg, opts) - return dalec.WithRunOptions(tdnfInstall(&cfg, "2.0", pkgs), w.tdnfCacheMount(cfg.root)) + return dalec.WithRunOptions(dnfInstall(&cfg, "2.0", pkgs, Mariner2TargetKey)) } func (w mariner2) BasePackages() []string { @@ -90,7 +90,3 @@ func (mariner2) WorkerImageConfig(ctx context.Context, resolver llb.ImageMetaRes return &cfg, nil } - -func (mariner2) tdnfCacheMount(root string) llb.RunOption { - return llb.AddMount(filepath.Join(root, tdnfCacheDir), llb.Scratch(), llb.AsPersistentCacheDir(tdnfCacheNameMariner2, llb.CacheMountLocked)) -} From c254fabbf0c67f92feff1dd0f7f8a31f32b32af2 Mon Sep 17 00:00:00 2001 From: Brian Goff Date: Thu, 29 Aug 2024 17:29:19 -0700 Subject: [PATCH 6/6] azlinux: add go mod cache when building rpms This offers a tremendous speedup for local development. Signed-off-by: Brian Goff --- frontend/azlinux/handle_rpm.go | 27 ++++++++++++++++++++++----- 1 file changed, 22 insertions(+), 5 deletions(-) diff --git a/frontend/azlinux/handle_rpm.go b/frontend/azlinux/handle_rpm.go index d475f674..2c3b45d4 100644 --- a/frontend/azlinux/handle_rpm.go +++ b/frontend/azlinux/handle_rpm.go @@ -207,6 +207,13 @@ func hasGolangBuildDep(spec *dalec.Spec, targetKey string) bool { return false } +func platformOrDefault(p *ocispecs.Platform) ocispecs.Platform { + if p == nil { + return platforms.DefaultSpec() + } + return *p +} + func createRPM(w worker, sOpt dalec.SourceOpts, spec *dalec.Spec, targetKey string, platform *ocispecs.Platform, opts ...llb.ConstraintsOpt) (llb.State, error) { br, err := createBuildroot(w, sOpt, spec, targetKey, opts...) if err != nil { @@ -219,13 +226,23 @@ func createRPM(w worker, sOpt dalec.SourceOpts, spec *dalec.Spec, targetKey stri } var runOpts []llb.RunOption - if !platformFuzzyMatches(platform) && hasGolangBuildDep(spec, targetKey) { - native, err := rpmWorker(w, sOpt, spec, targetKey, nil, opts...) - if err != nil { - return llb.Scratch(), err + if hasGolangBuildDep(spec, targetKey) { + if !platformFuzzyMatches(platform) { + native, err := rpmWorker(w, sOpt, spec, targetKey, nil, opts...) + if err != nil { + return llb.Scratch(), err + } + + runOpts = append(runOpts, nativeGoMount(native, platform)) } - runOpts = append(runOpts, nativeGoMount(native, platform)) + const goCacheDir = "/tmp/dalec/internal/gocache" + runOpts = append(runOpts, llb.AddEnv("GOCACHE", goCacheDir)) + + // Unfortunately, go cannot invalidate caches for cgo (rather, cgo with 'include' directives). + // As such we need to include the platform in our cache key. + cacheKey := targetKey + "-golang-" + platforms.Format(platformOrDefault(platform)) + runOpts = append(runOpts, llb.AddMount(goCacheDir, llb.Scratch(), llb.AsPersistentCacheDir(cacheKey, llb.CacheMountShared))) } specPath := filepath.Join("SPECS", spec.Name, spec.Name+".spec")