azlinux/mariner: Use provided platform for outputs

.github/workflows/ci.yml

@@ -31,6 +31,12 @@ on:
       - go.mod
       - go.sum
 
+env:
+  # Used in tests to determine if certain tests should be skipped.
+  # Setting this ensures that they are *not* skipped and instead make sure CI
+  # is setup to be able to properly run all tests.
+  DALEC_CI: "1"
+
 permissions:
   contents: read
 
@@ -93,6 +99,8 @@ jobs:
         uses: docker/setup-buildx-action@988b5a0280414f521da01fcc63a27aeeb4b104db # v3.6.1
       - name: download deps
         run: go mod download
+      - name: Setup QEMU
+        run: docker run --rm --privileged multiarch/qemu-user-static --reset -p yes
       - name: Run integaration tests
         run: go test -v -json ./test | go run ./cmd/test2json2gha
       - name: dump logs

frontend/azlinux/azlinux3.go

@@ -3,7 +3,6 @@ package azlinux
 import (
 	"context"
 	"encoding/json"
-	"path/filepath"
 
 	"github.com/Azure/dalec"
 	"github.com/moby/buildkit/client/llb"
@@ -13,8 +12,7 @@ import (
 )
 
 const (
-	AzLinux3TargetKey     = "azlinux3"
-	tdnfCacheNameAzlinux3 = "azlinux3-tdnf-cache"
+	AzLinux3TargetKey = "azlinux3"
 
 	// Azlinux3Ref is the image ref used for the base worker image
 	Azlinux3Ref = "mcr.microsoft.com/azurelinux/base/core:3.0"
@@ -49,6 +47,8 @@ func (w azlinux3) Base(sOpt dalec.SourceOpts, opts ...llb.ConstraintsOpt) (llb.S
 
 	img := llb.Image(Azlinux3Ref, llb.WithMetaResolver(sOpt.Resolver), dalec.WithConstraints(opts...))
 	return img.Run(
+		w.Install([]string{"dnf"}, installWithConstraints(opts), tdnfOnly),
+	).Run(
 		w.Install([]string{"rpm-build", "mariner-rpm-macros", "build-essential", "ca-certificates"}, installWithConstraints(opts)),
 		dalec.WithConstraints(opts...),
 	).Root(), nil
@@ -57,7 +57,7 @@ func (w azlinux3) Base(sOpt dalec.SourceOpts, opts ...llb.ConstraintsOpt) (llb.S
 func (w azlinux3) Install(pkgs []string, opts ...installOpt) llb.RunOption {
 	var cfg installConfig
 	setInstallOptions(&cfg, opts)
-	return dalec.WithRunOptions(tdnfInstall(&cfg, "3.0", pkgs), w.tdnfCacheMount(cfg.root))
+	return dalec.WithRunOptions(dnfInstall(&cfg, "3.0", pkgs, AzLinux3TargetKey))
 }
 
 func (w azlinux3) BasePackages() []string {
@@ -91,7 +91,3 @@ func (azlinux3) WorkerImageConfig(ctx context.Context, resolver llb.ImageMetaRes
 
 	return &cfg, nil
 }
-
-func (azlinux3) tdnfCacheMount(root string) llb.RunOption {
-	return llb.AddMount(filepath.Join(root, tdnfCacheDir), llb.Scratch(), llb.AsPersistentCacheDir(tdnfCacheNameAzlinux3, llb.CacheMountLocked))
-}

frontend/azlinux/handle_container.go

@@ -25,7 +25,7 @@ func handleContainer(w worker) gwclient.BuildFunc {
 
 			pg := dalec.ProgressGroup("Building " + targetKey + " container: " + spec.Name)
 
-			rpmDir, err := specToRpmLLB(ctx, w, client, spec, sOpt, targetKey, pg)
+			rpmDir, err := buildOutputRPM(ctx, w, client, spec, sOpt, targetKey, platform, pg)
 			if err != nil {
 				return nil, nil, fmt.Errorf("error creating rpm: %w", err)
 			}
@@ -35,7 +35,7 @@ func handleContainer(w worker) gwclient.BuildFunc {
 				return nil, nil, err
 			}
 
-			st, err := specToContainerLLB(w, spec, targetKey, rpmDir, rpms, sOpt, pg)
+			st, err := specToContainerLLB(w, spec, targetKey, rpmDir, rpms, sOpt, pg, dalec.WithPlatform(platform))
 			if err != nil {
 				return nil, nil, err
 			}

frontend/azlinux/handle_depsonly.go

@@ -21,7 +21,7 @@ func handleDepsOnly(w worker) gwclient.BuildFunc {
 				return nil, nil, err
 			}
 
-			baseImg, err := w.Base(sOpt, pg)
+			baseImg, err := w.Base(sOpt, pg, dalec.WithPlatform(platform))
 			if err != nil {
 				return nil, nil, err
 			}
@@ -36,7 +36,7 @@ func handleDepsOnly(w worker) gwclient.BuildFunc {
 				return nil, nil, err
 			}
 
-			st, err := specToContainerLLB(w, spec, targetKey, rpmDir, files, sOpt, pg)
+			st, err := specToContainerLLB(w, spec, targetKey, rpmDir, files, sOpt, pg, dalec.WithPlatform(platform))
 			if err != nil {
 				return nil, nil, err
 			}

frontend/azlinux/handle_rpm.go

@@ -4,13 +4,16 @@ import (
 	"context"
 	"fmt"
 	"path/filepath"
+	"strings"
 
 	"github.com/Azure/dalec"
 	"github.com/Azure/dalec/frontend"
 	"github.com/Azure/dalec/frontend/rpm"
+	"github.com/containerd/platforms"
 	"github.com/moby/buildkit/client/llb"
 	gwclient "github.com/moby/buildkit/frontend/gateway/client"
 	ocispecs "github.com/opencontainers/image-spec/specs-go/v1"
+	"github.com/pkg/errors"
 )
 
 func handleRPM(w worker) gwclient.BuildFunc {
@@ -26,7 +29,7 @@ func handleRPM(w worker) gwclient.BuildFunc {
 				return nil, nil, err
 			}
 
-			st, err := specToRpmLLB(ctx, w, client, spec, sOpt, targetKey, pg)
+			st, err := buildOutputRPM(ctx, w, client, spec, sOpt, targetKey, platform, pg)
 			if err != nil {
 				return nil, nil, err
 			}
@@ -52,11 +55,31 @@ func handleRPM(w worker) gwclient.BuildFunc {
 	}
 }
 
-// Creates and installs an rpm meta-package that requires the passed in deps as runtime-dependencies
-func installBuildDepsPackage(target string, packageName string, w worker, deps map[string]dalec.PackageConstraints, installOpts ...installOpt) installFunc {
+func platformFuzzyMatches(p *ocispecs.Platform) bool {
+	if p == nil {
+		return true
+	}
+
+	// Note, this is intentionally not doing a strict match here
+	// (e.g. [platforms.OnlyStrict])
+	// This is used to see if we can get some optimizations when building for a
+	// non-native platformm and in most cases the [platforms.Only] vector handles
+	// things like building armv7 on an arm64 machine, which should be able to run
+	// natively.
+	return platforms.Only(platforms.DefaultSpec()).Match(*p)
+}
+
+func installBuildDeps(w worker, sOpt dalec.SourceOpts, spec *dalec.Spec, targetKey string, platform *ocispecs.Platform, opts ...llb.ConstraintsOpt) (llb.StateOption, error) {
+	deps := spec.GetBuildDeps(targetKey)
+	if len(deps) == 0 {
+		return func(in llb.State) llb.State { return in }, nil
+	}
+
+	opts = append(opts, dalec.ProgressGroup("Install build deps"))
+
 	// depsOnly is a simple dalec spec that only includes build dependencies and their constraints
 	depsOnly := dalec.Spec{
-		Name:        fmt.Sprintf("%s-build-dependencies", packageName),
+		Name:        spec.Name + "-build-dependencies",
 		Description: "Provides build dependencies for mariner2 and azlinux3",
 		Version:     "1.0",
 		License:     "Apache 2.0",
@@ -66,72 +89,171 @@ func installBuildDepsPackage(target string, packageName string, w worker, deps m
 		},
 	}
 
-	return func(ctx context.Context, client gwclient.Client, sOpt dalec.SourceOpts) (llb.RunOption, error) {
-		pg := dalec.ProgressGroup("Building container for build dependencies")
+	// create an RPM with just the build dependencies, using our same base worker
+	rpmDir, err := createRPM(w, sOpt, &depsOnly, targetKey, platform, opts...)
+	if err != nil {
+		return nil, err
+	}
+
+	rpmMountDir := "/tmp/rpms"
+	pkg := []string{"/tmp/rpms/*/*.rpm"}
 
-		// create an RPM with just the build dependencies, using our same base worker
-		rpmDir, err := specToRpmLLB(ctx, w, client, &depsOnly, sOpt, target, pg)
+	if !platformFuzzyMatches(platform) {
+		base, err := w.Base(sOpt, opts...)
 		if err != nil {
 			return nil, err
 		}
 
-		var opts []llb.ConstraintsOpt
-		opts = append(opts, dalec.ProgressGroup("Install build deps"))
+		return func(in llb.State) llb.State {
+			return base.Run(
+				w.Install(
+					pkg,
+					withMounts(llb.AddMount(rpmMountDir, rpmDir, llb.SourcePath("/RPMS"))),
+					atRoot("/tmp/rootfs"),
+					withPlatform(platform),
+				),
+			).AddMount("/tmp/rootfs", in)
+		}, nil
+	}
 
-		rpmMountDir := "/tmp/rpms"
+	return func(in llb.State) llb.State {
+		return in.Run(
+			w.Install(
+				[]string{"/tmp/rpms/*/*.rpm"},
+				withMounts(llb.AddMount(rpmMountDir, rpmDir, llb.SourcePath("/RPMS"))),
+				installWithConstraints(opts),
+			),
+			dalec.WithConstraints(opts...),
+		).Root()
+	}, nil
+}
 
-		installOpts = append([]installOpt{
-			noGPGCheck,
-			withMounts(llb.AddMount(rpmMountDir, rpmDir, llb.SourcePath("/RPMS"))),
-			installWithConstraints(opts),
-		}, installOpts...)
+func rpmWorker(w worker, sOpt dalec.SourceOpts, spec *dalec.Spec, targetKey string, platform *ocispecs.Platform, opts ...llb.ConstraintsOpt) (llb.State, error) {
+	base, err := w.Base(sOpt, append(opts, dalec.WithPlatform(platform))...)
+	if err != nil {
+		return llb.Scratch(), err
+	}
 
-		// install the built RPMs into the worker itself
-		return w.Install([]string{"/tmp/rpms/*/*.rpm"}, installOpts...), nil
+	installDeps, err := installBuildDeps(w, sOpt, spec, targetKey, platform, opts...)
+	if err != nil {
+		return llb.Scratch(), err
 	}
+
+	base = base.With(installDeps)
+	return base, nil
 }
 
-func installBuildDeps(ctx context.Context, w worker, client gwclient.Client, spec *dalec.Spec, targetKey string, opts ...llb.ConstraintsOpt) (llb.StateOption, error) {
-	deps := spec.GetBuildDeps(targetKey)
-	if len(deps) == 0 {
-		return func(in llb.State) llb.State { return in }, nil
-	}
+func createBuildroot(w worker, sOpt dalec.SourceOpts, spec *dalec.Spec, targetKey string, opts ...llb.ConstraintsOpt) (llb.State, error) {
+	opts = append(opts, dalec.ProgressGroup("Prepare rpm build root: "+spec.Name))
 
-	sOpt, err := frontend.SourceOptFromClient(ctx, client)
+	// Always generate the build root using the native platform
+	// There is nothing it does that should require the requested target platform
+	native, err := w.Base(sOpt, opts...)
 	if err != nil {
-		return nil, err
+		return llb.Scratch(), err
 	}
 
-	opts = append(opts, dalec.ProgressGroup("Install build deps"))
+	if spec.HasGomods() {
+		// Since the spec has go mods in it, we need to make sure we have go
+		// installed in the image.
+		install, err := installBuildDeps(w, sOpt, spec, targetKey, nil, opts...)
+		if err != nil {
+			return llb.Scratch(), err
+		}
 
-	installOpt, err := installBuildDepsPackage(targetKey, spec.Name, w, deps, installWithConstraints(opts))(ctx, client, sOpt)
-	if err != nil {
-		return nil, err
+		native = native.With(install)
 	}
 
-	return func(in llb.State) llb.State {
-		return in.Run(installOpt, dalec.WithConstraints(opts...)).Root()
-	}, nil
+	return rpm.SpecToBuildrootLLB(native, spec, sOpt, targetKey, opts...)
 }
 
-func specToRpmLLB(ctx context.Context, w worker, client gwclient.Client, spec *dalec.Spec, sOpt dalec.SourceOpts, targetKey string, opts ...llb.ConstraintsOpt) (llb.State, error) {
-	base, err := w.Base(sOpt, opts...)
-	if err != nil {
-		return llb.Scratch(), err
+func nativeGoMount(native llb.State, p *ocispecs.Platform) llb.RunOption {
+	const (
+		gorootPath      = "/usr/lib/golang"
+		goBinPath       = "/usr/bin/go"
+		internalBinPath = "/tmp/internal/dalec/bin"
+	)
+
+	runOpts := []llb.RunOption{
+		llb.AddMount(gorootPath, native, llb.SourcePath(gorootPath), llb.Readonly),
+		llb.AddEnv("GOARCH", p.Architecture),
+		dalec.RunOptFunc(func(ei *llb.ExecInfo) {
+			if p.Variant != "" {
+				switch p.Architecture {
+				case "arm":
+					// GOARM cannot have the `v` prefix that would be in the platform struct
+					llb.AddEnv("GOARM", strings.TrimPrefix(p.Variant, "v")).SetRunOption(ei)
+				case "amd64":
+					// Unlike GOARM, GOAMD64 must have the `v` prefix (Which should be
+					// present in the platform struct)
+					llb.AddEnv("GOAMD64", p.Variant).SetRunOption(ei)
+				default:
+					// go does not support any other special sub-architectures currently.
+				}
+			}
+		}),
 	}
 
-	installOpt, err := installBuildDeps(ctx, w, client, spec, targetKey, opts...)
+	return dalec.WithRunOptions(runOpts...)
+}
+
+func hasGolangBuildDep(spec *dalec.Spec, targetKey string) bool {
+	deps := spec.GetBuildDeps(targetKey)
+	for pkg := range deps {
+		if pkg == "golang" || pkg == "msft-golang" {
+			return true
+		}
+	}
+	return false
+}
+
+func platformOrDefault(p *ocispecs.Platform) ocispecs.Platform {
+	if p == nil {
+		return platforms.DefaultSpec()
+	}
+	return *p
+}
+
+func createRPM(w worker, sOpt dalec.SourceOpts, spec *dalec.Spec, targetKey string, platform *ocispecs.Platform, opts ...llb.ConstraintsOpt) (llb.State, error) {
+	br, err := createBuildroot(w, sOpt, spec, targetKey, opts...)
 	if err != nil {
-		return llb.Scratch(), err
+		return llb.Scratch(), errors.Wrap(err, "error creating rpm build root")
 	}
-	base = base.With(installOpt)
 
-	br, err := rpm.SpecToBuildrootLLB(base, spec, sOpt, targetKey, opts...)
+	base, err := rpmWorker(w, sOpt, spec, targetKey, platform, opts...)
 	if err != nil {
-		return llb.Scratch(), err
+		return llb.Scratch(), nil
+	}
+
+	var runOpts []llb.RunOption
+	if hasGolangBuildDep(spec, targetKey) {
+		if !platformFuzzyMatches(platform) {
+			native, err := rpmWorker(w, sOpt, spec, targetKey, nil, opts...)
+			if err != nil {
+				return llb.Scratch(), err
+			}
+
+			runOpts = append(runOpts, nativeGoMount(native, platform))
+		}
+
+		const goCacheDir = "/tmp/dalec/internal/gocache"
+		runOpts = append(runOpts, llb.AddEnv("GOCACHE", goCacheDir))
+
+		// Unfortunately, go cannot invalidate caches for cgo (rather, cgo with 'include' directives).
+		// As such we need to include the platform in our cache key.
+		cacheKey := targetKey + "-golang-" + platforms.Format(platformOrDefault(platform))
+		runOpts = append(runOpts, llb.AddMount(goCacheDir, llb.Scratch(), llb.AsPersistentCacheDir(cacheKey, llb.CacheMountShared)))
 	}
+
 	specPath := filepath.Join("SPECS", spec.Name, spec.Name+".spec")
-	st := rpm.Build(br, base, specPath, opts...)
+	opts = append(opts, dalec.ProgressGroup("Create RPM: "+spec.Name))
+	return rpm.Build(br, base, specPath, runOpts, opts...), nil
+}
 
+func buildOutputRPM(ctx context.Context, w worker, client gwclient.Client, spec *dalec.Spec, sOpt dalec.SourceOpts, targetKey string, platform *ocispecs.Platform, opts ...llb.ConstraintsOpt) (llb.State, error) {
+	st, err := createRPM(w, sOpt, spec, targetKey, platform, opts...)
+	if err != nil {
+		return llb.Scratch(), err
+	}
 	return frontend.MaybeSign(ctx, client, st, spec, targetKey, sOpt)
 }