update deb and multi-src structure

Author: DannyBrito

docker-bake.hcl

@@ -62,7 +62,7 @@ target "runc-azlinux" {
         "DALEC_DISABLE_DIFF_MERGE" = DALEC_DISABLE_DIFF_MERGE
     }
     contexts = {
-      "dalec_frontend" = "target:frontend"
+        "dalec_frontend" = "target:frontend"
     }
     matrix = {
         distro = ["mariner2", "azlinux3"]
@@ -86,7 +86,7 @@ target "runc-jammy" {
         "DALEC_DISABLE_DIFF_MERGE" = DALEC_DISABLE_DIFF_MERGE
     }
     contexts = {
-      "dalec_frontend" = "target:frontend"
+        "dalec_frontend" = "target:frontend"
     }
     matrix = {
         tgt = ["deb", "container"]
@@ -124,9 +124,9 @@ target "test-fixture" {
     matrix = {
         f = DALEC_DISABLE_NESTED == "1" ? (
             ["http-src", "frontend", "local-context", "cmd-src-ref"]
-         ) : (
+        ) : (
             ["http-src", "frontend", "local-context", "cmd-src-ref", "nested"]
-         )
+        )
         tgt = ["mariner2/container"]
     }
     dockerfile = "test/fixtures/${f}.yml"
@@ -136,7 +136,7 @@ target "test-fixture" {
         "DALEC_DISABLE_DIFF_MERGE" = DALEC_DISABLE_DIFF_MERGE
     }
     contexts = {
-      "dalec_frontend" = "target:frontend"
+        "dalec_frontend" = "target:frontend"
     }
     target = tgt
 }
@@ -156,7 +156,7 @@ target "build" {
         "BUILDKIT_SYNTAX" = "dalec_frontend"
     }
     contexts = {
-      "dalec_frontend" = "target:frontend"
+        "dalec_frontend" = "target:frontend"
     }
     target = "${distro}/${tgt}"
     // only tag the container target
@@ -175,7 +175,7 @@ target "examples" {
         "BUILDKIT_SYNTAX" = "dalec_frontend"
     }
     contexts = {
-      "dalec_frontend" = "target:frontend"
+        "dalec_frontend" = "target:frontend"
     }
     target = "${distro}/container"
     dockerfile = "docs/examples/${f}.yml"
@@ -197,7 +197,7 @@ dependencies:
         "BUILDKIT_SYNTAX" = "dalec_frontend"
     }
     contexts = {
-      "dalec_frontend" = "target:frontend"
+        "dalec_frontend" = "target:frontend"
     }
     target = "${distro}/container/depsonly"
     tags = ["local/dalec/deps-only:${distro}"]
@@ -231,4 +231,3 @@ target "frontend-ci-full" {
     inherits = ["frontend-ci"]
     platforms = ["linux/amd64", "linux/arm64"]
 }
-

packaging/linux/deb/debroot.go

@@ -41,11 +41,6 @@ var patchHeader []byte
 //go:embed templates/debian_install_header.sh
 var debianInstall []byte
 
-func sanitizeSourceKey(key string) string {
-	replacer := strings.NewReplacer("_", "", "-", "", ".", "")
-	return replacer.Replace(key)
-}
-
 // This creates a directory in the debian root directory for each patch, and copies the patch files into it.
 // The format for each patch dir matches what would normally be under `debian/patches`, just that this is a separate dir for every source we are patching
 // This is purely for documenting in the source package how patches are applied in a more readable way than the big merged patch file.
@@ -136,20 +131,12 @@ func Debroot(ctx context.Context, sOpt dalec.SourceOpts, spec *dalec.Spec, worke
 	installers := createInstallScripts(worker, spec, dir, target)
 
 	const (
-		sourceFormat  = "3.0 (quilt)"
-		sourceOptions = "create-empty-orig"
+		sourceFormat = "3.0 (quilt)"
 	)
 
 	debian := base.
 		File(llb.Mkdir(filepath.Join(dir, "source"), 0o755), opts...).
-		With(func(in llb.State) llb.State {
-			if len(spec.Sources) == 0 {
-				return in
-			}
-			return in.
-				File(llb.Mkfile(filepath.Join(dir, "source/format"), 0o640, []byte(sourceFormat)), opts...).
-				File(llb.Mkfile(filepath.Join(dir, "source/options"), 0o640, []byte(sourceOptions)), opts...)
-		}).
+		File(llb.Mkfile(filepath.Join(dir, "source/format"), 0o640, []byte(sourceFormat)), opts...).
 		File(llb.Mkdir(filepath.Join(dir, "dalec"), 0o755), opts...).
 		File(llb.Mkfile(filepath.Join(dir, "source/include-binaries"), 0o640, append([]byte("dalec"), '\n')), opts...)
 
@@ -161,7 +148,7 @@ func Debroot(ctx context.Context, sOpt dalec.SourceOpts, spec *dalec.Spec, worke
 
 	states = append(states, dalecDir.File(llb.Mkfile(filepath.Join(dir, "dalec/build.sh"), 0o700, createBuildScript(spec, &cfg)), opts...))
 	states = append(states, dalecDir.File(llb.Mkfile(filepath.Join(dir, "dalec/patch.sh"), 0o700, createPatchScript(spec, &cfg)), opts...))
-	states = append(states, dalecDir.File(llb.Mkfile(filepath.Join(dir, "dalec/fix_sources.sh"), 0o700, fixupSources(spec, &cfg)), opts...))
+	states = append(states, dalecDir.File(llb.Mkfile(filepath.Join(dir, "dalec/fix_generators.sh"), 0o700, fixupGenerators(spec, &cfg)), opts...))
 	states = append(states, dalecDir.File(llb.Mkfile(filepath.Join(dir, "dalec/fix_perms.sh"), 0o700, fixupArtifactPerms(spec, target, &cfg)), opts...))
 
 	customEnable, err := customDHInstallSystemdPostinst(spec, target)
@@ -294,33 +281,10 @@ func fixupArtifactPerms(spec *dalec.Spec, target string, cfg *SourcePkgConfig) [
 }
 
 // For debian sources
-//  1. File backed sources are not in the correct format as expected by dalec.
-//  2. Sources with certain characters in the name had to be changed, so we need
-//     to bring those back.
-//
 // This is called from `debian/rules` after the source tarball has been extracted.
-func fixupSources(spec *dalec.Spec, cfg *SourcePkgConfig) []byte {
+func fixupGenerators(spec *dalec.Spec, cfg *SourcePkgConfig) []byte {
 	buf := bytes.NewBuffer(nil)
 	writeScriptHeader(buf, cfg)
-
-	// now, we need to find all the sources that are file-backed and fix them up
-	for name, src := range spec.Sources {
-		dirName := sanitizeSourceKey(name)
-
-		if dalec.SourceIsDir(src) {
-			if dirName == name {
-				continue
-			}
-			fmt.Fprintf(buf, "mv '%s' '%s'\n", dirName, name)
-			continue
-		}
-
-		fmt.Fprintf(buf, "mv '%s/%s' '%s.dalec.tmp' || (ls -lh %q; exit 2)\n", dirName, name, name, dirName)
-		fmt.Fprintf(buf, "rm -rf '%s'\n", dirName)
-		fmt.Fprintf(buf, "mv '%s.dalec.tmp' '%s'\n", name, name)
-		fmt.Fprintln(buf)
-	}
-
 	if spec.HasGomods() {
 		// Older go versions did not have support for the `GOMODCACHE` var
 		// This is a hack to try and make the build work by linking the go modules

packaging/linux/deb/pkg.go

@@ -93,7 +93,6 @@ func SourcePackage(ctx context.Context, sOpt dalec.SourceOpts, worker llb.State,
 	}
 
 	patches := createPatches(spec, sources, worker, dr, opts...)
-	debSources := debSources(worker, sources)
 
 	work := worker.Run(
 		dalec.ShArgs("set -e; ls -lh; dpkg-buildpackage -S -us -uc; mkdir -p /tmp/out; ls -lh; cp -r /work/"+spec.Name+"_"+spec.Version+"* /tmp/out; ls -lh /tmp/out"),
@@ -102,18 +101,29 @@ func SourcePackage(ctx context.Context, sOpt dalec.SourceOpts, worker llb.State,
 		llb.AddMount("/work/pkg/debian/patches", patches, llb.Readonly),
 		llb.AddEnv("DH_VERBOSE", "1"),
 		dalec.RunOptFunc(func(ei *llb.ExecInfo) {
-			// Mount all the tar+gz'd sources into the build which will get picked p by debbuild
-			for key, src := range debSources {
-				tarName := fmt.Sprintf("%s_%s.orig-%s.tar.gz", spec.Name, spec.Version, sanitizeSourceKey(key))
-				llb.AddMount(filepath.Join("/work", tarName), src, llb.SourcePath("src.tar.gz"), llb.Readonly).SetRunOption(ei)
-			}
+			debSources := TarDebSources(worker, spec, sources, "src.tar.gz", sOpt, opts...)
+			llb.AddMount("/work/"+spec.Name+"_"+spec.Version+".orig.tar.gz", debSources, llb.SourcePath("src.tar.gz")).SetRunOption(ei)
 		}),
 		dalec.WithConstraints(opts...),
 	)
 
 	return work.AddMount("/tmp/out", llb.Scratch()), nil
 }
 
+func BuildDebBinaryOnly(worker llb.State, spec *dalec.Spec, debroot llb.State, distroVersionID string, opts ...llb.ConstraintsOpt) (llb.State, error) {
+	dirName := filepath.Join("/work", spec.Name+"_"+spec.Version+"-"+spec.Revision)
+	st := worker.
+		Run(
+			dalec.ShArgs("set -e; ls -lh; dpkg-buildpackage -b -uc -us; mkdir -p /tmp/out; cp ../*.deb /tmp/out; ls -lh /tmp/out"),
+			llb.Dir(dirName),
+			llb.AddEnv("DH_VERBOSE", "1"),
+			llb.AddMount(dirName, debroot),
+			dalec.WithConstraints(opts...),
+		).AddMount("/tmp/out", llb.Scratch())
+
+	return dalec.MergeAtPath(llb.Scratch(), []llb.State{st}, "/"), nil
+}
+
 func BuildDeb(worker llb.State, spec *dalec.Spec, srcPkg llb.State, distroVersionID string, opts ...llb.ConstraintsOpt) (llb.State, error) {
 
 	dirName := filepath.Join("/work", spec.Name+"_"+spec.Version+"-"+spec.Revision)
@@ -129,13 +139,43 @@ func BuildDeb(worker llb.State, spec *dalec.Spec, srcPkg llb.State, distroVersio
 	return dalec.MergeAtPath(llb.Scratch(), []llb.State{st, srcPkg}, "/"), nil
 }
 
-func debSources(worker llb.State, sources map[string]llb.State, opts ...llb.ConstraintsOpt) map[string]llb.State {
-	out := make(map[string]llb.State, len(sources))
+func TarDebSources(work llb.State, spec *dalec.Spec, srcStates map[string]llb.State, dest string, sOpts dalec.SourceOpts, opts ...llb.ConstraintsOpt) llb.State {
+	outBase := "/tmp/out"
+	out := filepath.Join(outBase, filepath.Dir(dest))
 
-	for key, src := range sources {
-		st := dalec.Tar(worker, src, "src.tar.gz", opts...)
-		out[key] = st
-	}
+	worker := work.Run(
+		llb.AddMount("/src", llb.Scratch()),
+		dalec.RunOptFunc(func(ei *llb.ExecInfo) {
+			for key, state := range srcStates {
+
+				mountOpts := []llb.MountOption{}
+				src, ok := spec.Sources[key]
+
+				// If the source is not explicitly listed in the spec sources, assume it is a directory (e.g., for gomod dependencies)
+				isDir := true
+				if ok {
+					isDir = dalec.SourceIsDir(src)
+				}
+
+				if !isDir {
+					mountOpts = append(mountOpts, llb.SourcePath(filepath.Join("/", key)))
+				}
+
+				// If the tar contains only a single directory, dpkg will extract its contents directly into the root directory.
+				mounthPath := filepath.Join("/src", key)
+				if len(srcStates) == 1 && isDir {
+					mounthPath = filepath.Join("/src", key, key)
+				}
+
+				llb.AddMount(mounthPath, state, mountOpts...).SetRunOption(ei)
+			}
+		}),
+		dalec.ShArgs("tar -C /src -cvzf /tmp/st ."),
+		dalec.WithConstraints(opts...),
+	).Run(
+		llb.Args([]string{"/bin/sh", "-c", "mkdir -p " + out + " && mv /tmp/st " + filepath.Join(out, filepath.Base(dest))}),
+		dalec.WithConstraints(opts...),
+	)
 
-	return out
+	return worker.AddMount(outBase, llb.Scratch())
 }

packaging/linux/deb/templates/debian_rules.tmpl

@@ -12,18 +12,12 @@ override_dh_auto_install:
 override_dh_auto_build:
 	./debian/dalec/build.sh
 
+.PHONY: fix_generators
 
-.PHONY: fix_sources
+fix_generators:
+	./debian/dalec/fix_generators.sh
 
-# Move file-backed (Dalec) sources into the correct location
-# Mainly this is needed because with the way the debian sources are setup, our file-backed sources
-# end up getting placed in a sub-directory, e.g. for source "foo", we end up with "foo/foo".
-# For the purposes of making dalec builds work across different platforms, we need to move these so it is only a file, "foo".
-fix_sources:
-	./debian/dalec/fix_sources.sh
-
-
-override_dh_auto_configure: fix_sources
+override_dh_auto_configure: fix_generators
 	@# Stop debhelper from running auto configure, which should be part of the build steps
 
 # dh_dwz is problematic for (some) go builds and causes the build to error out.
@@ -38,4 +32,3 @@ override_dh_dwz:
 
 %:
 	dh $@ -v
-

targets/linux/deb/distro/install.go

@@ -139,12 +139,12 @@ func (d *Config) InstallBuildDeps(sOpt dalec.SourceOpts, spec *dalec.Spec, targe
 			opts := append(opts, dalec.ProgressGroup("Install build dependencies"))
 			opts = append([]llb.ConstraintsOpt{dalec.WithConstraint(c)}, opts...)
 
-			srcPkg, err := deb.SourcePackage(ctx, sOpt, in, depsSpec, targetKey, "", deb.SourcePkgConfig{}, opts...)
+			debRoot, err := deb.Debroot(ctx, sOpt, depsSpec, in, llb.Scratch(), targetKey, "", d.VersionID, deb.SourcePkgConfig{}, opts...)
 			if err != nil {
 				return in, err
 			}
 
-			pkg, err := deb.BuildDeb(in, depsSpec, srcPkg, "", opts...)
+			pkg, err := deb.BuildDebBinaryOnly(in, depsSpec, debRoot, "", opts...)
 			if err != nil {
 				return in, errors.Wrap(err, "error creating intermediate package for installing build dependencies")
 			}