Skip to content

Commit aff98e0

Browse files
committed
Extract some common thigns into helpers
- ShArgs - run a command in a shell - Sources - Iterate spec sources and get states - Tar - Make a tar from an llb.State Also fixes some typos. Signed-off-by: Brian Goff <[email protected]>
1 parent 8039060 commit aff98e0

15 files changed

+105
-108
lines changed

frontend/azlinux/handle_container.go

Lines changed: 2 additions & 2 deletions
Original file line numberDiff line numberDiff line change
@@ -130,14 +130,14 @@ func readRPMs(ctx context.Context, client gwclient.Client, st llb.State) ([]stri
130130
return out, nil
131131
}
132132

133-
func specToContainerLLB(w worker, client gwclient.Client, spec *dalec.Spec, target string, rpmDir llb.State, files []string, sOpt dalec.SourceOpts, opts ...llb.ConstraintsOpt) (llb.State, error) {
133+
func specToContainerLLB(w worker, client gwclient.Client, spec *dalec.Spec, targetKey string, rpmDir llb.State, files []string, sOpt dalec.SourceOpts, opts ...llb.ConstraintsOpt) (llb.State, error) {
134134
opts = append(opts, dalec.ProgressGroup("Install RPMs"))
135135
const workPath = "/tmp/rootfs"
136136

137137
builderImg := w.Base(client, opts...)
138138

139139
rootfs := llb.Scratch()
140-
if ref := dalec.GetBaseOutputImage(spec, target); ref != "" {
140+
if ref := dalec.GetBaseOutputImage(spec, targetKey); ref != "" {
141141
rootfs = llb.Image(ref, llb.WithMetaResolver(sOpt.Resolver), dalec.WithConstraints(opts...))
142142
}
143143

frontend/azlinux/handle_depsonly.go

Lines changed: 1 addition & 1 deletion
Original file line numberDiff line numberDiff line change
@@ -17,7 +17,7 @@ func handleDepsOnly(w worker) gwclient.BuildFunc {
1717
pg := dalec.ProgressGroup("Build mariner2 deps-only container: " + spec.Name)
1818
baseImg := w.Base(client, pg)
1919
rpmDir := baseImg.Run(
20-
shArgs(`set -ex; dir="/tmp/rpms/RPMS/$(uname -m)"; mkdir -p "${dir}"; tdnf install -y --releasever=2.0 --downloadonly --alldeps --downloaddir "${dir}" `+strings.Join(spec.GetRuntimeDeps(targetKey), " ")),
20+
dalec.ShArgs(`set -ex; dir="/tmp/rpms/RPMS/$(uname -m)"; mkdir -p "${dir}"; tdnf install -y --releasever=2.0 --downloadonly --alldeps --downloaddir "${dir}" `+strings.Join(spec.GetRuntimeDeps(targetKey), " ")),
2121
pg,
2222
).
2323
AddMount("/tmp/rpms", llb.Scratch())

frontend/azlinux/handle_rpm.go

Lines changed: 0 additions & 4 deletions
Original file line numberDiff line numberDiff line change
@@ -52,10 +52,6 @@ func handleRPM(w worker) gwclient.BuildFunc {
5252
}
5353
}
5454

55-
func shArgs(cmd string) llb.RunOption {
56-
return llb.Args([]string{"sh", "-c", cmd})
57-
}
58-
5955
func installBuildDeps(w worker, spec *dalec.Spec, targetKey string, opts ...llb.ConstraintsOpt) llb.StateOption {
6056
return func(in llb.State) llb.State {
6157
deps := spec.GetBuildDeps(targetKey)

frontend/azlinux/install.go

Lines changed: 1 addition & 1 deletion
Original file line numberDiff line numberDiff line change
@@ -117,6 +117,6 @@ func tdnfInstall(cfg *installConfig, relVer string, pkgs []string) llb.RunOption
117117
cmdArgs += "; " + manifestPath
118118
}
119119

120-
runOpts = append(runOpts, shArgs(cmdArgs))
120+
runOpts = append(runOpts, dalec.ShArgs(cmdArgs))
121121
return dalec.WithRunOptions(runOpts...)
122122
}

frontend/debug/handle_sources.go

Lines changed: 4 additions & 8 deletions
Original file line numberDiff line numberDiff line change
@@ -19,16 +19,12 @@ func Sources(ctx context.Context, client gwclient.Client) (*client.Result, error
1919
return nil, nil, err
2020
}
2121

22-
sources := make([]llb.State, 0, len(spec.Sources))
23-
for name, src := range spec.Sources {
24-
st, err := src.AsState(name, sOpt)
25-
if err != nil {
26-
return nil, nil, err
27-
}
28-
sources = append(sources, st)
22+
sources, err := dalec.Sources(spec, sOpt)
23+
if err != nil {
24+
return nil, nil, err
2925
}
3026

31-
def, err := dalec.MergeAtPath(llb.Scratch(), sources, "/").Marshal(ctx)
27+
def, err := dalec.MergeAtPath(llb.Scratch(), dalec.SortedMapValues(sources), "/").Marshal(ctx)
3228
if err != nil {
3329
return nil, nil, err
3430
}

frontend/rpm/handle_sources.go

Lines changed: 24 additions & 54 deletions
Original file line numberDiff line numberDiff line change
@@ -3,7 +3,6 @@ package rpm
33
import (
44
"context"
55
"fmt"
6-
"path/filepath"
76

87
"github.com/Azure/dalec"
98
"github.com/Azure/dalec/frontend"
@@ -13,35 +12,6 @@ import (
1312
"github.com/pkg/errors"
1413
)
1514

16-
// TarImageRef is the image used to create tarballs of sources
17-
// This is purposefully exported so it can be overridden at compile time if needed.
18-
// Currently this image needs /bin/sh and tar in $PATH
19-
var TarImageRef = "busybox:latest"
20-
21-
func shArgs(cmd string) llb.RunOption {
22-
return llb.Args([]string{"sh", "-c", cmd})
23-
}
24-
25-
func tar(src llb.State, dest string, opts ...llb.ConstraintsOpt) llb.State {
26-
tarImg := llb.Image(TarImageRef, dalec.WithConstraints(opts...))
27-
28-
// Put the output tar in a consistent location regardless of `dest`
29-
// This way if `dest` changes we don't have to rebuild the tarball, which can be expensive.
30-
outBase := "/tmp/out"
31-
out := filepath.Join(outBase, filepath.Dir(dest))
32-
worker := tarImg.Run(
33-
llb.AddMount("/src", src, llb.Readonly),
34-
shArgs("tar -C /src -cvzf /tmp/st ."),
35-
dalec.WithConstraints(opts...),
36-
).
37-
Run(
38-
shArgs("mkdir -p "+out+" && mv /tmp/st "+filepath.Join(out, filepath.Base(dest))),
39-
dalec.WithConstraints(opts...),
40-
)
41-
42-
return worker.AddMount(outBase, llb.Scratch())
43-
}
44-
4515
func HandleSources(wf WorkerFunc) gwclient.BuildFunc {
4616
return func(ctx context.Context, client gwclient.Client) (*gwclient.Result, error) {
4717
return frontend.BuildWithPlatform(ctx, client, func(ctx context.Context, client gwclient.Client, platform *ocispecs.Platform, spec *dalec.Spec, targetKey string) (gwclient.Reference, *dalec.DockerImageSpec, error) {
@@ -85,39 +55,39 @@ func HandleSources(wf WorkerFunc) gwclient.BuildFunc {
8555
}
8656

8757
func Dalec2SourcesLLB(worker llb.State, spec *dalec.Spec, sOpt dalec.SourceOpts, opts ...llb.ConstraintsOpt) ([]llb.State, error) {
88-
// Sort the map keys so that the order is consistent This shouldn't be
89-
// needed when MergeOp is supported, but when it is not this will improve
90-
// cache hits for callers of this function.
91-
sorted := dalec.SortMapKeys(spec.Sources)
92-
93-
out := make([]llb.State, 0, len(spec.Sources))
94-
95-
for _, k := range sorted {
96-
src := spec.Sources[k]
97-
isDir := dalec.SourceIsDir(src)
98-
99-
pg := dalec.ProgressGroup("Add spec source: " + k)
100-
st, err := src.AsState(k, sOpt, append(opts, pg)...)
101-
if err != nil {
102-
return nil, err
103-
}
58+
sources, err := dalec.Sources(spec, sOpt)
59+
if err != nil {
60+
return nil, err
61+
}
62+
out := make([]llb.State, 0, len(sources))
10463

105-
if isDir {
106-
out = append(out, tar(st, k+".tar.gz", append(opts, pg)...))
107-
} else {
108-
out = append(out, st)
109-
}
64+
withPG := func(s string) []llb.ConstraintsOpt {
65+
return append(opts, dalec.ProgressGroup(s))
11066
}
11167

112-
opts = append(opts, dalec.ProgressGroup("Add gomod sources"))
113-
st, err := spec.GomodDeps(sOpt, worker, opts...)
68+
st, err := spec.GomodDeps(sOpt, worker, withPG("Add gomod sources")...)
11469
if err != nil {
11570
return nil, errors.Wrap(err, "error adding gomod sources")
11671
}
11772

73+
sorted := dalec.SortMapKeys(sources)
74+
for _, k := range sorted {
75+
st := sources[k]
76+
if dalec.SourceIsDir(spec.Sources[k]) {
77+
st = st.With(sourceTar(worker, k, withPG("Tar source: "+k)...))
78+
}
79+
out = append(out, st)
80+
}
81+
11882
if st != nil {
119-
out = append(out, tar(*st, gomodsName+".tar.gz", opts...))
83+
out = append(out, st.With(sourceTar(worker, gomodsName, withPG("Tar gomod deps")...)))
12084
}
12185

12286
return out, nil
12387
}
88+
89+
func sourceTar(worker llb.State, key string, opts ...llb.ConstraintsOpt) llb.StateOption {
90+
return func(in llb.State) llb.State {
91+
return dalec.Tar(worker, in, key+".tar.gz", opts...)
92+
}
93+
}

frontend/rpm/rpmbuild.go

Lines changed: 1 addition & 1 deletion
Original file line numberDiff line numberDiff line change
@@ -29,7 +29,7 @@ func Build(topDir, workerImg llb.State, specPath string, opts ...llb.Constraints
2929
// TODO(cpuguy83): specPath would ideally never change.
3030
// We don't want to have to re-run this step just because the path changed, this should be based on inputs only (ie the content of the rpm spec, sources, etc)
3131
// The path should not be an input.
32-
shArgs(`rpmbuild --define "_topdir /build/top" --define "_srcrpmdir /build/out/SRPMS" --define "_rpmdir /build/out/RPMS" --buildroot /build/tmp/work -ba `+specPath),
32+
dalec.ShArgs(`rpmbuild --define "_topdir /build/top" --define "_srcrpmdir /build/out/SRPMS" --define "_rpmdir /build/out/RPMS" --buildroot /build/tmp/work -ba `+specPath),
3333
llb.AddMount("/build/top", topDir),
3434
llb.AddMount("/build/tmp", llb.Scratch(), llb.Tmpfs()),
3535
llb.Dir("/build/top"),

frontend/windows/handle_zip.go

Lines changed: 7 additions & 22 deletions
Original file line numberDiff line numberDiff line change
@@ -60,20 +60,9 @@ func handleZip(ctx context.Context, client gwclient.Client) (*gwclient.Result, e
6060
const gomodsName = "__gomods"
6161

6262
func specToSourcesLLB(worker llb.State, spec *dalec.Spec, sOpt dalec.SourceOpts, opts ...llb.ConstraintsOpt) (map[string]llb.State, error) {
63-
out := make(map[string]llb.State, len(spec.Sources))
64-
for k, src := range spec.Sources {
65-
displayRef, err := src.GetDisplayRef()
66-
if err != nil {
67-
return nil, err
68-
}
69-
70-
pg := dalec.ProgressGroup("Add spec source: " + k + " " + displayRef)
71-
st, err := src.AsState(k, sOpt, append(opts, pg)...)
72-
if err != nil {
73-
return nil, errors.Wrapf(err, "error creating source state for %q", k)
74-
}
75-
76-
out[k] = st
63+
out, err := dalec.Sources(spec, sOpt, opts...)
64+
if err != nil {
65+
return nil, errors.Wrap(err, "error preparign spec sources")
7766
}
7867

7968
opts = append(opts, dalec.ProgressGroup("Add gomod sources"))
@@ -99,7 +88,7 @@ func installBuildDeps(deps []string) llb.StateOption {
9988
slices.Sort(sorted)
10089

10190
return s.Run(
102-
shArgs("apt-get update && apt-get install -y "+strings.Join(sorted, " ")),
91+
dalec.ShArgs("apt-get update && apt-get install -y "+strings.Join(sorted, " ")),
10392
dalec.WithMountedAptCache(aptCachePrefix),
10493
).Root()
10594
}
@@ -148,7 +137,7 @@ func buildBinaries(ctx context.Context, spec *dalec.Spec, worker llb.State, clie
148137
script := generateInvocationScript(binaries)
149138

150139
st := worker.Run(
151-
shArgs(script.String()),
140+
dalec.ShArgs(script.String()),
152141
llb.Dir("/build"),
153142
withSourcesMounted("/build", patched, spec.Sources),
154143
llb.AddMount("/tmp/scripts", buildScript),
@@ -170,7 +159,7 @@ func buildBinaries(ctx context.Context, spec *dalec.Spec, worker llb.State, clie
170159
func getZipLLB(worker llb.State, name string, artifacts llb.State) llb.State {
171160
outName := filepath.Join(outputDir, name+".zip")
172161
zipped := worker.Run(
173-
shArgs("zip "+outName+" *"),
162+
dalec.ShArgs("zip "+outName+" *"),
174163
llb.Dir("/tmp/artifacts"),
175164
llb.AddMount("/tmp/artifacts", artifacts),
176165
).AddMount(outputDir, llb.Scratch())
@@ -192,15 +181,11 @@ func workerImg(sOpt dalec.SourceOpts, opts ...llb.ConstraintsOpt) llb.State {
192181
// TODO: support named context override... also this should possibly be its own image, maybe?
193182
return llb.Image(workerImgRef, llb.WithMetaResolver(sOpt.Resolver), dalec.WithConstraints(opts...)).
194183
Run(
195-
shArgs("apt-get update && apt-get install -y build-essential binutils-mingw-w64 g++-mingw-w64-x86-64 gcc git make pkg-config quilt zip"),
184+
dalec.ShArgs("apt-get update && apt-get install -y build-essential binutils-mingw-w64 g++-mingw-w64-x86-64 gcc git make pkg-config quilt zip"),
196185
dalec.WithMountedAptCache(aptCachePrefix),
197186
).Root()
198187
}
199188

200-
func shArgs(cmd string) llb.RunOption {
201-
return llb.Args([]string{"sh", "-c", cmd})
202-
}
203-
204189
func createBuildScript(spec *dalec.Spec) llb.State {
205190
buf := bytes.NewBuffer(nil)
206191

generator_gomod.go

Lines changed: 1 addition & 1 deletion
Original file line numberDiff line numberDiff line change
@@ -38,7 +38,7 @@ func withGomod(g *SourceGenerator, srcSt, worker llb.State, opts ...llb.Constrai
3838
srcMount = llb.AddMount(workDir, srcSt)
3939
}
4040
return worker.Run(
41-
shArgs("go mod download"),
41+
ShArgs("go mod download"),
4242
llb.AddEnv("GOMODCACHE", gomodCacheDir),
4343
llb.Dir(workDir),
4444
srcMount,

helpers.go

Lines changed: 12 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -385,3 +385,15 @@ func (s *Spec) GetSigner(targetKey string) (*PackageSigner, bool) {
385385
func hasValidSigner(pc *PackageConfig) bool {
386386
return pc != nil && pc.Signer != nil && pc.Signer.Image != ""
387387
}
388+
389+
// SortMapValues is like [maps.Values], but the list is sorted based on the map key
390+
func SortedMapValues[T any](m map[string]T) []T {
391+
keys := SortMapKeys(m)
392+
393+
out := make([]T, 0, len(keys))
394+
for _, k := range keys {
395+
out = append(out, m[k])
396+
}
397+
398+
return out
399+
}

0 commit comments

Comments
 (0)