diff --git a/.go-env.sh b/.go-env.sh index 7d46576..62c5860 100755 --- a/.go-env.sh +++ b/.go-env.sh @@ -79,25 +79,6 @@ if [ -z "${GOLANG_IMAGE:-}" ]; then go="${go%.*}" # strip to just X.Y fi GOLANG_IMAGE="golang:$go" - - # handle riscv64 "gracefully" (no golang image yet because no stable distro releases yet) - { - if ! docker image inspect --format '.' "$GOLANG_IMAGE" &> /dev/null && ! docker pull "$GOLANG_IMAGE"; then - if [ -n "${BASHBREW_ARCH:-}" ] && docker buildx inspect "bashbrew-$BASHBREW_ARCH" &> /dev/null; then - # a very rough hack to avoid: - # ERROR: failed to solve: failed to solve with frontend dockerfile.v0: failed to read dockerfile: failed to load cache key: subdir not supported yet - # (we need buildkit/buildx for --build-context, but newer buildkit than our dockerd might have for build-from-git-with-subdir) - export BUILDX_BUILDER="bashbrew-$BASHBREW_ARCH" - fi - ( - set -x - # TODO make this more dynamic, less hard-coded 🙈 - # https://github.com/docker-library/golang/blob/ea6bbce8c9b13acefed0f5507336be01f0918f97/1.21/bookworm/Dockerfile - GOLANG_IMAGE='golang:1.21' # to be explicit - docker buildx build --load --tag "$GOLANG_IMAGE" --build-context 'buildpack-deps:bookworm-scm=docker-image://buildpack-deps:unstable-scm' 'https://github.com/docker-library/golang.git#ea6bbce8c9b13acefed0f5507336be01f0918f97:1.21/bookworm' - ) - fi - } >&2 fi args+=( diff --git a/cmd/holes/README.md b/cmd/holes/README.md new file mode 100644 index 0000000..b08d081 --- /dev/null +++ b/cmd/holes/README.md @@ -0,0 +1,43 @@ +# cmd/holes + +`builds.json` is regenerated from scratch on every meta run. A **hole** is any `(tag, arch)` pair that appears in `sources.json` but has no currently-resolved build in `builds.json`. This happens in two distinct ways: + +- **`resolved: null`** -- the entry exists (parents are all resolved, buildId is computed) but the staging image hasn't been built yet. +- **Entirely absent** -- `cmd/builds` emitted nothing for this `(sourceId, arch)` because a DOI parent's staging image is itself unresolved (`close(outChan)`). The entry cannot appear in `builds.json` at all until the parent resolves. + +`builds.json` cannot be used alone to detect entirely-absent holes; `sources.json` is required as the authoritative list of what *should* exist. + +## Why holes matter + +`library/IMAGE:TAG` is a single atomic OCI index listing all arches. Pushing any update to it requires supplying descriptors for *every* expected arch -- you cannot leave one arch out without dropping it from the index entirely. Meanwhile arches build at very different rates (hours for amd64, days for riscv64), so "wait for all arches before publishing" is not viable. Holes must be filled with the most recent previously-resolved content for each `(tag, arch)` while the new build is in flight. + +## Key constraints on the search key + +Holes are independent per `(tag, arch)` -- not per `(sourceId, arch)`. The reasons: + +- **`sourceId` is not stable across re-adds.** A tag removed from the library and re-added later may have a new sourceId (changed Dockerfile), breaking any lookup keyed on sourceId. The tag name is what users pull; it is the right identity. +- **Each tag is a distinct semantic identity.** `"golang:1.22"` and `"golang:1.22.6"` share a builds.json entry when both point at the same build, but they must be looked up independently: `"golang:1.22.6"` should never serve a fallback from `"golang:1.22.5"`, even though `"golang:1.22"` should. + +Tags come from `source.arches[arch].tags`, not `archTags` -- `archTags` may be empty in non-DOI repos (e.g. `debuerreotype/debuerreotype:latest` rather than `amd64/debian:bookworm`). + +## Known imprecision: multi-variant arches + +For architectures with multiple OS-version variants under the same tag (e.g. `windows-amd64` with ltsc2019 and ltsc2022 both contributing to `"docker:24"`), `(tag, arch)` conflates all variants. There is no clean way to track which specific variant is the hole without keying by sourceId -- which breaks re-added tags. The fallback found in history may correspond to the wrong OS version. This is accepted imprecision. + +## Git history search + +The git history of the meta repo contains every prior `builds.json` state. Walking it backward is the only source of fallback data that does not require live registry queries (which are expensive enough that put-shared runs only every ~3 hours). The walk must be a single linear backward pass over *all* holes simultaneously -- exponential or binary search is not safe because a tag can be added, fully built, *and* removed within a gap of skipped commits, leaving no evidence on either side of the jump. + +## Performance + +Blob enumeration uses `git log --raw` (a single native-git command) rather than go-git's commit-graph traversal, which decompresses every commit and tree object in-process and is substantially slower. + +Blobs are parsed in parallel (`runtime.NumCPU()` workers, each with its own `git cat-file --batch` subprocess to avoid go-git's internal pack-file mutex). A coordinator goroutine cancels the scan the moment every initially-unfilled hole has been filled, so recently-built tags cost only a few blobs' worth of work. + +An optional third argument `prev-holes.json` pre-fills holes from the previous run's output before touching git history. The previous `holes.json` is a valid cache because any `(tag, arch)` that was a hole on the prior run and is still a hole now can reuse the same fallback OCI index -- and as holes get resolved during builds they disappear from the current holes set (resolved in `builds.json`) so the cache never returns data for a hole that no longer exists. On a warm run (most holes already in the cache) the git history search only covers newly-appeared holes, which are typically a small fraction of the total. Typical invocation: + +```console +$ holes sources.json builds.json holes.json > holes-new.json && mv holes-new.json holes.json +``` + +Setting `$HOLES_SINCE` to any expression `git log --since` understands (eg `2.weeks`, `30.days`, `2024-01-01`) limits the history search to that window. Holes still unfilled after the window are left empty (library deploy skips them). This trades correctness -- a tag last resolved before the cutoff gets no fallback -- for a bounded worst-case runtime. It is most useful when brand-new tags (which can never be filled from history) regularly appear in large batches and would otherwise force a full scan by preventing the all-holes-filled early exit. It composes well with the cache: the cache handles previously-known holes instantly, and `$HOLES_SINCE` bounds the search for any new ones. diff --git a/cmd/holes/main.go b/cmd/holes/main.go new file mode 100644 index 0000000..d7e13ea --- /dev/null +++ b/cmd/holes/main.go @@ -0,0 +1,518 @@ +package main + +import ( + "bufio" + "context" + "encoding/json" + "fmt" + "io" + "math" + "os" + "os/exec" + "path/filepath" + "runtime" + "strconv" + "strings" + "sync" + + // encoding/json/v2, but not yet + jsonv2 "github.com/go-json-experiment/json" + "github.com/go-json-experiment/json/jsontext" + + ocispec "github.com/opencontainers/image-spec/specs-go/v1" +) + +// hole represents a single (tag, arch) pair with no resolved entry in the current builds.json. Each (tag, arch) is an independent hole: "golang:1.22" on amd64 and "golang:1.22.6" on amd64 have completely different historical content and must be looked up separately. +// +// "tag" is a value from source.arches[arch].tags (e.g. "golang:1.22" or "debuerreotype/debuerreotype:latest") -- NOT the arch-prefixed archTag, which may be empty in non-DOI repos. +// +// Note: for architectures with multiple OS-version variants (e.g. windows-amd64 with ltsc2019 and ltsc2022), the (tag, arch) key conflates all variants. We accept this imprecision; there is no clean way to track partial holes within a single (tag, arch) pair. +type hole struct { + tag string + arch string +} + +// tagArchKey returns the map key used in keyToHole. A null byte is safe since neither tags nor arch names ever contain one. +func tagArchKey(tag, arch string) string { + return tag + "\x00" + arch +} + +// blobMatch is a single resolved entry found in a historical builds.json blob that matches an unfilled hole. +type blobMatch struct { + slotIdx int // position in newest-to-oldest order (lower = more recent) + holeIdx int + resolved *ocispec.Index +} + +func main() { + if len(os.Args) < 3 { + fmt.Fprintf(os.Stderr, "usage: %s sources.json builds.json [prev-holes.json]\n", os.Args[0]) + os.Exit(1) + } + sourcesFile := os.Args[1] + buildsFile := os.Args[2] + prevHolesFile := "" + if len(os.Args) >= 4 { + prevHolesFile = os.Args[3] + } + + // HOLES_SINCE, if set, is passed directly to "git log --since=..." to limit how far back the + // history search reaches; accepts any expression git understands (e.g. "2.weeks", "30.days", + // "2024-01-01"); see README for trade-offs + holesSince := os.Getenv("HOLES_SINCE") + + // --- Step 1: parse sources.json for the full expected (sourceId, arch) set --- + + sourcesF, err := os.Open(sourcesFile) + if err != nil { + panic(err) + } + // sources.json is an object whose values each have a sourceId and an arches map; we need sourceId + arches[*].tags (archTags may be empty) + var sources map[string]struct { + SourceID string `json:"sourceId"` + Arches map[string]struct { + Tags []string `json:"tags"` + } `json:"arches"` + } + if err := json.NewDecoder(sourcesF).Decode(&sources); err != nil { + panic(err) + } + sourcesF.Close() + + // --- Step 2: parse builds.json to find which (sourceId, arch) are resolved --- + + buildsF, err := os.Open(buildsFile) + if err != nil { + panic(err) + } + // we only need sourceId, arch, and whether resolved is non-null + var builds map[string]struct { + Build struct { + SourceID string `json:"sourceId"` + Arch string `json:"arch"` + Resolved *ocispec.Index `json:"resolved"` + } `json:"build"` + } + if err := json.NewDecoder(buildsF).Decode(&builds); err != nil { + panic(err) + } + buildsF.Close() + + resolvedSet := make(map[string]bool, len(builds)) // key: sourceId+"-"+arch + for _, entry := range builds { + if entry.Build.Resolved != nil { + resolvedSet[entry.Build.SourceID+"-"+entry.Build.Arch] = true + } + } + + // --- Step 3: compute holes --- + // a hole is any (tag, arch) from sources.json where the backing (sourceId, arch) has no resolved entry in builds.json -- whether entirely absent (unresolved parent chain) or present with resolved == null + // each (tag, arch) is independent: "golang:1.22" may have a historical fallback while "golang:1.22.6" does not + + var holes []hole + // keyToHoles maps tagArchKey(tag, arch) to the list of indices in holes that share that (tag, arch). Normally exactly one entry; multiple entries arise when several sources share the same (tag, arch) -- e.g. windows-amd64 with ltsc2019 AND ltsc2022 both contributing to "docker:24". + // + // Known, accepted imprecision: because we can only search git history by (tag, arch), we cannot tell which specific OS-version variant is the hole, so we may fill slots with fallbacks for the wrong variant. + keyToHoles := map[string][]int{} + + for _, src := range sources { + for arch, archData := range src.Arches { + if resolvedSet[src.SourceID+"-"+arch] { + continue + } + for _, tag := range archData.Tags { + k := tagArchKey(tag, arch) + idx := len(holes) + holes = append(holes, hole{tag: tag, arch: arch}) + keyToHoles[k] = append(keyToHoles[k], idx) + } + } + } + + if len(holes) == 0 { + fmt.Println("{}") + return + } + fmt.Fprintf(os.Stderr, "searching git history for %d hole(s)\n", len(holes)) + + // results[i] holds the best resolved OCI index found for holes[i]; nil means unfilled. + // bestSlot[i] tracks the slot index (lower = more recent) of the result in holes[i]: + // math.MaxInt = no result yet (unfilled); counted in initialRemaining; git will search for it + // math.MaxInt-1 = filled from prev-holes.json cache; not counted; git can still improve it + // math.MaxInt-2 = cache recorded null (previously searched, not found); not counted; git can + // still passively improve it if a match is encountered while scanning for others + // 0..N = filled from git history blob at that slot index + results := make([]*ocispec.Index, len(holes)) + bestSlot := make([]int, len(holes)) + for i := range bestSlot { + bestSlot[i] = math.MaxInt + } + + // --- Step 3.5: pre-fill holes from previous holes.json (cache) --- + // The previous run's holes.json is a valid cache: any (tag, arch) that was a hole then + // and is still a hole now can reuse the same fallback resolved index without re-scanning + // git history. Holes that have since been filled (resolved in builds.json) are no longer + // in the current holes set, so the cache never returns data for a hole that no longer exists. + // + // For the windows multi-variant case where multiple holes share the same (tag, arch) key, + // cached entries are distributed across the current holes for that key in order. + + if prevHolesFile != "" { + var prevHoles map[string]map[string][]*ocispec.Index + f, err := os.Open(prevHolesFile) + if err != nil { + fmt.Fprintf(os.Stderr, "warning: could not open %s: %v\n", prevHolesFile, err) + } else { + if err := json.NewDecoder(f).Decode(&prevHoles); err != nil { + fmt.Fprintf(os.Stderr, "warning: ignoring malformed %s: %v\n", prevHolesFile, err) + } + f.Close() + } + + cacheHits := 0 + cacheNulls := 0 + for tag, arches := range prevHoles { + for arch, cachedIndices := range arches { + holeIndices := keyToHoles[tagArchKey(tag, arch)] + if cachedIndices == nil { + // null in previous holes.json: was searched and not found; skip git search + // (git can still passively improve these if a match is encountered for other holes) + for _, holeIdx := range holeIndices { + bestSlot[holeIdx] = math.MaxInt - 2 + cacheNulls++ + } + continue + } + for j, holeIdx := range holeIndices { + if j >= len(cachedIndices) || cachedIndices[j] == nil { + break + } + results[holeIdx] = cachedIndices[j] + bestSlot[holeIdx] = math.MaxInt - 1 // filled from cache; git can improve + cacheHits++ + } + } + } + if cacheHits > 0 || cacheNulls > 0 { + fmt.Fprintf(os.Stderr, "pre-filled %d hole(s) from %s (%d unfillable skipped)\n", cacheHits, prevHolesFile, cacheNulls) + } + } + + // count holes with no cache result at all (math.MaxInt); math.MaxInt-1 (filled) and + // math.MaxInt-2 (null/unfillable) are both excluded -- neither needs a git search + initialRemaining := 0 + for _, s := range bestSlot { + if s == math.MaxInt { + initialRemaining++ + } + } + + if initialRemaining == 0 { + fmt.Fprintf(os.Stderr, "all holes accounted for by cache; skipping git history search\n") + } else { + // --- Step 4: locate the git repo that contains builds.json --- + // Use native git rather than go-git: "git rev-parse --show-toplevel" is fast and handles + // DetectDotGit automatically. + + absBuilds, err := filepath.Abs(buildsFile) + if err != nil { + panic(err) + } + rootCmd := exec.Command("git", "rev-parse", "--show-toplevel") + rootCmd.Dir = filepath.Dir(absBuilds) + rootOut, err := rootCmd.Output() + if err != nil { + panic(fmt.Errorf("locating git repo root: %w", err)) + } + repoRoot := strings.TrimRight(string(rootOut), "\r\n") + relBuilds, err := filepath.Rel(repoRoot, absBuilds) + if err != nil { + panic(err) + } + + // --- Step 5a: collect ordered unique blob hashes via git log --- + // "git log --raw" emits one diff-stat line per commit showing old and new blob hashes; + // we extract the new blob hash for each version of builds.json in newest-to-oldest order. + // This is a single fast git command rather than go-git decompressing every commit+tree object. + + type blobSlot struct { + idx int + blobHash string // 40-char hex + } + + logArgs := []string{"log", "--raw", "--no-abbrev", "--format="} + if holesSince != "" { + logArgs = append(logArgs, "--since="+holesSince) + } + logArgs = append(logArgs, "--", relBuilds) + logCmd := exec.Command("git", logArgs...) + logCmd.Dir = repoRoot + logOut, err := logCmd.StdoutPipe() + if err != nil { + panic(err) + } + if err := logCmd.Start(); err != nil { + panic(err) + } + + var slots []blobSlot + seenBlob := make(map[string]bool) + + logScanner := bufio.NewScanner(logOut) + for logScanner.Scan() { + line := logScanner.Text() + if !strings.HasPrefix(line, ":") { + continue // blank lines and non-diff lines + } + parts := strings.Fields(line) + if len(parts) < 4 { + continue + } + newBlob := parts[3] + if newBlob == "0000000000000000000000000000000000000000" { + continue // deletion: builds.json was removed in this commit + } + if seenBlob[newBlob] { + continue // identical blob already queued + } + seenBlob[newBlob] = true + slots = append(slots, blobSlot{len(slots), newBlob}) + } + if err := logScanner.Err(); err != nil { + panic(err) + } + if err := logCmd.Wait(); err != nil { + panic(fmt.Errorf("git log: %w", err)) + } + + if holesSince != "" { + fmt.Fprintf(os.Stderr, "note: history search limited to builds since %q ($HOLES_SINCE); %d hole(s) may go unfilled\n", holesSince, initialRemaining) + } + fmt.Fprintf(os.Stderr, "scanning %d unique blob version(s) of %s\n", len(slots), relBuilds) + + // --- Step 5b: parse blobs in parallel with early exit --- + // Each worker drives its own "git cat-file --batch" process (no go-git internal mutex). + // A coordinator goroutine receives all matches and cancels the context once every + // initially-unfilled hole has been filled; the feeder goroutine stops immediately. + // Cache-filled holes (bestSlot == math.MaxInt-1) are not counted in initialRemaining + // but can still be improved to a more recent git entry for free. + + ctx, cancel := context.WithCancel(context.Background()) + defer cancel() + + matches := make(chan blobMatch, 256) + + // coordinator: single goroutine owns bestSlot/results (no locks needed), cancels when done + var coordDone sync.WaitGroup + coordDone.Add(1) + go func() { + defer coordDone.Done() + remaining := initialRemaining + for m := range matches { + if m.slotIdx < bestSlot[m.holeIdx] { + if bestSlot[m.holeIdx] == math.MaxInt { + remaining-- + if remaining == 0 { + cancel() // all initially-unfilled holes filled; stop feeder and workers + } + } + bestSlot[m.holeIdx] = m.slotIdx + results[m.holeIdx] = m.resolved + } + } + }() + + numWorkers := runtime.NumCPU() + if numWorkers > len(slots) { + numWorkers = len(slots) + } + + jobs := make(chan blobSlot, numWorkers) + + // feeder: stops the moment ctx is cancelled (all initially-unfilled holes filled) + go func() { + defer close(jobs) + for _, slot := range slots { + select { + case jobs <- slot: + case <-ctx.Done(): + return + } + } + }() + + var wg sync.WaitGroup + for w := range numWorkers { + wg.Add(1) + go func(workerID int) { + defer wg.Done() + + cmd := exec.Command("git", "cat-file", "--batch") + cmd.Dir = repoRoot + catIn, err := cmd.StdinPipe() + if err != nil { + fmt.Fprintf(os.Stderr, "warning: worker %d: creating git cat-file stdin pipe: %v\n", workerID, err) + return + } + catOut, err := cmd.StdoutPipe() + if err != nil { + fmt.Fprintf(os.Stderr, "warning: worker %d: creating git cat-file stdout pipe: %v\n", workerID, err) + return + } + if err := cmd.Start(); err != nil { + fmt.Fprintf(os.Stderr, "warning: worker %d: starting git cat-file: %v\n", workerID, err) + return + } + + catBuf := bufio.NewReader(catOut) + + loop: + for slot := range jobs { + if ctx.Err() != nil { + break loop // all initially-unfilled holes filled; stop requesting new blobs + } + + if _, err := fmt.Fprintf(catIn, "%s\n", slot.blobHash); err != nil { + fmt.Fprintf(os.Stderr, "warning: worker %d: writing to git cat-file (blob %s): %v\n", workerID, slot.blobHash, err) + continue + } + + // response header: " blob \n" or " missing\n" + line, err := catBuf.ReadString('\n') + if err != nil { + fmt.Fprintf(os.Stderr, "warning: worker %d: reading git cat-file header (blob %s): %v\n", workerID, slot.blobHash, err) + continue + } + parts := strings.Fields(line) + if len(parts) < 3 || parts[1] != "blob" { + fmt.Fprintf(os.Stderr, "warning: worker %d: unexpected git cat-file response %q for blob %s\n", workerID, strings.TrimRight(line, "\n"), slot.blobHash) + continue + } + size, err := strconv.ParseInt(parts[2], 10, 64) + if err != nil { + fmt.Fprintf(os.Stderr, "warning: worker %d: bad size %q for blob %s: %v\n", workerID, parts[2], slot.blobHash, err) + continue + } + + // LimitedReader prevents the decoder from reading into the next blob's header + lr := &io.LimitedReader{R: catBuf, N: size} + if err := searchBlob(ctx, lr, keyToHoles, slot.idx, matches); err != nil { + fmt.Fprintf(os.Stderr, "warning: worker %d: skipping blob %s: %v\n", workerID, slot.blobHash, err) + } + // drain any unread bytes (after a parse error or ctx-triggered early return) + if _, err := io.Copy(io.Discard, lr); err != nil { + fmt.Fprintf(os.Stderr, "warning: worker %d: draining blob %s: %v\n", workerID, slot.blobHash, err) + } + if _, err := catBuf.ReadByte(); err != nil { // trailing LF after blob content + fmt.Fprintf(os.Stderr, "warning: worker %d: reading trailing newline for blob %s: %v\n", workerID, slot.blobHash, err) + } + } + + go io.Copy(io.Discard, catOut) // let git flush and exit without blocking on a full pipe + catIn.Close() + cmd.Wait() + }(w) + } + + wg.Wait() + close(matches) // signal coordinator that no more matches are coming + coordDone.Wait() // drain any in-flight matches before reading results + + unfilledCount := 0 + for _, s := range bestSlot { + if s == math.MaxInt { + unfilledCount++ + } + } + if unfilledCount > 0 { + fmt.Fprintf(os.Stderr, "warning: %d hole(s) could not be filled from git history\n", unfilledCount) + } + } + + // --- Step 6: emit holes.json --- + + // holes.json: { tag -> { arch -> [ resolved OCI index, ... ] } } + // O(1) lookup by (tag, arch); the list normally has one entry; it has more when multiple OS-version variants (e.g. Windows) share the same (tag, arch). + // Unfillable holes (no resolved entry anywhere in history) are recorded as null rather than + // omitted: this lets the next run recognise them as "previously searched, still not found" + // and skip the git search for them, rather than scanning history again to reach the same result. + output := map[string]map[string][]*ocispec.Index{} + for i, h := range holes { + if output[h.tag] == nil { + output[h.tag] = map[string][]*ocispec.Index{} + } + if results[i] != nil { + output[h.tag][h.arch] = append(output[h.tag][h.arch], results[i]) + } else if _, exists := output[h.tag][h.arch]; !exists { + output[h.tag][h.arch] = nil // null sentinel: searched, not found + } + } + + enc := json.NewEncoder(os.Stdout) + enc.SetIndent("", "\t") + if err := enc.Encode(output); err != nil { + panic(err) + } +} + +// searchBlob streams a single builds.json blob (from any historical commit), looking for resolved entries whose (tag, arch) pairs match any hole. Matches are sent to the matches channel tagged with slotIdx (lower = more recent); the coordinator goroutine owns bestSlot/results so no synchronisation is needed here. ctx is checked between entries: once all initially-unfilled holes are globally filled the caller cancels ctx and this function returns early, letting the caller drain the remaining unread bytes. +// +// Uses encoding/json/v2 (jsontext + jsonv2.UnmarshalDecode) to stream the top-level object one entry at a time, avoiding loading the entire blob into memory at once. +func searchBlob(ctx context.Context, r io.Reader, keyToHoles map[string][]int, slotIdx int, matches chan<- blobMatch) error { + dec := jsontext.NewDecoder(r) + + // consume the opening '{' of the top-level builds.json object + if tok, err := dec.ReadToken(); err != nil { + return err + } else if tok.Kind() != '{' { + return fmt.Errorf("expected '{', got %v", tok) + } + + // minimal shape we care about in each builds.json entry + // json/v2 discards unknown fields by default + type entry struct { + Build struct { + Arch string `json:"arch"` + Resolved *ocispec.Index `json:"resolved"` + } `json:"build"` + Source struct { + Arches map[string]struct { + Tags []string `json:"tags"` + } `json:"arches"` + } `json:"source"` + } + + for dec.PeekKind() != '}' { + if ctx.Err() != nil { + return nil // all initially-unfilled holes globally filled; return early, caller drains remainder + } + + // read the buildId key (discard it) + if _, err := dec.ReadToken(); err != nil { + return err + } + + // decode the full value for this buildId into our minimal struct + var e entry + if err := jsonv2.UnmarshalDecode(dec, &e); err != nil { + return err + } + + if e.Build.Resolved == nil { + continue + } + + arch := e.Build.Arch + archData, ok := e.Source.Arches[arch] + if !ok { + continue + } + for _, tag := range archData.Tags { + for _, holeIdx := range keyToHoles[tagArchKey(tag, arch)] { + matches <- blobMatch{slotIdx: slotIdx, holeIdx: holeIdx, resolved: e.Build.Resolved} + } + } + } + + return nil +} diff --git a/go.mod b/go.mod index ba7c5c4..b410df1 100644 --- a/go.mod +++ b/go.mod @@ -1,11 +1,11 @@ module github.com/docker-library/meta-scripts -// ideally this would be the single source of truth for this entire repository, but riscv64 means this bleeds into .go-env.sh too -- if changing this, see that file too -go 1.21 +go 1.26 require ( cuelabs.dev/go/oci/ociregistry v0.0.0-20240214163758-5ebe80b0a9a6 github.com/docker-library/bashbrew v0.1.11 + github.com/go-json-experiment/json v0.0.0-20260623181947-01eb4420fa68 github.com/opencontainers/go-digest v1.0.0 github.com/opencontainers/image-spec v1.1.0 golang.org/x/time v0.5.0 @@ -15,6 +15,7 @@ require ( github.com/containerd/containerd v1.6.19 // indirect github.com/golang/protobuf v1.5.2 // indirect github.com/sirupsen/logrus v1.9.0 // indirect + golang.org/x/net v0.16.0 // indirect golang.org/x/sys v0.13.0 // indirect google.golang.org/genproto v0.0.0-20221207170731-23e4bf6bdc37 // indirect google.golang.org/grpc v1.51.0 // indirect diff --git a/go.sum b/go.sum index e4ae7a6..279c68c 100644 --- a/go.sum +++ b/go.sum @@ -5,6 +5,8 @@ github.com/davecgh/go-spew v1.1.1 h1:vj9j/u1bqnvCEfJOwUhtlOARqs3+rkHYY13jYWTU97c github.com/davecgh/go-spew v1.1.1/go.mod h1:J7Y8YcW2NihsgmVo/mv3lAwl/skON4iLHjSsI+c5H38= github.com/docker-library/bashbrew v0.1.11 h1:9S6jYFu0+RaqEAfvS2lh7jcaDkcvFi2maB2aU3yb0TM= github.com/docker-library/bashbrew v0.1.11/go.mod h1:6fyRRSm4vgBAgTw87EsfOT7wXKsc4JA9I5cdQJmwOm8= +github.com/go-json-experiment/json v0.0.0-20260623181947-01eb4420fa68 h1:KZaTBSyshWX3MP5jukJcNSuXDQTO+rNpt0J564dX/eg= +github.com/go-json-experiment/json v0.0.0-20260623181947-01eb4420fa68/go.mod h1:tphK2c80bpPhMOI4v6bIc2xWywPfbqi1Z06+RcrMkDg= github.com/go-quicktest/qt v1.100.0 h1:I7iSLgIwNp0E0UnSvKJzs7ig0jg/Iq83zsZjtQNW7jY= github.com/go-quicktest/qt v1.100.0/go.mod h1:leyLsQ4jksGmF1KaQEyabnqGIiJTbOU5S46QegToEj4= github.com/golang/protobuf v1.5.0/go.mod h1:FsONVRAS9T7sI+LIUmWTfcYkHO4aIWwzhcaSAoJOfIk= @@ -34,13 +36,13 @@ github.com/stretchr/testify v1.7.0 h1:nwc3DEeHmmLAfoZucVR881uASk0Mfjw8xYJ99tb5Cc github.com/stretchr/testify v1.7.0/go.mod h1:6Fq8oRcR53rry900zMqJjRRixrwX3KX962/h/Wwjteg= github.com/tianon/cuelabs-oci/ociregistry v0.0.0-20240329232705-b652d611e4b3 h1:kfAfFbiZ+2ZErqgqKtaMge1qeeE/0rnxuTl21G7fSwk= github.com/tianon/cuelabs-oci/ociregistry v0.0.0-20240329232705-b652d611e4b3/go.mod h1:pK23AUVXuNzzTpfMCA06sxZGeVQ/75FdVtW249de9Uo= -golang.org/x/net v0.7.0 h1:rJrUqqhjsgNp7KqAIc25s9pZnjU7TUcSY7HcVZjdn1g= -golang.org/x/net v0.7.0/go.mod h1:2Tu9+aMcznHK/AK1HMvgo6xiTLG5rD5rZLDS+rp2Bjs= +golang.org/x/net v0.16.0 h1:7eBu7KsSvFDtSXUIDbh3aqlK4DPsZ1rByC8PFfBThos= +golang.org/x/net v0.16.0/go.mod h1:NxSsAGuq816PNPmqtQdLE42eU2Fs7NoRIZrHJAlaCOE= golang.org/x/sys v0.0.0-20220715151400-c0bba94af5f8/go.mod h1:oPkhp1MJrh7nUepCBck5+mAzfO9JrbApNNgaTdGDITg= golang.org/x/sys v0.13.0 h1:Af8nKPmuFypiUBjVoU9V20FiaFXOcuZI21p0ycVYYGE= golang.org/x/sys v0.13.0/go.mod h1:oPkhp1MJrh7nUepCBck5+mAzfO9JrbApNNgaTdGDITg= -golang.org/x/text v0.7.0 h1:4BRB4x83lYWy72KwLD/qYDuTu7q9PjSagHvijDw7cLo= -golang.org/x/text v0.7.0/go.mod h1:mrYo+phRRbMaCq/xk9113O4dZlRixOauAjOtrjsXDZ8= +golang.org/x/text v0.13.0 h1:ablQoSUd0tRdKxZewP80B+BaqeKJuVhuRxj/dkrun3k= +golang.org/x/text v0.13.0/go.mod h1:TvPlkZtksWOMsz7fbANvkp4WM8x/WCo/om8BMLbz+aE= golang.org/x/time v0.5.0 h1:o7cqy6amK/52YcAKIPlM3a+Fpj35zvRj2TP+e1xFSfk= golang.org/x/time v0.5.0/go.mod h1:3BpzKBy/shNhVucY/MWOyx10tF3SFh9QdLuxbVysPQM= golang.org/x/tools v0.14.0 h1:jvNa2pY0M4r62jkRQ6RwEZZyPcymeL9XZMLBbV7U2nc=