From b13a491cb080c3b4058d3bdb7f811f0bba4daabb Mon Sep 17 00:00:00 2001 From: Tianon Gravi Date: Fri, 31 Jul 2026 14:01:05 -0700 Subject: [PATCH 1/4] WIP: holes-filling --- .go-env.sh | 19 ---- cmd/holes/README.md | 29 +++++ cmd/holes/main.go | 271 ++++++++++++++++++++++++++++++++++++++++++++ go.mod | 24 +++- go.sum | 124 +++++++++++++++++++- 5 files changed, 442 insertions(+), 25 deletions(-) create mode 100644 cmd/holes/README.md create mode 100644 cmd/holes/main.go diff --git a/.go-env.sh b/.go-env.sh index 7d46576..62c5860 100755 --- a/.go-env.sh +++ b/.go-env.sh @@ -79,25 +79,6 @@ if [ -z "${GOLANG_IMAGE:-}" ]; then go="${go%.*}" # strip to just X.Y fi GOLANG_IMAGE="golang:$go" - - # handle riscv64 "gracefully" (no golang image yet because no stable distro releases yet) - { - if ! docker image inspect --format '.' "$GOLANG_IMAGE" &> /dev/null && ! docker pull "$GOLANG_IMAGE"; then - if [ -n "${BASHBREW_ARCH:-}" ] && docker buildx inspect "bashbrew-$BASHBREW_ARCH" &> /dev/null; then - # a very rough hack to avoid: - # ERROR: failed to solve: failed to solve with frontend dockerfile.v0: failed to read dockerfile: failed to load cache key: subdir not supported yet - # (we need buildkit/buildx for --build-context, but newer buildkit than our dockerd might have for build-from-git-with-subdir) - export BUILDX_BUILDER="bashbrew-$BASHBREW_ARCH" - fi - ( - set -x - # TODO make this more dynamic, less hard-coded 🙈 - # https://github.com/docker-library/golang/blob/ea6bbce8c9b13acefed0f5507336be01f0918f97/1.21/bookworm/Dockerfile - GOLANG_IMAGE='golang:1.21' # to be explicit - docker buildx build --load --tag "$GOLANG_IMAGE" --build-context 'buildpack-deps:bookworm-scm=docker-image://buildpack-deps:unstable-scm' 'https://github.com/docker-library/golang.git#ea6bbce8c9b13acefed0f5507336be01f0918f97:1.21/bookworm' - ) - fi - } >&2 fi args+=( diff --git a/cmd/holes/README.md b/cmd/holes/README.md new file mode 100644 index 0000000..1a6cd88 --- /dev/null +++ b/cmd/holes/README.md @@ -0,0 +1,29 @@ +# cmd/holes + +`builds.json` is regenerated from scratch on every meta run. A **hole** is any `(tag, arch)` pair that appears in `sources.json` but has no currently-resolved build in `builds.json`. This happens in two distinct ways: + +- **`resolved: null`** -- the entry exists (parents are all resolved, buildId is computed) but the staging image hasn't been built yet. +- **Entirely absent** -- `cmd/builds` emitted nothing for this `(sourceId, arch)` because a DOI parent's staging image is itself unresolved (`close(outChan)`). The entry cannot appear in `builds.json` at all until the parent resolves. + +`builds.json` cannot be used alone to detect entirely-absent holes; `sources.json` is required as the authoritative list of what *should* exist. + +## Why holes matter + +`library/IMAGE:TAG` is a single atomic OCI index listing all arches. Pushing any update to it requires supplying descriptors for *every* expected arch -- you cannot leave one arch out without dropping it from the index entirely. Meanwhile arches build at very different rates (hours for amd64, days for riscv64), so "wait for all arches before publishing" is not viable. Holes must be filled with the most recent previously-resolved content for each `(tag, arch)` while the new build is in flight. + +## Key constraints on the search key + +Holes are independent per `(tag, arch)` -- not per `(sourceId, arch)`. The reasons: + +- **`sourceId` is not stable across re-adds.** A tag removed from the library and re-added later may have a new sourceId (changed Dockerfile), breaking any lookup keyed on sourceId. The tag name is what users pull; it is the right identity. +- **Each tag is a distinct semantic identity.** `"golang:1.22"` and `"golang:1.22.6"` share a builds.json entry when both point at the same build, but they must be looked up independently: `"golang:1.22.6"` should never serve a fallback from `"golang:1.22.5"`, even though `"golang:1.22"` should. + +Tags come from `source.arches[arch].tags`, not `archTags` -- `archTags` may be empty in non-DOI repos (e.g. `debuerreotype/debuerreotype:latest` rather than `amd64/debian:bookworm`). + +## Known imprecision: multi-variant arches + +For architectures with multiple OS-version variants under the same tag (e.g. `windows-amd64` with ltsc2019 and ltsc2022 both contributing to `"docker:24"`), `(tag, arch)` conflates all variants. There is no clean way to track which specific variant is the hole without keying by sourceId -- which breaks re-added tags. The fallback found in history may correspond to the wrong OS version. This is accepted imprecision. + +## Git history search + +The git history of the meta repo contains every prior `builds.json` state. Walking it backward is the only source of fallback data that does not require live registry queries (which are expensive enough that put-shared runs only every ~3 hours). The walk must be a single linear backward pass over *all* holes simultaneously -- exponential or binary search is not safe because a tag can be added, fully built, *and* removed within a gap of skipped commits, leaving no evidence on either side of the jump. diff --git a/cmd/holes/main.go b/cmd/holes/main.go new file mode 100644 index 0000000..24a6f74 --- /dev/null +++ b/cmd/holes/main.go @@ -0,0 +1,271 @@ +package main + +import ( + "encoding/json" + "fmt" + "io" + "os" + "path/filepath" + + // encoding/json/v2, but not yet + jsonv2 "github.com/go-json-experiment/json" + "github.com/go-json-experiment/json/jsontext" + + gogit "github.com/go-git/go-git/v5" + "github.com/go-git/go-git/v5/plumbing/object" + "github.com/go-git/go-git/v5/plumbing/storer" + ocispec "github.com/opencontainers/image-spec/specs-go/v1" +) + +// hole represents a single (tag, arch) pair with no resolved entry in the current builds.json. Each (tag, arch) is an independent hole: "golang:1.22" on amd64 and "golang:1.22.6" on amd64 have completely different historical content and must be looked up separately. +// +// "tag" is a value from source.arches[arch].tags (e.g. "golang:1.22" or "debuerreotype/debuerreotype:latest") -- NOT the arch-prefixed archTag, which may be empty in non-DOI repos. +// +// Note: for architectures with multiple OS-version variants (e.g. windows-amd64 with ltsc2019 and ltsc2022), the (tag, arch) key conflates all variants. We accept this imprecision; there is no clean way to track partial holes within a single (tag, arch) pair. +type hole struct { + tag string + arch string +} + +// tagArchKey returns the map key used in keyToHole. A null byte is safe since neither tags nor arch names ever contain one. +func tagArchKey(tag, arch string) string { + return tag + "\x00" + arch +} + +func main() { + if len(os.Args) < 3 { + fmt.Fprintf(os.Stderr, "usage: %s sources.json builds.json\n", os.Args[0]) + os.Exit(1) + } + sourcesFile := os.Args[1] + buildsFile := os.Args[2] + + // --- Step 1: parse sources.json for the full expected (sourceId, arch) set --- + + sourcesF, err := os.Open(sourcesFile) + if err != nil { + panic(err) + } + // sources.json is an object whose values each have a sourceId and an arches map; we need sourceId + arches[*].tags (archTags may be empty) + var sources map[string]struct { + SourceID string `json:"sourceId"` + Arches map[string]struct { + Tags []string `json:"tags"` + } `json:"arches"` + } + if err := json.NewDecoder(sourcesF).Decode(&sources); err != nil { + panic(err) + } + sourcesF.Close() + + // --- Step 2: parse builds.json to find which (sourceId, arch) are resolved --- + + buildsF, err := os.Open(buildsFile) + if err != nil { + panic(err) + } + // we only need sourceId, arch, and whether resolved is non-null + var builds map[string]struct { + Build struct { + SourceID string `json:"sourceId"` + Arch string `json:"arch"` + Resolved *ocispec.Index `json:"resolved"` + } `json:"build"` + } + if err := json.NewDecoder(buildsF).Decode(&builds); err != nil { + panic(err) + } + buildsF.Close() + + resolvedSet := make(map[string]bool, len(builds)) // key: sourceId+"-"+arch + for _, entry := range builds { + if entry.Build.Resolved != nil { + resolvedSet[entry.Build.SourceID+"-"+entry.Build.Arch] = true + } + } + + // --- Step 3: compute holes --- + // a hole is any (tag, arch) from sources.json where the backing (sourceId, arch) has no resolved entry in builds.json -- whether entirely absent (unresolved parent chain) or present with resolved == null + // each (tag, arch) is independent: "golang:1.22" may have a historical fallback while "golang:1.22.6" does not + + var holes []hole + // keyToHoles maps tagArchKey(tag, arch) to the list of indices in holes that share that (tag, arch). Normally exactly one entry; multiple entries arise when several sources share the same (tag, arch) -- e.g. windows-amd64 with ltsc2019 AND ltsc2022 both contributing to "docker:24". + // + // Known, accepted imprecision: because we can only search git history by (tag, arch), we cannot tell which specific OS-version variant is the hole, so we may fill slots with fallbacks for the wrong variant. + keyToHoles := map[string][]int{} + + for _, src := range sources { + for arch, archData := range src.Arches { + if resolvedSet[src.SourceID+"-"+arch] { + continue + } + for _, tag := range archData.Tags { + k := tagArchKey(tag, arch) + idx := len(holes) + holes = append(holes, hole{tag: tag, arch: arch}) + keyToHoles[k] = append(keyToHoles[k], idx) + } + } + } + + if len(holes) == 0 { + fmt.Println("{}") + return + } + fmt.Fprintf(os.Stderr, "searching git history for %d hole(s)\n", len(holes)) + + // --- Step 4: open the git repo that contains builds.json --- + + absBuilds, err := filepath.Abs(buildsFile) + if err != nil { + panic(err) + } + repo, err := gogit.PlainOpenWithOptions(filepath.Dir(absBuilds), &gogit.PlainOpenOptions{ + DetectDotGit: true, + }) + if err != nil { + panic(err) + } + wt, err := repo.Worktree() + if err != nil { + panic(err) + } + // builds.json path relative to the worktree root, for commit.File() calls + relBuilds, err := filepath.Rel(wt.Filesystem.Root(), absBuilds) + if err != nil { + panic(err) + } + + // --- Step 5: walk git history, filling holes as we go --- + + results := make([]*ocispec.Index, len(holes)) // nil == still unfilled + unfilledCount := len(holes) + + head, err := repo.Head() + if err != nil { + panic(err) + } + commitIter, err := repo.Log(&gogit.LogOptions{From: head.Hash()}) + if err != nil { + panic(err) + } + + err = commitIter.ForEach(func(c *object.Commit) error { + if unfilledCount == 0 { + return storer.ErrStop + } + + file, err := c.File(relBuilds) + if err != nil { + // builds.json absent in this commit (e.g. very early repo history) + return nil + } + reader, err := file.Reader() + if err != nil { + return nil + } + defer reader.Close() + + newFilled, err := searchBlob(reader, keyToHoles, results) + if err != nil { + fmt.Fprintf(os.Stderr, "warning: skipping blob in commit %s: %v\n", c.Hash, err) + return nil + } + unfilledCount -= newFilled + if newFilled > 0 { + fmt.Fprintf(os.Stderr, "filled %d hole(s) from commit %s (%d remaining)\n", newFilled, c.Hash, unfilledCount) + } + return nil + }) + if err != nil { + panic(err) + } + + if unfilledCount > 0 { + fmt.Fprintf(os.Stderr, "warning: %d hole(s) could not be filled from git history\n", unfilledCount) + } + + // --- Step 6: emit holes.json --- + + // holes.json: { tag -> { arch -> [ resolved OCI index, ... ] } } + // O(1) lookup by (tag, arch); the list normally has one entry; it has more when multiple OS-version variants (e.g. Windows) share the same (tag, arch) + output := map[string]map[string][]*ocispec.Index{} + for i, h := range holes { + if results[i] == nil { + continue // no historical data found; library deploy will skip this hole + } + if output[h.tag] == nil { + output[h.tag] = map[string][]*ocispec.Index{} + } + output[h.tag][h.arch] = append(output[h.tag][h.arch], results[i]) + } + + enc := json.NewEncoder(os.Stdout) + enc.SetIndent("", "\t") + if err := enc.Encode(output); err != nil { + panic(err) + } +} + +// searchBlob streams a single builds.json blob (from any historical commit), looking for resolved entries whose (tag, arch) pairs match unfilled holes. It returns the number of newly-filled holes. +// +// Uses encoding/json/v2 (jsontext + jsonv2.UnmarshalDecode) to stream the top-level object one entry at a time, avoiding loading the entire blob into memory at once. +func searchBlob(r io.Reader, keyToHoles map[string][]int, results []*ocispec.Index) (int, error) { + dec := jsontext.NewDecoder(r) + + // consume the opening '{' of the top-level builds.json object + if tok, err := dec.ReadToken(); err != nil { + return 0, err + } else if tok.Kind() != '{' { + return 0, fmt.Errorf("expected '{', got %v", tok) + } + + // minimal shape we care about in each builds.json entry + // json/v2 discards unknown fields by default + type entry struct { + Build struct { + Arch string `json:"arch"` + Resolved *ocispec.Index `json:"resolved"` + } `json:"build"` + Source struct { + Arches map[string]struct { + Tags []string `json:"tags"` + } `json:"arches"` + } `json:"source"` + } + + newlyFilled := 0 + for dec.PeekKind() != '}' { + // read the buildId key (discard it) + if _, err := dec.ReadToken(); err != nil { + return newlyFilled, err + } + + // decode the full value for this buildId into our minimal struct + var e entry + if err := jsonv2.UnmarshalDecode(dec, &e); err != nil { + return newlyFilled, err + } + + if e.Build.Resolved == nil { + continue + } + + arch := e.Build.Arch + archData, ok := e.Source.Arches[arch] + if !ok { + continue + } + for _, tag := range archData.Tags { + for _, holeIdx := range keyToHoles[tagArchKey(tag, arch)] { + if results[holeIdx] != nil { + continue // already filled by a more-recent commit + } + results[holeIdx] = e.Build.Resolved + newlyFilled++ + } + } + } + + return newlyFilled, nil +} diff --git a/go.mod b/go.mod index ba7c5c4..ad324e5 100644 --- a/go.mod +++ b/go.mod @@ -1,24 +1,44 @@ module github.com/docker-library/meta-scripts -// ideally this would be the single source of truth for this entire repository, but riscv64 means this bleeds into .go-env.sh too -- if changing this, see that file too -go 1.21 +go 1.26 require ( cuelabs.dev/go/oci/ociregistry v0.0.0-20240214163758-5ebe80b0a9a6 github.com/docker-library/bashbrew v0.1.11 + github.com/go-git/go-git/v5 v5.5.1 + github.com/go-json-experiment/json v0.0.0-20260623181947-01eb4420fa68 github.com/opencontainers/go-digest v1.0.0 github.com/opencontainers/image-spec v1.1.0 golang.org/x/time v0.5.0 ) require ( + github.com/Microsoft/go-winio v0.6.0 // indirect + github.com/ProtonMail/go-crypto v0.0.0-20221026131551-cf6655e29de4 // indirect + github.com/acomagu/bufpipe v1.0.3 // indirect + github.com/cloudflare/circl v1.3.1 // indirect github.com/containerd/containerd v1.6.19 // indirect + github.com/emirpasic/gods v1.18.1 // indirect + github.com/go-git/gcfg v1.5.0 // indirect + github.com/go-git/go-billy/v5 v5.3.1 // indirect github.com/golang/protobuf v1.5.2 // indirect + github.com/imdario/mergo v0.3.13 // indirect + github.com/jbenet/go-context v0.0.0-20150711004518-d14ea06fba99 // indirect + github.com/kevinburke/ssh_config v1.2.0 // indirect + github.com/pjbgf/sha1cd v0.2.3 // indirect + github.com/sergi/go-diff v1.2.0 // indirect github.com/sirupsen/logrus v1.9.0 // indirect + github.com/skeema/knownhosts v1.1.0 // indirect + github.com/xanzy/ssh-agent v0.3.3 // indirect + golang.org/x/crypto v0.14.0 // indirect + golang.org/x/mod v0.13.0 // indirect + golang.org/x/net v0.16.0 // indirect golang.org/x/sys v0.13.0 // indirect + golang.org/x/tools v0.14.0 // indirect google.golang.org/genproto v0.0.0-20221207170731-23e4bf6bdc37 // indirect google.golang.org/grpc v1.51.0 // indirect google.golang.org/protobuf v1.28.1 // indirect + gopkg.in/warnings.v0 v0.1.2 // indirect ) // https://github.com/cue-labs/oci/pull/29 diff --git a/go.sum b/go.sum index e4ae7a6..bdc54cf 100644 --- a/go.sum +++ b/go.sum @@ -1,10 +1,40 @@ +github.com/Microsoft/go-winio v0.5.2/go.mod h1:WpS1mjBmmwHBEWmogvA2mj8546UReBk4v8QkMxJ6pZY= +github.com/Microsoft/go-winio v0.6.0 h1:slsWYD/zyx7lCXoZVlvQrj0hPTM1HI4+v1sIda2yDvg= +github.com/Microsoft/go-winio v0.6.0/go.mod h1:cTAf44im0RAYeL23bpB+fzCyDH2MJiz2BO69KH/soAE= +github.com/ProtonMail/go-crypto v0.0.0-20221026131551-cf6655e29de4 h1:ra2OtmuW0AE5csawV4YXMNGNQQXvLRps3z2Z59OPO+I= +github.com/ProtonMail/go-crypto v0.0.0-20221026131551-cf6655e29de4/go.mod h1:UBYPn8k0D56RtnR8RFQMjmh4KrZzWJ5o7Z9SYjossQ8= +github.com/acomagu/bufpipe v1.0.3 h1:fxAGrHZTgQ9w5QqVItgzwj235/uYZYgbXitB+dLupOk= +github.com/acomagu/bufpipe v1.0.3/go.mod h1:mxdxdup/WdsKVreO5GpW4+M/1CE2sMG4jeGJ2sYmHc4= +github.com/anmitsu/go-shlex v0.0.0-20200514113438-38f4b401e2be h1:9AeTilPcZAjCFIImctFaOjnTIavg87rW78vTPkQqLI8= +github.com/anmitsu/go-shlex v0.0.0-20200514113438-38f4b401e2be/go.mod h1:ySMOLuWl6zY27l47sB3qLNK6tF2fkHG55UZxx8oIVo4= +github.com/armon/go-socks5 v0.0.0-20160902184237-e75332964ef5 h1:0CwZNZbxp69SHPdPJAN/hZIm0C4OItdklCFmMRWYpio= +github.com/armon/go-socks5 v0.0.0-20160902184237-e75332964ef5/go.mod h1:wHh0iHkYZB8zMSxRWpUBQtwG5a7fFgvEO+odwuTv2gs= +github.com/bwesterb/go-ristretto v1.2.0/go.mod h1:fUIoIZaG73pV5biE2Blr2xEzDoMj7NFEuV9ekS419A0= +github.com/cloudflare/circl v1.1.0/go.mod h1:prBCrKB9DV4poKZY1l9zBXg2QJY7mvgRvtMxxK7fi4I= +github.com/cloudflare/circl v1.3.1 h1:4OVCZRL62ijwEwxnF6I7hLwxvIYi3VaZt8TflkqtrtA= +github.com/cloudflare/circl v1.3.1/go.mod h1:+CauBF6R70Jqcyl8N2hC8pAXYbWkGIezuSbuGLtRhnw= github.com/containerd/containerd v1.6.19 h1:F0qgQPrG0P2JPgwpxWxYavrVeXAG0ezUIB9Z/4FTUAU= github.com/containerd/containerd v1.6.19/go.mod h1:HZCDMn4v/Xl2579/MvtOC2M206i+JJ6VxFWU/NetrGY= +github.com/creack/pty v1.1.9/go.mod h1:oKZEueFk5CKHvIhNR5MUki03XCEU+Q6VDXinZuGJ33E= github.com/davecgh/go-spew v1.1.0/go.mod h1:J7Y8YcW2NihsgmVo/mv3lAwl/skON4iLHjSsI+c5H38= github.com/davecgh/go-spew v1.1.1 h1:vj9j/u1bqnvCEfJOwUhtlOARqs3+rkHYY13jYWTU97c= github.com/davecgh/go-spew v1.1.1/go.mod h1:J7Y8YcW2NihsgmVo/mv3lAwl/skON4iLHjSsI+c5H38= github.com/docker-library/bashbrew v0.1.11 h1:9S6jYFu0+RaqEAfvS2lh7jcaDkcvFi2maB2aU3yb0TM= github.com/docker-library/bashbrew v0.1.11/go.mod h1:6fyRRSm4vgBAgTw87EsfOT7wXKsc4JA9I5cdQJmwOm8= +github.com/emirpasic/gods v1.18.1 h1:FXtiHYKDGKCW2KzwZKx0iC0PQmdlorYgdFG9jPXJ1Bc= +github.com/emirpasic/gods v1.18.1/go.mod h1:8tpGGwCnJ5H4r6BWwaV6OrWmMoPhUl5jm/FMNAnJvWQ= +github.com/gliderlabs/ssh v0.3.5 h1:OcaySEmAQJgyYcArR+gGGTHCyE7nvhEMTlYY+Dp8CpY= +github.com/gliderlabs/ssh v0.3.5/go.mod h1:8XB4KraRrX39qHhT6yxPsHedjA08I/uBVwj4xC+/+z4= +github.com/go-git/gcfg v1.5.0 h1:Q5ViNfGF8zFgyJWPqYwA7qGFoMTEiBmdlkcfRmpIMa4= +github.com/go-git/gcfg v1.5.0/go.mod h1:5m20vg6GwYabIxaOonVkTdrILxQMpEShl1xiMF4ua+E= +github.com/go-git/go-billy/v5 v5.3.1 h1:CPiOUAzKtMRvolEKw+bG1PLRpT7D3LIs3/3ey4Aiu34= +github.com/go-git/go-billy/v5 v5.3.1/go.mod h1:pmpqyWchKfYfrkb/UVH4otLvyi/5gJlGI4Hb3ZqZ3W0= +github.com/go-git/go-git-fixtures/v4 v4.3.1 h1:y5z6dd3qi8Hl+stezc8p3JxDkoTRqMAlKnXHuzrfjTQ= +github.com/go-git/go-git-fixtures/v4 v4.3.1/go.mod h1:8LHG1a3SRW71ettAD/jW13h8c6AqjVSeL11RAdgaqpo= +github.com/go-git/go-git/v5 v5.5.1 h1:5vtv2TB5PM/gPM+EvsHJ16hJh4uAkdGcKilcwY7FYwo= +github.com/go-git/go-git/v5 v5.5.1/go.mod h1:uz5PQ3d0gz7mSgzZhSJToM6ALPaKCdSnl58/Xb5hzr8= +github.com/go-json-experiment/json v0.0.0-20260623181947-01eb4420fa68 h1:KZaTBSyshWX3MP5jukJcNSuXDQTO+rNpt0J564dX/eg= +github.com/go-json-experiment/json v0.0.0-20260623181947-01eb4420fa68/go.mod h1:tphK2c80bpPhMOI4v6bIc2xWywPfbqi1Z06+RcrMkDg= github.com/go-quicktest/qt v1.100.0 h1:I7iSLgIwNp0E0UnSvKJzs7ig0jg/Iq83zsZjtQNW7jY= github.com/go-quicktest/qt v1.100.0/go.mod h1:leyLsQ4jksGmF1KaQEyabnqGIiJTbOU5S46QegToEj4= github.com/golang/protobuf v1.5.0/go.mod h1:FsONVRAS9T7sI+LIUmWTfcYkHO4aIWwzhcaSAoJOfIk= @@ -13,38 +43,115 @@ github.com/golang/protobuf v1.5.2/go.mod h1:XVQd3VNwM+JqD3oG2Ue2ip4fOMUkwXdXDdiu github.com/google/go-cmp v0.5.5/go.mod h1:v8dTdLbMG2kIc/vJvl+f65V22dbkXbowE6jgT/gNBxE= github.com/google/go-cmp v0.5.9 h1:O2Tfq5qg4qc4AmwVlvv0oLiVAGB7enBSJ2x2DqQFi38= github.com/google/go-cmp v0.5.9/go.mod h1:17dUlkBOakJ0+DkrSSNjCkIjxS6bF9zb3elmeNGIjoY= +github.com/imdario/mergo v0.3.13 h1:lFzP57bqS/wsqKssCGmtLAb8A0wKjLGrve2q3PPVcBk= +github.com/imdario/mergo v0.3.13/go.mod h1:4lJ1jqUDcsbIECGy0RUJAXNIhg+6ocWgb1ALK2O4oXg= +github.com/jbenet/go-context v0.0.0-20150711004518-d14ea06fba99 h1:BQSFePA1RWJOlocH6Fxy8MmwDt+yVQYULKfN0RoTN8A= +github.com/jbenet/go-context v0.0.0-20150711004518-d14ea06fba99/go.mod h1:1lJo3i6rXxKeerYnT8Nvf0QmHCRC1n8sfWVwXF2Frvo= +github.com/jessevdk/go-flags v1.5.0/go.mod h1:Fw0T6WPc1dYxT4mKEZRfG5kJhaTDP9pj1c2EWnYs/m4= +github.com/kevinburke/ssh_config v1.2.0 h1:x584FjTGwHzMwvHx18PXxbBVzfnxogHaAReU4gf13a4= +github.com/kevinburke/ssh_config v1.2.0/go.mod h1:CT57kijsi8u/K/BOFA39wgDQJ9CxiF4nAY/ojJ6r6mM= +github.com/kr/pretty v0.1.0/go.mod h1:dAy3ld7l9f0ibDNOQOHHMYYIIbhfbHSm3C4ZsoJORNo= +github.com/kr/pretty v0.2.1/go.mod h1:ipq/a2n7PKx3OHsz4KJII5eveXtPO4qwEXGdVfWzfnI= github.com/kr/pretty v0.3.1 h1:flRD4NNwYAUpkphVc1HcthR4KEIFJ65n8Mw5qdRn3LE= github.com/kr/pretty v0.3.1/go.mod h1:hoEshYVHaxMs3cyo3Yncou5ZscifuDolrwPKZanG3xk= +github.com/kr/pty v1.1.1/go.mod h1:pFQYn66WHrOpPYNljwOMqo10TkYh1fy3cYio2l3bCsQ= +github.com/kr/text v0.1.0/go.mod h1:4Jbv+DJW3UT/LiOwJeYQe1efqtUx/iVham/4vfdArNI= github.com/kr/text v0.2.0 h1:5Nx0Ya0ZqY2ygV366QzturHI13Jq95ApcVaJBhpS+AY= github.com/kr/text v0.2.0/go.mod h1:eLer722TekiGuMkidMxC/pM04lWEeraHUUmBw8l2grE= +github.com/matryer/is v1.2.0 h1:92UTHpy8CDwaJ08GqLDzhhuixiBUUD1p3AU6PHddz4A= +github.com/matryer/is v1.2.0/go.mod h1:2fLPjFQM9rhQ15aVEtbuwhJinnOqrmgXPNdZsdwlWXA= +github.com/niemeyer/pretty v0.0.0-20200227124842-a10e7caefd8e/go.mod h1:zD1mROLANZcx1PVRCS0qkT7pwLkGfwJo4zjcN/Tysno= github.com/opencontainers/go-digest v1.0.0 h1:apOUWs51W5PlhuyGyz9FCeeBIOUDA/6nW8Oi/yOhh5U= github.com/opencontainers/go-digest v1.0.0/go.mod h1:0JzlMkj0TRzQZfJkVvzbP0HBR3IKzErnv2BNG4W4MAM= github.com/opencontainers/image-spec v1.1.0 h1:8SG7/vwALn54lVB/0yZ/MMwhFrPYtpEHQb2IpWsCzug= github.com/opencontainers/image-spec v1.1.0/go.mod h1:W4s4sFTMaBeK1BQLXbG4AdM2szdn85PY75RI83NrTrM= +github.com/pjbgf/sha1cd v0.2.3 h1:uKQP/7QOzNtKYH7UTohZLcjF5/55EnTw0jO/Ru4jZwI= +github.com/pjbgf/sha1cd v0.2.3/go.mod h1:HOK9QrgzdHpbc2Kzip0Q1yi3M2MFGPADtR6HjG65m5M= github.com/pkg/errors v0.9.1 h1:FEBLx1zS214owpjy7qsBeixbURkuhQAwrK5UwLGTwt4= github.com/pkg/errors v0.9.1/go.mod h1:bwawxfHBFNV+L2hUp1rHADufV3IMtnDRdf1r5NINEl0= github.com/pmezard/go-difflib v1.0.0 h1:4DBwDE0NGyQoBHbLQYPwSUPoCMWR5BEzIk/f1lZbAQM= github.com/pmezard/go-difflib v1.0.0/go.mod h1:iKH77koFhYxTK1pcRnkKkqfTogsbg7gZNVY4sRDYZ/4= github.com/rogpeppe/go-internal v1.10.1-0.20230524175051-ec119421bb97 h1:3RPlVWzZ/PDqmVuf/FKHARG5EMid/tl7cv54Sw/QRVY= github.com/rogpeppe/go-internal v1.10.1-0.20230524175051-ec119421bb97/go.mod h1:ddIwULY96R17DhadqLgMfk9H9tvdUzkipdSkR5nkCZA= +github.com/sergi/go-diff v1.1.0/go.mod h1:STckp+ISIX8hZLjrqAeVduY0gWCT9IjLuqbuNXdaHfM= +github.com/sergi/go-diff v1.2.0 h1:XU+rvMAioB0UC3q1MFrIQy4Vo5/4VsRDQQXHsEya6xQ= +github.com/sergi/go-diff v1.2.0/go.mod h1:STckp+ISIX8hZLjrqAeVduY0gWCT9IjLuqbuNXdaHfM= +github.com/sirupsen/logrus v1.7.0/go.mod h1:yWOB1SBYBC5VeMP7gHvWumXLIWorT60ONWic61uBYv0= github.com/sirupsen/logrus v1.9.0 h1:trlNQbNUG3OdDrDil03MCb1H2o9nJ1x4/5LYw7byDE0= github.com/sirupsen/logrus v1.9.0/go.mod h1:naHLuLoDiP4jHNo9R0sCBMtWGeIprob74mVsIT4qYEQ= +github.com/skeema/knownhosts v1.1.0 h1:Wvr9V0MxhjRbl3f9nMnKnFfiWTJmtECJ9Njkea3ysW0= +github.com/skeema/knownhosts v1.1.0/go.mod h1:sKFq3RD6/TKZkSWn8boUbDC7Qkgcv+8XXijpFO6roag= github.com/stretchr/objx v0.1.0/go.mod h1:HFkY916IF+rwdDfMAkV7OtwuqBVzrE8GR6GFx+wExME= +github.com/stretchr/testify v1.2.2/go.mod h1:a8OnRcib4nhh0OaRAV+Yts87kKdq0PP7pXfy6kDkUVs= +github.com/stretchr/testify v1.4.0/go.mod h1:j7eGeouHqKxXV5pUuKE4zz7dFj8WfuZ+81PSLYec5m4= github.com/stretchr/testify v1.7.0 h1:nwc3DEeHmmLAfoZucVR881uASk0Mfjw8xYJ99tb5CcY= github.com/stretchr/testify v1.7.0/go.mod h1:6Fq8oRcR53rry900zMqJjRRixrwX3KX962/h/Wwjteg= github.com/tianon/cuelabs-oci/ociregistry v0.0.0-20240329232705-b652d611e4b3 h1:kfAfFbiZ+2ZErqgqKtaMge1qeeE/0rnxuTl21G7fSwk= github.com/tianon/cuelabs-oci/ociregistry v0.0.0-20240329232705-b652d611e4b3/go.mod h1:pK23AUVXuNzzTpfMCA06sxZGeVQ/75FdVtW249de9Uo= -golang.org/x/net v0.7.0 h1:rJrUqqhjsgNp7KqAIc25s9pZnjU7TUcSY7HcVZjdn1g= -golang.org/x/net v0.7.0/go.mod h1:2Tu9+aMcznHK/AK1HMvgo6xiTLG5rD5rZLDS+rp2Bjs= +github.com/xanzy/ssh-agent v0.3.3 h1:+/15pJfg/RsTxqYcX6fHqOXZwwMP+2VyYWJeWM2qQFM= +github.com/xanzy/ssh-agent v0.3.3/go.mod h1:6dzNDKs0J9rVPHPhaGCukekBHKqfl+L3KghI1Bc68Uw= +github.com/yuin/goldmark v1.4.13/go.mod h1:6yULJ656Px+3vBD8DxQVa3kxgyrAnzto9xy5taEt/CY= +golang.org/x/crypto v0.0.0-20190308221718-c2843e01d9a2/go.mod h1:djNgcEr1/C05ACkg1iLfiJU5Ep61QUkGW8qpdssI0+w= +golang.org/x/crypto v0.0.0-20210921155107-089bfa567519/go.mod h1:GvvjBRRGRdwPK5ydBHafDWAxML/pGHZbMvKqRZ5+Abc= +golang.org/x/crypto v0.0.0-20220525230936-793ad666bf5e/go.mod h1:IxCIyHEi3zRg3s0A5j5BB6A9Jmi73HwBIUl50j+osU4= +golang.org/x/crypto v0.0.0-20220622213112-05595931fe9d/go.mod h1:IxCIyHEi3zRg3s0A5j5BB6A9Jmi73HwBIUl50j+osU4= +golang.org/x/crypto v0.0.0-20220826181053-bd7e27e6170d/go.mod h1:IxCIyHEi3zRg3s0A5j5BB6A9Jmi73HwBIUl50j+osU4= +golang.org/x/crypto v0.3.0/go.mod h1:hebNnKkNXi2UzZN1eVRvBB7co0a+JxK6XbPiWVs/3J4= +golang.org/x/crypto v0.14.0 h1:wBqGXzWJW6m1XrIKlAH0Hs1JJ7+9KBwnIO8v66Q9cHc= +golang.org/x/crypto v0.14.0/go.mod h1:MVFd36DqK4CsrnJYDkBA3VC4m2GkXAM0PvzMCn4JQf4= +golang.org/x/mod v0.6.0-dev.0.20220419223038-86c51ed26bb4/go.mod h1:jJ57K6gSWd91VN4djpZkiMVwK6gcyfeH4XE8wZrZaV4= +golang.org/x/mod v0.13.0 h1:I/DsJXRlw/8l/0c24sM9yb0T4z9liZTduXvdAWYiysY= +golang.org/x/mod v0.13.0/go.mod h1:hTbmBsO62+eylJbnUtE2MGJUyE7QWk4xUqPFrRgJ+7c= +golang.org/x/net v0.0.0-20190620200207-3b0461eec859/go.mod h1:z5CRVTTTmAJ677TzLLGU+0bjPO0LkuOLi4/5GtJWs/s= +golang.org/x/net v0.0.0-20210226172049-e18ecbb05110/go.mod h1:m0MpNAwzfU5UDzcl9v0D8zg8gWTRqZa9RBIspLL5mdg= +golang.org/x/net v0.0.0-20211112202133-69e39bad7dc2/go.mod h1:9nx3DQGgdP8bBQD5qxJ1jj9UTztislL4KSBs9R2vV5Y= +golang.org/x/net v0.0.0-20220722155237-a158d28d115b/go.mod h1:XRhObCWvk6IyKnWLug+ECip1KBveYUHfp+8e9klMJ9c= +golang.org/x/net v0.0.0-20220826154423-83b083e8dc8b/go.mod h1:YDH+HFinaLZZlnHAfSS6ZXJJ9M9t4Dl22yv3iI2vPwk= +golang.org/x/net v0.2.0/go.mod h1:KqCZLdyyvdV855qA2rE3GC2aiw5xGR5TEjj8smXukLY= +golang.org/x/net v0.16.0 h1:7eBu7KsSvFDtSXUIDbh3aqlK4DPsZ1rByC8PFfBThos= +golang.org/x/net v0.16.0/go.mod h1:NxSsAGuq816PNPmqtQdLE42eU2Fs7NoRIZrHJAlaCOE= +golang.org/x/sync v0.0.0-20190423024810-112230192c58/go.mod h1:RxMgew5VJxzue5/jJTE5uejpjVlOe/izrB70Jof72aM= +golang.org/x/sync v0.0.0-20220722155255-886fb9371eb4/go.mod h1:RxMgew5VJxzue5/jJTE5uejpjVlOe/izrB70Jof72aM= +golang.org/x/sync v0.4.0 h1:zxkM55ReGkDlKSM+Fu41A+zmbZuaPVbGMzvvdUPznYQ= +golang.org/x/sync v0.4.0/go.mod h1:FU7BRWz2tNW+3quACPkgCx/L+uEAv1htQ0V83Z9Rj+Y= +golang.org/x/sys v0.0.0-20190215142949-d0b11bdaac8a/go.mod h1:STP8DvDyc/dI5b8T5hshtkjS+E42TnysNCUPdjciGhY= +golang.org/x/sys v0.0.0-20191026070338-33540a1f6037/go.mod h1:h1NjWce9XRLGQEsW7wpKNCjG9DtNlClVuFLEZdDNbEs= +golang.org/x/sys v0.0.0-20200302150141-5c8b2ff67527/go.mod h1:h1NjWce9XRLGQEsW7wpKNCjG9DtNlClVuFLEZdDNbEs= +golang.org/x/sys v0.0.0-20201119102817-f84b799fce68/go.mod h1:h1NjWce9XRLGQEsW7wpKNCjG9DtNlClVuFLEZdDNbEs= +golang.org/x/sys v0.0.0-20210124154548-22da62e12c0c/go.mod h1:h1NjWce9XRLGQEsW7wpKNCjG9DtNlClVuFLEZdDNbEs= +golang.org/x/sys v0.0.0-20210320140829-1e4c9ba3b0c4/go.mod h1:h1NjWce9XRLGQEsW7wpKNCjG9DtNlClVuFLEZdDNbEs= +golang.org/x/sys v0.0.0-20210423082822-04245dca01da/go.mod h1:h1NjWce9XRLGQEsW7wpKNCjG9DtNlClVuFLEZdDNbEs= +golang.org/x/sys v0.0.0-20210615035016-665e8c7367d1/go.mod h1:oPkhp1MJrh7nUepCBck5+mAzfO9JrbApNNgaTdGDITg= +golang.org/x/sys v0.0.0-20211007075335-d3039528d8ac/go.mod h1:oPkhp1MJrh7nUepCBck5+mAzfO9JrbApNNgaTdGDITg= +golang.org/x/sys v0.0.0-20220520151302-bc2c85ada10a/go.mod h1:oPkhp1MJrh7nUepCBck5+mAzfO9JrbApNNgaTdGDITg= golang.org/x/sys v0.0.0-20220715151400-c0bba94af5f8/go.mod h1:oPkhp1MJrh7nUepCBck5+mAzfO9JrbApNNgaTdGDITg= +golang.org/x/sys v0.0.0-20220722155257-8c9f86f7a55f/go.mod h1:oPkhp1MJrh7nUepCBck5+mAzfO9JrbApNNgaTdGDITg= +golang.org/x/sys v0.0.0-20220728004956-3c1f35247d10/go.mod h1:oPkhp1MJrh7nUepCBck5+mAzfO9JrbApNNgaTdGDITg= +golang.org/x/sys v0.0.0-20220825204002-c680a09ffe64/go.mod h1:oPkhp1MJrh7nUepCBck5+mAzfO9JrbApNNgaTdGDITg= +golang.org/x/sys v0.2.0/go.mod h1:oPkhp1MJrh7nUepCBck5+mAzfO9JrbApNNgaTdGDITg= golang.org/x/sys v0.13.0 h1:Af8nKPmuFypiUBjVoU9V20FiaFXOcuZI21p0ycVYYGE= golang.org/x/sys v0.13.0/go.mod h1:oPkhp1MJrh7nUepCBck5+mAzfO9JrbApNNgaTdGDITg= -golang.org/x/text v0.7.0 h1:4BRB4x83lYWy72KwLD/qYDuTu7q9PjSagHvijDw7cLo= -golang.org/x/text v0.7.0/go.mod h1:mrYo+phRRbMaCq/xk9113O4dZlRixOauAjOtrjsXDZ8= +golang.org/x/term v0.0.0-20201126162022-7de9c90e9dd1/go.mod h1:bj7SfCRtBDWHUb9snDiAeCFNEtKQo2Wmx5Cou7ajbmo= +golang.org/x/term v0.0.0-20210927222741-03fcf44c2211/go.mod h1:jbD1KX2456YbFQfuXm/mYQcufACuNUgVhRMnK/tPxf8= +golang.org/x/term v0.0.0-20220722155259-a9ba230a4035/go.mod h1:jbD1KX2456YbFQfuXm/mYQcufACuNUgVhRMnK/tPxf8= +golang.org/x/term v0.2.0/go.mod h1:TVmDHMZPmdnySmBfhjOoOdhjzdE1h4u1VwSiw2l1Nuc= +golang.org/x/term v0.13.0 h1:bb+I9cTfFazGW51MZqBVmZy7+JEJMouUHTUSKVQLBek= +golang.org/x/term v0.13.0/go.mod h1:LTmsnFJwVN6bCy1rVCoS+qHT1HhALEFxKncY3WNNh4U= +golang.org/x/text v0.3.0/go.mod h1:NqM8EUOU14njkJ3fqMW+pc6Ldnwhi/IjpwHt7yyuwOQ= +golang.org/x/text v0.3.3/go.mod h1:5Zoc/QRtKVWzQhOtBMvqHzDpF6irO9z98xDceosuGiQ= +golang.org/x/text v0.3.6/go.mod h1:5Zoc/QRtKVWzQhOtBMvqHzDpF6irO9z98xDceosuGiQ= +golang.org/x/text v0.3.7/go.mod h1:u+2+/6zg+i71rQMx5EYifcz6MCKuco9NR6JIITiCfzQ= +golang.org/x/text v0.4.0/go.mod h1:mrYo+phRRbMaCq/xk9113O4dZlRixOauAjOtrjsXDZ8= +golang.org/x/text v0.13.0 h1:ablQoSUd0tRdKxZewP80B+BaqeKJuVhuRxj/dkrun3k= +golang.org/x/text v0.13.0/go.mod h1:TvPlkZtksWOMsz7fbANvkp4WM8x/WCo/om8BMLbz+aE= golang.org/x/time v0.5.0 h1:o7cqy6amK/52YcAKIPlM3a+Fpj35zvRj2TP+e1xFSfk= golang.org/x/time v0.5.0/go.mod h1:3BpzKBy/shNhVucY/MWOyx10tF3SFh9QdLuxbVysPQM= +golang.org/x/tools v0.0.0-20180917221912-90fa682c2a6e/go.mod h1:n7NCudcB/nEzxVGmLbDWY5pfWTLqBcC2KZ6jyYvM4mQ= +golang.org/x/tools v0.0.0-20191119224855-298f0cb1881e/go.mod h1:b+2E5dAYhXwXZwtnZ6UAqBI28+e2cm9otk0dWdXHAEo= +golang.org/x/tools v0.1.12/go.mod h1:hNGJHUnrk76NpqgfD5Aqm5Crs+Hm0VOH/i9J2+nxYbc= golang.org/x/tools v0.14.0 h1:jvNa2pY0M4r62jkRQ6RwEZZyPcymeL9XZMLBbV7U2nc= golang.org/x/tools v0.14.0/go.mod h1:uYBEerGOWcJyEORxN+Ek8+TT266gXkNlHdJBwexUsBg= +golang.org/x/xerrors v0.0.0-20190717185122-a985d3407aa7/go.mod h1:I/5z698sn9Ka8TeJc9MKroUUfqBBauWjQqLJ2OPfmY0= golang.org/x/xerrors v0.0.0-20191204190536-9bdfabe68543/go.mod h1:I/5z698sn9Ka8TeJc9MKroUUfqBBauWjQqLJ2OPfmY0= google.golang.org/genproto v0.0.0-20221207170731-23e4bf6bdc37 h1:jmIfw8+gSvXcZSgaFAGyInDXeWzUhvYH57G/5GKMn70= google.golang.org/genproto v0.0.0-20221207170731-23e4bf6bdc37/go.mod h1:RGgjbofJ8xD9Sq1VVhDM1Vok1vRONV+rg+CjzG4SZKM= @@ -55,7 +162,16 @@ google.golang.org/protobuf v1.26.0/go.mod h1:9q0QmTI4eRPtz6boOQmLYwt+qCgq0jsYwAQ google.golang.org/protobuf v1.28.1 h1:d0NfwRgPtno5B1Wa6L2DAG+KivqkdutMf1UhdNx175w= google.golang.org/protobuf v1.28.1/go.mod h1:HV8QOd/L58Z+nl8r43ehVNZIU/HEI6OcFqwMG9pJV4I= gopkg.in/check.v1 v0.0.0-20161208181325-20d25e280405/go.mod h1:Co6ibVJAznAaIkqp8huTwlJQCZ016jof/cbN4VW5Yz0= +gopkg.in/check.v1 v1.0.0-20190902080502-41f04d3bba15/go.mod h1:Co6ibVJAznAaIkqp8huTwlJQCZ016jof/cbN4VW5Yz0= +gopkg.in/check.v1 v1.0.0-20200227125254-8fa46927fb4f/go.mod h1:Co6ibVJAznAaIkqp8huTwlJQCZ016jof/cbN4VW5Yz0= +gopkg.in/check.v1 v1.0.0-20201130134442-10cb98267c6c h1:Hei/4ADfdWqJk1ZMxUNpqntNwaWcugrBjAiHlqqRiVk= +gopkg.in/check.v1 v1.0.0-20201130134442-10cb98267c6c/go.mod h1:JHkPIbrfpd72SG/EVd6muEfDQjcINNoR0C8j2r3qZ4Q= +gopkg.in/warnings.v0 v0.1.2 h1:wFXVbFY8DY5/xOe1ECiWdKCzZlxgshcYVNkBHstARME= +gopkg.in/warnings.v0 v0.1.2/go.mod h1:jksf8JmL6Qr/oQM2OXTHunEvvTAsrWBLb6OOjuVWRNI= +gopkg.in/yaml.v2 v2.2.2/go.mod h1:hI93XBmqTisBFMUTm0b8Fm+jr3Dg1NNxqwp+5A1VGuI= +gopkg.in/yaml.v2 v2.2.4/go.mod h1:hI93XBmqTisBFMUTm0b8Fm+jr3Dg1NNxqwp+5A1VGuI= gopkg.in/yaml.v3 v3.0.0-20200313102051-9f266ea9e77c/go.mod h1:K4uyk7z7BCEPqu6E+C64Yfv1cQ7kz7rIZviUmN+EgEM= +gopkg.in/yaml.v3 v3.0.0/go.mod h1:K4uyk7z7BCEPqu6E+C64Yfv1cQ7kz7rIZviUmN+EgEM= gopkg.in/yaml.v3 v3.0.1 h1:fxVm/GzAzEWqLHuvctI91KS9hhNmmWOoWu0XTYJS7CA= gopkg.in/yaml.v3 v3.0.1/go.mod h1:K4uyk7z7BCEPqu6E+C64Yfv1cQ7kz7rIZviUmN+EgEM= gotest.tools/v3 v3.0.3 h1:4AuOwCGf4lLR9u3YOe2awrHygurzhO/HeQ6laiA6Sx0= From 0c82ab8f905fd3f7f829e22211cedba090d0ea0b Mon Sep 17 00:00:00 2001 From: Tianon Gravi Date: Mon, 3 Aug 2026 10:21:45 -0700 Subject: [PATCH 2/4] WIP: holes filling take 2 --- cmd/holes/main.go | 268 ++++++++++++++++++++++++++++++++++++---------- 1 file changed, 212 insertions(+), 56 deletions(-) diff --git a/cmd/holes/main.go b/cmd/holes/main.go index 24a6f74..a5e1520 100644 --- a/cmd/holes/main.go +++ b/cmd/holes/main.go @@ -1,19 +1,24 @@ package main import ( + "bufio" + "context" "encoding/json" "fmt" "io" + "math" "os" + "os/exec" "path/filepath" + "runtime" + "strconv" + "strings" + "sync" // encoding/json/v2, but not yet jsonv2 "github.com/go-json-experiment/json" "github.com/go-json-experiment/json/jsontext" - gogit "github.com/go-git/go-git/v5" - "github.com/go-git/go-git/v5/plumbing/object" - "github.com/go-git/go-git/v5/plumbing/storer" ocispec "github.com/opencontainers/image-spec/specs-go/v1" ) @@ -32,6 +37,13 @@ func tagArchKey(tag, arch string) string { return tag + "\x00" + arch } +// blobMatch is a single resolved entry found in a historical builds.json blob that matches an unfilled hole. +type blobMatch struct { + slotIdx int // position in newest-to-oldest order (lower = more recent) + holeIdx int + resolved *ocispec.Index +} + func main() { if len(os.Args) < 3 { fmt.Fprintf(os.Stderr, "usage: %s sources.json builds.json\n", os.Args[0]) @@ -114,73 +126,218 @@ func main() { } fmt.Fprintf(os.Stderr, "searching git history for %d hole(s)\n", len(holes)) - // --- Step 4: open the git repo that contains builds.json --- + // --- Step 4: locate the git repo that contains builds.json --- + // Use native git rather than go-git: "git rev-parse --show-toplevel" is fast and handles + // DetectDotGit automatically. absBuilds, err := filepath.Abs(buildsFile) if err != nil { panic(err) } - repo, err := gogit.PlainOpenWithOptions(filepath.Dir(absBuilds), &gogit.PlainOpenOptions{ - DetectDotGit: true, - }) + rootCmd := exec.Command("git", "rev-parse", "--show-toplevel") + rootCmd.Dir = filepath.Dir(absBuilds) + rootOut, err := rootCmd.Output() if err != nil { - panic(err) + panic(fmt.Errorf("locating git repo root: %w", err)) } - wt, err := repo.Worktree() - if err != nil { - panic(err) - } - // builds.json path relative to the worktree root, for commit.File() calls - relBuilds, err := filepath.Rel(wt.Filesystem.Root(), absBuilds) + repoRoot := strings.TrimRight(string(rootOut), "\r\n") + relBuilds, err := filepath.Rel(repoRoot, absBuilds) if err != nil { panic(err) } - // --- Step 5: walk git history, filling holes as we go --- + // --- Step 5a: collect ordered unique blob hashes via git log --- + // "git log --raw" emits one diff-stat line per commit showing old and new blob hashes; + // we extract the new blob hash for each version of builds.json in newest-to-oldest order. + // This is a single fast git command rather than go-git decompressing every commit+tree object. - results := make([]*ocispec.Index, len(holes)) // nil == still unfilled - unfilledCount := len(holes) + type blobSlot struct { + idx int + blobHash string // 40-char hex + } - head, err := repo.Head() + logCmd := exec.Command("git", "log", "--raw", "--no-abbrev", "--format=", "--", relBuilds) + logCmd.Dir = repoRoot + logOut, err := logCmd.StdoutPipe() if err != nil { panic(err) } - commitIter, err := repo.Log(&gogit.LogOptions{From: head.Hash()}) - if err != nil { + if err := logCmd.Start(); err != nil { panic(err) } - err = commitIter.ForEach(func(c *object.Commit) error { - if unfilledCount == 0 { - return storer.ErrStop - } + var slots []blobSlot + seenBlob := make(map[string]bool) - file, err := c.File(relBuilds) - if err != nil { - // builds.json absent in this commit (e.g. very early repo history) - return nil + logScanner := bufio.NewScanner(logOut) + for logScanner.Scan() { + line := logScanner.Text() + if !strings.HasPrefix(line, ":") { + continue // blank lines and non-diff lines } - reader, err := file.Reader() - if err != nil { - return nil + parts := strings.Fields(line) + if len(parts) < 4 { + continue } - defer reader.Close() - - newFilled, err := searchBlob(reader, keyToHoles, results) - if err != nil { - fmt.Fprintf(os.Stderr, "warning: skipping blob in commit %s: %v\n", c.Hash, err) - return nil + newBlob := parts[3] + if newBlob == "0000000000000000000000000000000000000000" { + continue // deletion: builds.json was removed in this commit } - unfilledCount -= newFilled - if newFilled > 0 { - fmt.Fprintf(os.Stderr, "filled %d hole(s) from commit %s (%d remaining)\n", newFilled, c.Hash, unfilledCount) + if seenBlob[newBlob] { + continue // identical blob already queued } - return nil - }) - if err != nil { + seenBlob[newBlob] = true + slots = append(slots, blobSlot{len(slots), newBlob}) + } + if err := logScanner.Err(); err != nil { panic(err) } + if err := logCmd.Wait(); err != nil { + panic(fmt.Errorf("git log: %w", err)) + } + fmt.Fprintf(os.Stderr, "scanning %d unique blob version(s) of %s\n", len(slots), relBuilds) + + // --- Step 5b: parse blobs in parallel with early exit --- + // Each worker drives its own "git cat-file --batch" process (no go-git internal mutex). + // A coordinator goroutine receives all matches and cancels the context once every hole has + // been filled; the feeder goroutine stops sending new jobs immediately on cancellation. + // In the common case (holes filled from very recent history), this terminates after parsing + // only the first few blobs rather than all ~14k. + + results := make([]*ocispec.Index, len(holes)) + bestSlot := make([]int, len(holes)) + for i := range bestSlot { + bestSlot[i] = math.MaxInt + } + + ctx, cancel := context.WithCancel(context.Background()) + defer cancel() + + matches := make(chan blobMatch, 256) + + // coordinator: single goroutine owns bestSlot/results (no locks needed), cancels when done + var coordDone sync.WaitGroup + coordDone.Add(1) + go func() { + defer coordDone.Done() + remaining := len(holes) + for m := range matches { + if m.slotIdx < bestSlot[m.holeIdx] { + if bestSlot[m.holeIdx] == math.MaxInt { + remaining-- + if remaining == 0 { + cancel() // all holes filled; stop feeder and workers + } + } + bestSlot[m.holeIdx] = m.slotIdx + results[m.holeIdx] = m.resolved + } + } + }() + + numWorkers := runtime.NumCPU() + if numWorkers > len(slots) { + numWorkers = len(slots) + } + + jobs := make(chan blobSlot, numWorkers) + + // feeder: stops the moment ctx is cancelled (all holes filled) + go func() { + defer close(jobs) + for _, slot := range slots { + select { + case jobs <- slot: + case <-ctx.Done(): + return + } + } + }() + + var wg sync.WaitGroup + for w := range numWorkers { + wg.Add(1) + go func(workerID int) { + defer wg.Done() + + cmd := exec.Command("git", "cat-file", "--batch") + cmd.Dir = repoRoot + catIn, err := cmd.StdinPipe() + if err != nil { + fmt.Fprintf(os.Stderr, "warning: worker %d: creating git cat-file stdin pipe: %v\n", workerID, err) + return + } + catOut, err := cmd.StdoutPipe() + if err != nil { + fmt.Fprintf(os.Stderr, "warning: worker %d: creating git cat-file stdout pipe: %v\n", workerID, err) + return + } + if err := cmd.Start(); err != nil { + fmt.Fprintf(os.Stderr, "warning: worker %d: starting git cat-file: %v\n", workerID, err) + return + } + + catBuf := bufio.NewReader(catOut) + + loop: + for slot := range jobs { + if ctx.Err() != nil { + break loop // all holes filled; stop requesting new blobs + } + + if _, err := fmt.Fprintf(catIn, "%s\n", slot.blobHash); err != nil { + fmt.Fprintf(os.Stderr, "warning: worker %d: writing to git cat-file (blob %s): %v\n", workerID, slot.blobHash, err) + continue + } + + // response header: " blob \n" or " missing\n" + line, err := catBuf.ReadString('\n') + if err != nil { + fmt.Fprintf(os.Stderr, "warning: worker %d: reading git cat-file header (blob %s): %v\n", workerID, slot.blobHash, err) + continue + } + parts := strings.Fields(line) + if len(parts) < 3 || parts[1] != "blob" { + fmt.Fprintf(os.Stderr, "warning: worker %d: unexpected git cat-file response %q for blob %s\n", workerID, strings.TrimRight(line, "\n"), slot.blobHash) + continue + } + size, err := strconv.ParseInt(parts[2], 10, 64) + if err != nil { + fmt.Fprintf(os.Stderr, "warning: worker %d: bad size %q for blob %s: %v\n", workerID, parts[2], slot.blobHash, err) + continue + } + + // LimitedReader prevents the decoder from reading into the next blob's header + lr := &io.LimitedReader{R: catBuf, N: size} + if err := searchBlob(ctx, lr, keyToHoles, slot.idx, matches); err != nil { + fmt.Fprintf(os.Stderr, "warning: worker %d: skipping blob %s: %v\n", workerID, slot.blobHash, err) + } + // drain any unread bytes (after a parse error or ctx-triggered early return) + if _, err := io.Copy(io.Discard, lr); err != nil { + fmt.Fprintf(os.Stderr, "warning: worker %d: draining blob %s: %v\n", workerID, slot.blobHash, err) + } + if _, err := catBuf.ReadByte(); err != nil { // trailing LF after blob content + fmt.Fprintf(os.Stderr, "warning: worker %d: reading trailing newline for blob %s: %v\n", workerID, slot.blobHash, err) + } + } + + go io.Copy(io.Discard, catOut) // let git flush and exit without blocking on a full pipe + catIn.Close() + cmd.Wait() + }(w) + } + + wg.Wait() + close(matches) // signal coordinator that no more matches are coming + coordDone.Wait() // drain any in-flight matches before reading results + + unfilledCount := 0 + for _, s := range bestSlot { + if s == math.MaxInt { + unfilledCount++ + } + } if unfilledCount > 0 { fmt.Fprintf(os.Stderr, "warning: %d hole(s) could not be filled from git history\n", unfilledCount) } @@ -207,17 +364,17 @@ func main() { } } -// searchBlob streams a single builds.json blob (from any historical commit), looking for resolved entries whose (tag, arch) pairs match unfilled holes. It returns the number of newly-filled holes. +// searchBlob streams a single builds.json blob (from any historical commit), looking for resolved entries whose (tag, arch) pairs match any hole. Matches are sent to the matches channel tagged with slotIdx (lower = more recent); the coordinator goroutine owns bestSlot/results so no synchronisation is needed here. ctx is checked between entries: once all holes are globally filled the caller cancels ctx and this function returns early, letting the caller drain the remaining unread bytes. // // Uses encoding/json/v2 (jsontext + jsonv2.UnmarshalDecode) to stream the top-level object one entry at a time, avoiding loading the entire blob into memory at once. -func searchBlob(r io.Reader, keyToHoles map[string][]int, results []*ocispec.Index) (int, error) { +func searchBlob(ctx context.Context, r io.Reader, keyToHoles map[string][]int, slotIdx int, matches chan<- blobMatch) error { dec := jsontext.NewDecoder(r) // consume the opening '{' of the top-level builds.json object if tok, err := dec.ReadToken(); err != nil { - return 0, err + return err } else if tok.Kind() != '{' { - return 0, fmt.Errorf("expected '{', got %v", tok) + return fmt.Errorf("expected '{', got %v", tok) } // minimal shape we care about in each builds.json entry @@ -234,17 +391,20 @@ func searchBlob(r io.Reader, keyToHoles map[string][]int, results []*ocispec.Ind } `json:"source"` } - newlyFilled := 0 for dec.PeekKind() != '}' { + if ctx.Err() != nil { + return nil // all holes globally filled; return early, caller drains remainder + } + // read the buildId key (discard it) if _, err := dec.ReadToken(); err != nil { - return newlyFilled, err + return err } // decode the full value for this buildId into our minimal struct var e entry if err := jsonv2.UnmarshalDecode(dec, &e); err != nil { - return newlyFilled, err + return err } if e.Build.Resolved == nil { @@ -258,14 +418,10 @@ func searchBlob(r io.Reader, keyToHoles map[string][]int, results []*ocispec.Ind } for _, tag := range archData.Tags { for _, holeIdx := range keyToHoles[tagArchKey(tag, arch)] { - if results[holeIdx] != nil { - continue // already filled by a more-recent commit - } - results[holeIdx] = e.Build.Resolved - newlyFilled++ + matches <- blobMatch{slotIdx: slotIdx, holeIdx: holeIdx, resolved: e.Build.Resolved} } } } - return newlyFilled, nil + return nil } From 5a277b4bb0e3dff1f64b93e701f02252e5c9c52c Mon Sep 17 00:00:00 2001 From: Tianon Gravi Date: Mon, 17 Aug 2026 16:10:15 -0700 Subject: [PATCH 3/4] WIP: holes filling take 3 --- cmd/holes/README.md | 14 ++ cmd/holes/main.go | 463 ++++++++++++++++++++++++++------------------ 2 files changed, 291 insertions(+), 186 deletions(-) diff --git a/cmd/holes/README.md b/cmd/holes/README.md index 1a6cd88..b08d081 100644 --- a/cmd/holes/README.md +++ b/cmd/holes/README.md @@ -27,3 +27,17 @@ For architectures with multiple OS-version variants under the same tag (e.g. `wi ## Git history search The git history of the meta repo contains every prior `builds.json` state. Walking it backward is the only source of fallback data that does not require live registry queries (which are expensive enough that put-shared runs only every ~3 hours). The walk must be a single linear backward pass over *all* holes simultaneously -- exponential or binary search is not safe because a tag can be added, fully built, *and* removed within a gap of skipped commits, leaving no evidence on either side of the jump. + +## Performance + +Blob enumeration uses `git log --raw` (a single native-git command) rather than go-git's commit-graph traversal, which decompresses every commit and tree object in-process and is substantially slower. + +Blobs are parsed in parallel (`runtime.NumCPU()` workers, each with its own `git cat-file --batch` subprocess to avoid go-git's internal pack-file mutex). A coordinator goroutine cancels the scan the moment every initially-unfilled hole has been filled, so recently-built tags cost only a few blobs' worth of work. + +An optional third argument `prev-holes.json` pre-fills holes from the previous run's output before touching git history. The previous `holes.json` is a valid cache because any `(tag, arch)` that was a hole on the prior run and is still a hole now can reuse the same fallback OCI index -- and as holes get resolved during builds they disappear from the current holes set (resolved in `builds.json`) so the cache never returns data for a hole that no longer exists. On a warm run (most holes already in the cache) the git history search only covers newly-appeared holes, which are typically a small fraction of the total. Typical invocation: + +```console +$ holes sources.json builds.json holes.json > holes-new.json && mv holes-new.json holes.json +``` + +Setting `$HOLES_SINCE` to any expression `git log --since` understands (eg `2.weeks`, `30.days`, `2024-01-01`) limits the history search to that window. Holes still unfilled after the window are left empty (library deploy skips them). This trades correctness -- a tag last resolved before the cutoff gets no fallback -- for a bounded worst-case runtime. It is most useful when brand-new tags (which can never be filled from history) regularly appear in large batches and would otherwise force a full scan by preventing the all-holes-filled early exit. It composes well with the cache: the cache handles previously-known holes instantly, and `$HOLES_SINCE` bounds the search for any new ones. diff --git a/cmd/holes/main.go b/cmd/holes/main.go index a5e1520..d7e13ea 100644 --- a/cmd/holes/main.go +++ b/cmd/holes/main.go @@ -46,11 +46,20 @@ type blobMatch struct { func main() { if len(os.Args) < 3 { - fmt.Fprintf(os.Stderr, "usage: %s sources.json builds.json\n", os.Args[0]) + fmt.Fprintf(os.Stderr, "usage: %s sources.json builds.json [prev-holes.json]\n", os.Args[0]) os.Exit(1) } sourcesFile := os.Args[1] buildsFile := os.Args[2] + prevHolesFile := "" + if len(os.Args) >= 4 { + prevHolesFile = os.Args[3] + } + + // HOLES_SINCE, if set, is passed directly to "git log --since=..." to limit how far back the + // history search reaches; accepts any expression git understands (e.g. "2.weeks", "30.days", + // "2024-01-01"); see README for trade-offs + holesSince := os.Getenv("HOLES_SINCE") // --- Step 1: parse sources.json for the full expected (sourceId, arch) set --- @@ -126,235 +135,317 @@ func main() { } fmt.Fprintf(os.Stderr, "searching git history for %d hole(s)\n", len(holes)) - // --- Step 4: locate the git repo that contains builds.json --- - // Use native git rather than go-git: "git rev-parse --show-toplevel" is fast and handles - // DetectDotGit automatically. - - absBuilds, err := filepath.Abs(buildsFile) - if err != nil { - panic(err) - } - rootCmd := exec.Command("git", "rev-parse", "--show-toplevel") - rootCmd.Dir = filepath.Dir(absBuilds) - rootOut, err := rootCmd.Output() - if err != nil { - panic(fmt.Errorf("locating git repo root: %w", err)) - } - repoRoot := strings.TrimRight(string(rootOut), "\r\n") - relBuilds, err := filepath.Rel(repoRoot, absBuilds) - if err != nil { - panic(err) + // results[i] holds the best resolved OCI index found for holes[i]; nil means unfilled. + // bestSlot[i] tracks the slot index (lower = more recent) of the result in holes[i]: + // math.MaxInt = no result yet (unfilled); counted in initialRemaining; git will search for it + // math.MaxInt-1 = filled from prev-holes.json cache; not counted; git can still improve it + // math.MaxInt-2 = cache recorded null (previously searched, not found); not counted; git can + // still passively improve it if a match is encountered while scanning for others + // 0..N = filled from git history blob at that slot index + results := make([]*ocispec.Index, len(holes)) + bestSlot := make([]int, len(holes)) + for i := range bestSlot { + bestSlot[i] = math.MaxInt } - // --- Step 5a: collect ordered unique blob hashes via git log --- - // "git log --raw" emits one diff-stat line per commit showing old and new blob hashes; - // we extract the new blob hash for each version of builds.json in newest-to-oldest order. - // This is a single fast git command rather than go-git decompressing every commit+tree object. + // --- Step 3.5: pre-fill holes from previous holes.json (cache) --- + // The previous run's holes.json is a valid cache: any (tag, arch) that was a hole then + // and is still a hole now can reuse the same fallback resolved index without re-scanning + // git history. Holes that have since been filled (resolved in builds.json) are no longer + // in the current holes set, so the cache never returns data for a hole that no longer exists. + // + // For the windows multi-variant case where multiple holes share the same (tag, arch) key, + // cached entries are distributed across the current holes for that key in order. + + if prevHolesFile != "" { + var prevHoles map[string]map[string][]*ocispec.Index + f, err := os.Open(prevHolesFile) + if err != nil { + fmt.Fprintf(os.Stderr, "warning: could not open %s: %v\n", prevHolesFile, err) + } else { + if err := json.NewDecoder(f).Decode(&prevHoles); err != nil { + fmt.Fprintf(os.Stderr, "warning: ignoring malformed %s: %v\n", prevHolesFile, err) + } + f.Close() + } - type blobSlot struct { - idx int - blobHash string // 40-char hex + cacheHits := 0 + cacheNulls := 0 + for tag, arches := range prevHoles { + for arch, cachedIndices := range arches { + holeIndices := keyToHoles[tagArchKey(tag, arch)] + if cachedIndices == nil { + // null in previous holes.json: was searched and not found; skip git search + // (git can still passively improve these if a match is encountered for other holes) + for _, holeIdx := range holeIndices { + bestSlot[holeIdx] = math.MaxInt - 2 + cacheNulls++ + } + continue + } + for j, holeIdx := range holeIndices { + if j >= len(cachedIndices) || cachedIndices[j] == nil { + break + } + results[holeIdx] = cachedIndices[j] + bestSlot[holeIdx] = math.MaxInt - 1 // filled from cache; git can improve + cacheHits++ + } + } + } + if cacheHits > 0 || cacheNulls > 0 { + fmt.Fprintf(os.Stderr, "pre-filled %d hole(s) from %s (%d unfillable skipped)\n", cacheHits, prevHolesFile, cacheNulls) + } } - logCmd := exec.Command("git", "log", "--raw", "--no-abbrev", "--format=", "--", relBuilds) - logCmd.Dir = repoRoot - logOut, err := logCmd.StdoutPipe() - if err != nil { - panic(err) - } - if err := logCmd.Start(); err != nil { - panic(err) + // count holes with no cache result at all (math.MaxInt); math.MaxInt-1 (filled) and + // math.MaxInt-2 (null/unfillable) are both excluded -- neither needs a git search + initialRemaining := 0 + for _, s := range bestSlot { + if s == math.MaxInt { + initialRemaining++ + } } - var slots []blobSlot - seenBlob := make(map[string]bool) + if initialRemaining == 0 { + fmt.Fprintf(os.Stderr, "all holes accounted for by cache; skipping git history search\n") + } else { + // --- Step 4: locate the git repo that contains builds.json --- + // Use native git rather than go-git: "git rev-parse --show-toplevel" is fast and handles + // DetectDotGit automatically. - logScanner := bufio.NewScanner(logOut) - for logScanner.Scan() { - line := logScanner.Text() - if !strings.HasPrefix(line, ":") { - continue // blank lines and non-diff lines - } - parts := strings.Fields(line) - if len(parts) < 4 { - continue + absBuilds, err := filepath.Abs(buildsFile) + if err != nil { + panic(err) } - newBlob := parts[3] - if newBlob == "0000000000000000000000000000000000000000" { - continue // deletion: builds.json was removed in this commit + rootCmd := exec.Command("git", "rev-parse", "--show-toplevel") + rootCmd.Dir = filepath.Dir(absBuilds) + rootOut, err := rootCmd.Output() + if err != nil { + panic(fmt.Errorf("locating git repo root: %w", err)) } - if seenBlob[newBlob] { - continue // identical blob already queued + repoRoot := strings.TrimRight(string(rootOut), "\r\n") + relBuilds, err := filepath.Rel(repoRoot, absBuilds) + if err != nil { + panic(err) } - seenBlob[newBlob] = true - slots = append(slots, blobSlot{len(slots), newBlob}) - } - if err := logScanner.Err(); err != nil { - panic(err) - } - if err := logCmd.Wait(); err != nil { - panic(fmt.Errorf("git log: %w", err)) - } - - fmt.Fprintf(os.Stderr, "scanning %d unique blob version(s) of %s\n", len(slots), relBuilds) - // --- Step 5b: parse blobs in parallel with early exit --- - // Each worker drives its own "git cat-file --batch" process (no go-git internal mutex). - // A coordinator goroutine receives all matches and cancels the context once every hole has - // been filled; the feeder goroutine stops sending new jobs immediately on cancellation. - // In the common case (holes filled from very recent history), this terminates after parsing - // only the first few blobs rather than all ~14k. - - results := make([]*ocispec.Index, len(holes)) - bestSlot := make([]int, len(holes)) - for i := range bestSlot { - bestSlot[i] = math.MaxInt - } + // --- Step 5a: collect ordered unique blob hashes via git log --- + // "git log --raw" emits one diff-stat line per commit showing old and new blob hashes; + // we extract the new blob hash for each version of builds.json in newest-to-oldest order. + // This is a single fast git command rather than go-git decompressing every commit+tree object. - ctx, cancel := context.WithCancel(context.Background()) - defer cancel() - - matches := make(chan blobMatch, 256) - - // coordinator: single goroutine owns bestSlot/results (no locks needed), cancels when done - var coordDone sync.WaitGroup - coordDone.Add(1) - go func() { - defer coordDone.Done() - remaining := len(holes) - for m := range matches { - if m.slotIdx < bestSlot[m.holeIdx] { - if bestSlot[m.holeIdx] == math.MaxInt { - remaining-- - if remaining == 0 { - cancel() // all holes filled; stop feeder and workers - } - } - bestSlot[m.holeIdx] = m.slotIdx - results[m.holeIdx] = m.resolved - } + type blobSlot struct { + idx int + blobHash string // 40-char hex } - }() - numWorkers := runtime.NumCPU() - if numWorkers > len(slots) { - numWorkers = len(slots) - } + logArgs := []string{"log", "--raw", "--no-abbrev", "--format="} + if holesSince != "" { + logArgs = append(logArgs, "--since="+holesSince) + } + logArgs = append(logArgs, "--", relBuilds) + logCmd := exec.Command("git", logArgs...) + logCmd.Dir = repoRoot + logOut, err := logCmd.StdoutPipe() + if err != nil { + panic(err) + } + if err := logCmd.Start(); err != nil { + panic(err) + } - jobs := make(chan blobSlot, numWorkers) + var slots []blobSlot + seenBlob := make(map[string]bool) - // feeder: stops the moment ctx is cancelled (all holes filled) - go func() { - defer close(jobs) - for _, slot := range slots { - select { - case jobs <- slot: - case <-ctx.Done(): - return + logScanner := bufio.NewScanner(logOut) + for logScanner.Scan() { + line := logScanner.Text() + if !strings.HasPrefix(line, ":") { + continue // blank lines and non-diff lines } - } - }() - - var wg sync.WaitGroup - for w := range numWorkers { - wg.Add(1) - go func(workerID int) { - defer wg.Done() - - cmd := exec.Command("git", "cat-file", "--batch") - cmd.Dir = repoRoot - catIn, err := cmd.StdinPipe() - if err != nil { - fmt.Fprintf(os.Stderr, "warning: worker %d: creating git cat-file stdin pipe: %v\n", workerID, err) - return + parts := strings.Fields(line) + if len(parts) < 4 { + continue } - catOut, err := cmd.StdoutPipe() - if err != nil { - fmt.Fprintf(os.Stderr, "warning: worker %d: creating git cat-file stdout pipe: %v\n", workerID, err) - return + newBlob := parts[3] + if newBlob == "0000000000000000000000000000000000000000" { + continue // deletion: builds.json was removed in this commit } - if err := cmd.Start(); err != nil { - fmt.Fprintf(os.Stderr, "warning: worker %d: starting git cat-file: %v\n", workerID, err) - return + if seenBlob[newBlob] { + continue // identical blob already queued } + seenBlob[newBlob] = true + slots = append(slots, blobSlot{len(slots), newBlob}) + } + if err := logScanner.Err(); err != nil { + panic(err) + } + if err := logCmd.Wait(); err != nil { + panic(fmt.Errorf("git log: %w", err)) + } - catBuf := bufio.NewReader(catOut) - - loop: - for slot := range jobs { - if ctx.Err() != nil { - break loop // all holes filled; stop requesting new blobs + if holesSince != "" { + fmt.Fprintf(os.Stderr, "note: history search limited to builds since %q ($HOLES_SINCE); %d hole(s) may go unfilled\n", holesSince, initialRemaining) + } + fmt.Fprintf(os.Stderr, "scanning %d unique blob version(s) of %s\n", len(slots), relBuilds) + + // --- Step 5b: parse blobs in parallel with early exit --- + // Each worker drives its own "git cat-file --batch" process (no go-git internal mutex). + // A coordinator goroutine receives all matches and cancels the context once every + // initially-unfilled hole has been filled; the feeder goroutine stops immediately. + // Cache-filled holes (bestSlot == math.MaxInt-1) are not counted in initialRemaining + // but can still be improved to a more recent git entry for free. + + ctx, cancel := context.WithCancel(context.Background()) + defer cancel() + + matches := make(chan blobMatch, 256) + + // coordinator: single goroutine owns bestSlot/results (no locks needed), cancels when done + var coordDone sync.WaitGroup + coordDone.Add(1) + go func() { + defer coordDone.Done() + remaining := initialRemaining + for m := range matches { + if m.slotIdx < bestSlot[m.holeIdx] { + if bestSlot[m.holeIdx] == math.MaxInt { + remaining-- + if remaining == 0 { + cancel() // all initially-unfilled holes filled; stop feeder and workers + } + } + bestSlot[m.holeIdx] = m.slotIdx + results[m.holeIdx] = m.resolved } + } + }() - if _, err := fmt.Fprintf(catIn, "%s\n", slot.blobHash); err != nil { - fmt.Fprintf(os.Stderr, "warning: worker %d: writing to git cat-file (blob %s): %v\n", workerID, slot.blobHash, err) - continue + numWorkers := runtime.NumCPU() + if numWorkers > len(slots) { + numWorkers = len(slots) + } + + jobs := make(chan blobSlot, numWorkers) + + // feeder: stops the moment ctx is cancelled (all initially-unfilled holes filled) + go func() { + defer close(jobs) + for _, slot := range slots { + select { + case jobs <- slot: + case <-ctx.Done(): + return } + } + }() - // response header: " blob \n" or " missing\n" - line, err := catBuf.ReadString('\n') + var wg sync.WaitGroup + for w := range numWorkers { + wg.Add(1) + go func(workerID int) { + defer wg.Done() + + cmd := exec.Command("git", "cat-file", "--batch") + cmd.Dir = repoRoot + catIn, err := cmd.StdinPipe() if err != nil { - fmt.Fprintf(os.Stderr, "warning: worker %d: reading git cat-file header (blob %s): %v\n", workerID, slot.blobHash, err) - continue - } - parts := strings.Fields(line) - if len(parts) < 3 || parts[1] != "blob" { - fmt.Fprintf(os.Stderr, "warning: worker %d: unexpected git cat-file response %q for blob %s\n", workerID, strings.TrimRight(line, "\n"), slot.blobHash) - continue + fmt.Fprintf(os.Stderr, "warning: worker %d: creating git cat-file stdin pipe: %v\n", workerID, err) + return } - size, err := strconv.ParseInt(parts[2], 10, 64) + catOut, err := cmd.StdoutPipe() if err != nil { - fmt.Fprintf(os.Stderr, "warning: worker %d: bad size %q for blob %s: %v\n", workerID, parts[2], slot.blobHash, err) - continue - } - - // LimitedReader prevents the decoder from reading into the next blob's header - lr := &io.LimitedReader{R: catBuf, N: size} - if err := searchBlob(ctx, lr, keyToHoles, slot.idx, matches); err != nil { - fmt.Fprintf(os.Stderr, "warning: worker %d: skipping blob %s: %v\n", workerID, slot.blobHash, err) + fmt.Fprintf(os.Stderr, "warning: worker %d: creating git cat-file stdout pipe: %v\n", workerID, err) + return } - // drain any unread bytes (after a parse error or ctx-triggered early return) - if _, err := io.Copy(io.Discard, lr); err != nil { - fmt.Fprintf(os.Stderr, "warning: worker %d: draining blob %s: %v\n", workerID, slot.blobHash, err) + if err := cmd.Start(); err != nil { + fmt.Fprintf(os.Stderr, "warning: worker %d: starting git cat-file: %v\n", workerID, err) + return } - if _, err := catBuf.ReadByte(); err != nil { // trailing LF after blob content - fmt.Fprintf(os.Stderr, "warning: worker %d: reading trailing newline for blob %s: %v\n", workerID, slot.blobHash, err) + + catBuf := bufio.NewReader(catOut) + + loop: + for slot := range jobs { + if ctx.Err() != nil { + break loop // all initially-unfilled holes filled; stop requesting new blobs + } + + if _, err := fmt.Fprintf(catIn, "%s\n", slot.blobHash); err != nil { + fmt.Fprintf(os.Stderr, "warning: worker %d: writing to git cat-file (blob %s): %v\n", workerID, slot.blobHash, err) + continue + } + + // response header: " blob \n" or " missing\n" + line, err := catBuf.ReadString('\n') + if err != nil { + fmt.Fprintf(os.Stderr, "warning: worker %d: reading git cat-file header (blob %s): %v\n", workerID, slot.blobHash, err) + continue + } + parts := strings.Fields(line) + if len(parts) < 3 || parts[1] != "blob" { + fmt.Fprintf(os.Stderr, "warning: worker %d: unexpected git cat-file response %q for blob %s\n", workerID, strings.TrimRight(line, "\n"), slot.blobHash) + continue + } + size, err := strconv.ParseInt(parts[2], 10, 64) + if err != nil { + fmt.Fprintf(os.Stderr, "warning: worker %d: bad size %q for blob %s: %v\n", workerID, parts[2], slot.blobHash, err) + continue + } + + // LimitedReader prevents the decoder from reading into the next blob's header + lr := &io.LimitedReader{R: catBuf, N: size} + if err := searchBlob(ctx, lr, keyToHoles, slot.idx, matches); err != nil { + fmt.Fprintf(os.Stderr, "warning: worker %d: skipping blob %s: %v\n", workerID, slot.blobHash, err) + } + // drain any unread bytes (after a parse error or ctx-triggered early return) + if _, err := io.Copy(io.Discard, lr); err != nil { + fmt.Fprintf(os.Stderr, "warning: worker %d: draining blob %s: %v\n", workerID, slot.blobHash, err) + } + if _, err := catBuf.ReadByte(); err != nil { // trailing LF after blob content + fmt.Fprintf(os.Stderr, "warning: worker %d: reading trailing newline for blob %s: %v\n", workerID, slot.blobHash, err) + } } - } - go io.Copy(io.Discard, catOut) // let git flush and exit without blocking on a full pipe - catIn.Close() - cmd.Wait() - }(w) - } + go io.Copy(io.Discard, catOut) // let git flush and exit without blocking on a full pipe + catIn.Close() + cmd.Wait() + }(w) + } - wg.Wait() - close(matches) // signal coordinator that no more matches are coming - coordDone.Wait() // drain any in-flight matches before reading results + wg.Wait() + close(matches) // signal coordinator that no more matches are coming + coordDone.Wait() // drain any in-flight matches before reading results - unfilledCount := 0 - for _, s := range bestSlot { - if s == math.MaxInt { - unfilledCount++ + unfilledCount := 0 + for _, s := range bestSlot { + if s == math.MaxInt { + unfilledCount++ + } + } + if unfilledCount > 0 { + fmt.Fprintf(os.Stderr, "warning: %d hole(s) could not be filled from git history\n", unfilledCount) } - } - if unfilledCount > 0 { - fmt.Fprintf(os.Stderr, "warning: %d hole(s) could not be filled from git history\n", unfilledCount) } // --- Step 6: emit holes.json --- // holes.json: { tag -> { arch -> [ resolved OCI index, ... ] } } - // O(1) lookup by (tag, arch); the list normally has one entry; it has more when multiple OS-version variants (e.g. Windows) share the same (tag, arch) + // O(1) lookup by (tag, arch); the list normally has one entry; it has more when multiple OS-version variants (e.g. Windows) share the same (tag, arch). + // Unfillable holes (no resolved entry anywhere in history) are recorded as null rather than + // omitted: this lets the next run recognise them as "previously searched, still not found" + // and skip the git search for them, rather than scanning history again to reach the same result. output := map[string]map[string][]*ocispec.Index{} for i, h := range holes { - if results[i] == nil { - continue // no historical data found; library deploy will skip this hole - } if output[h.tag] == nil { output[h.tag] = map[string][]*ocispec.Index{} } - output[h.tag][h.arch] = append(output[h.tag][h.arch], results[i]) + if results[i] != nil { + output[h.tag][h.arch] = append(output[h.tag][h.arch], results[i]) + } else if _, exists := output[h.tag][h.arch]; !exists { + output[h.tag][h.arch] = nil // null sentinel: searched, not found + } } enc := json.NewEncoder(os.Stdout) @@ -364,7 +455,7 @@ func main() { } } -// searchBlob streams a single builds.json blob (from any historical commit), looking for resolved entries whose (tag, arch) pairs match any hole. Matches are sent to the matches channel tagged with slotIdx (lower = more recent); the coordinator goroutine owns bestSlot/results so no synchronisation is needed here. ctx is checked between entries: once all holes are globally filled the caller cancels ctx and this function returns early, letting the caller drain the remaining unread bytes. +// searchBlob streams a single builds.json blob (from any historical commit), looking for resolved entries whose (tag, arch) pairs match any hole. Matches are sent to the matches channel tagged with slotIdx (lower = more recent); the coordinator goroutine owns bestSlot/results so no synchronisation is needed here. ctx is checked between entries: once all initially-unfilled holes are globally filled the caller cancels ctx and this function returns early, letting the caller drain the remaining unread bytes. // // Uses encoding/json/v2 (jsontext + jsonv2.UnmarshalDecode) to stream the top-level object one entry at a time, avoiding loading the entire blob into memory at once. func searchBlob(ctx context.Context, r io.Reader, keyToHoles map[string][]int, slotIdx int, matches chan<- blobMatch) error { @@ -393,7 +484,7 @@ func searchBlob(ctx context.Context, r io.Reader, keyToHoles map[string][]int, s for dec.PeekKind() != '}' { if ctx.Err() != nil { - return nil // all holes globally filled; return early, caller drains remainder + return nil // all initially-unfilled holes globally filled; return early, caller drains remainder } // read the buildId key (discard it) From 0001015701399bb7dcb6444c7d2e6b01f5c13076 Mon Sep 17 00:00:00 2001 From: Tianon Gravi Date: Fri, 21 Aug 2026 13:10:43 -0700 Subject: [PATCH 4/4] Ditch noise --- go.mod | 19 ---------- go.sum | 114 --------------------------------------------------------- 2 files changed, 133 deletions(-) diff --git a/go.mod b/go.mod index ad324e5..b410df1 100644 --- a/go.mod +++ b/go.mod @@ -5,7 +5,6 @@ go 1.26 require ( cuelabs.dev/go/oci/ociregistry v0.0.0-20240214163758-5ebe80b0a9a6 github.com/docker-library/bashbrew v0.1.11 - github.com/go-git/go-git/v5 v5.5.1 github.com/go-json-experiment/json v0.0.0-20260623181947-01eb4420fa68 github.com/opencontainers/go-digest v1.0.0 github.com/opencontainers/image-spec v1.1.0 @@ -13,32 +12,14 @@ require ( ) require ( - github.com/Microsoft/go-winio v0.6.0 // indirect - github.com/ProtonMail/go-crypto v0.0.0-20221026131551-cf6655e29de4 // indirect - github.com/acomagu/bufpipe v1.0.3 // indirect - github.com/cloudflare/circl v1.3.1 // indirect github.com/containerd/containerd v1.6.19 // indirect - github.com/emirpasic/gods v1.18.1 // indirect - github.com/go-git/gcfg v1.5.0 // indirect - github.com/go-git/go-billy/v5 v5.3.1 // indirect github.com/golang/protobuf v1.5.2 // indirect - github.com/imdario/mergo v0.3.13 // indirect - github.com/jbenet/go-context v0.0.0-20150711004518-d14ea06fba99 // indirect - github.com/kevinburke/ssh_config v1.2.0 // indirect - github.com/pjbgf/sha1cd v0.2.3 // indirect - github.com/sergi/go-diff v1.2.0 // indirect github.com/sirupsen/logrus v1.9.0 // indirect - github.com/skeema/knownhosts v1.1.0 // indirect - github.com/xanzy/ssh-agent v0.3.3 // indirect - golang.org/x/crypto v0.14.0 // indirect - golang.org/x/mod v0.13.0 // indirect golang.org/x/net v0.16.0 // indirect golang.org/x/sys v0.13.0 // indirect - golang.org/x/tools v0.14.0 // indirect google.golang.org/genproto v0.0.0-20221207170731-23e4bf6bdc37 // indirect google.golang.org/grpc v1.51.0 // indirect google.golang.org/protobuf v1.28.1 // indirect - gopkg.in/warnings.v0 v0.1.2 // indirect ) // https://github.com/cue-labs/oci/pull/29 diff --git a/go.sum b/go.sum index bdc54cf..279c68c 100644 --- a/go.sum +++ b/go.sum @@ -1,38 +1,10 @@ -github.com/Microsoft/go-winio v0.5.2/go.mod h1:WpS1mjBmmwHBEWmogvA2mj8546UReBk4v8QkMxJ6pZY= -github.com/Microsoft/go-winio v0.6.0 h1:slsWYD/zyx7lCXoZVlvQrj0hPTM1HI4+v1sIda2yDvg= -github.com/Microsoft/go-winio v0.6.0/go.mod h1:cTAf44im0RAYeL23bpB+fzCyDH2MJiz2BO69KH/soAE= -github.com/ProtonMail/go-crypto v0.0.0-20221026131551-cf6655e29de4 h1:ra2OtmuW0AE5csawV4YXMNGNQQXvLRps3z2Z59OPO+I= -github.com/ProtonMail/go-crypto v0.0.0-20221026131551-cf6655e29de4/go.mod h1:UBYPn8k0D56RtnR8RFQMjmh4KrZzWJ5o7Z9SYjossQ8= -github.com/acomagu/bufpipe v1.0.3 h1:fxAGrHZTgQ9w5QqVItgzwj235/uYZYgbXitB+dLupOk= -github.com/acomagu/bufpipe v1.0.3/go.mod h1:mxdxdup/WdsKVreO5GpW4+M/1CE2sMG4jeGJ2sYmHc4= -github.com/anmitsu/go-shlex v0.0.0-20200514113438-38f4b401e2be h1:9AeTilPcZAjCFIImctFaOjnTIavg87rW78vTPkQqLI8= -github.com/anmitsu/go-shlex v0.0.0-20200514113438-38f4b401e2be/go.mod h1:ySMOLuWl6zY27l47sB3qLNK6tF2fkHG55UZxx8oIVo4= -github.com/armon/go-socks5 v0.0.0-20160902184237-e75332964ef5 h1:0CwZNZbxp69SHPdPJAN/hZIm0C4OItdklCFmMRWYpio= -github.com/armon/go-socks5 v0.0.0-20160902184237-e75332964ef5/go.mod h1:wHh0iHkYZB8zMSxRWpUBQtwG5a7fFgvEO+odwuTv2gs= -github.com/bwesterb/go-ristretto v1.2.0/go.mod h1:fUIoIZaG73pV5biE2Blr2xEzDoMj7NFEuV9ekS419A0= -github.com/cloudflare/circl v1.1.0/go.mod h1:prBCrKB9DV4poKZY1l9zBXg2QJY7mvgRvtMxxK7fi4I= -github.com/cloudflare/circl v1.3.1 h1:4OVCZRL62ijwEwxnF6I7hLwxvIYi3VaZt8TflkqtrtA= -github.com/cloudflare/circl v1.3.1/go.mod h1:+CauBF6R70Jqcyl8N2hC8pAXYbWkGIezuSbuGLtRhnw= github.com/containerd/containerd v1.6.19 h1:F0qgQPrG0P2JPgwpxWxYavrVeXAG0ezUIB9Z/4FTUAU= github.com/containerd/containerd v1.6.19/go.mod h1:HZCDMn4v/Xl2579/MvtOC2M206i+JJ6VxFWU/NetrGY= -github.com/creack/pty v1.1.9/go.mod h1:oKZEueFk5CKHvIhNR5MUki03XCEU+Q6VDXinZuGJ33E= github.com/davecgh/go-spew v1.1.0/go.mod h1:J7Y8YcW2NihsgmVo/mv3lAwl/skON4iLHjSsI+c5H38= github.com/davecgh/go-spew v1.1.1 h1:vj9j/u1bqnvCEfJOwUhtlOARqs3+rkHYY13jYWTU97c= github.com/davecgh/go-spew v1.1.1/go.mod h1:J7Y8YcW2NihsgmVo/mv3lAwl/skON4iLHjSsI+c5H38= github.com/docker-library/bashbrew v0.1.11 h1:9S6jYFu0+RaqEAfvS2lh7jcaDkcvFi2maB2aU3yb0TM= github.com/docker-library/bashbrew v0.1.11/go.mod h1:6fyRRSm4vgBAgTw87EsfOT7wXKsc4JA9I5cdQJmwOm8= -github.com/emirpasic/gods v1.18.1 h1:FXtiHYKDGKCW2KzwZKx0iC0PQmdlorYgdFG9jPXJ1Bc= -github.com/emirpasic/gods v1.18.1/go.mod h1:8tpGGwCnJ5H4r6BWwaV6OrWmMoPhUl5jm/FMNAnJvWQ= -github.com/gliderlabs/ssh v0.3.5 h1:OcaySEmAQJgyYcArR+gGGTHCyE7nvhEMTlYY+Dp8CpY= -github.com/gliderlabs/ssh v0.3.5/go.mod h1:8XB4KraRrX39qHhT6yxPsHedjA08I/uBVwj4xC+/+z4= -github.com/go-git/gcfg v1.5.0 h1:Q5ViNfGF8zFgyJWPqYwA7qGFoMTEiBmdlkcfRmpIMa4= -github.com/go-git/gcfg v1.5.0/go.mod h1:5m20vg6GwYabIxaOonVkTdrILxQMpEShl1xiMF4ua+E= -github.com/go-git/go-billy/v5 v5.3.1 h1:CPiOUAzKtMRvolEKw+bG1PLRpT7D3LIs3/3ey4Aiu34= -github.com/go-git/go-billy/v5 v5.3.1/go.mod h1:pmpqyWchKfYfrkb/UVH4otLvyi/5gJlGI4Hb3ZqZ3W0= -github.com/go-git/go-git-fixtures/v4 v4.3.1 h1:y5z6dd3qi8Hl+stezc8p3JxDkoTRqMAlKnXHuzrfjTQ= -github.com/go-git/go-git-fixtures/v4 v4.3.1/go.mod h1:8LHG1a3SRW71ettAD/jW13h8c6AqjVSeL11RAdgaqpo= -github.com/go-git/go-git/v5 v5.5.1 h1:5vtv2TB5PM/gPM+EvsHJ16hJh4uAkdGcKilcwY7FYwo= -github.com/go-git/go-git/v5 v5.5.1/go.mod h1:uz5PQ3d0gz7mSgzZhSJToM6ALPaKCdSnl58/Xb5hzr8= github.com/go-json-experiment/json v0.0.0-20260623181947-01eb4420fa68 h1:KZaTBSyshWX3MP5jukJcNSuXDQTO+rNpt0J564dX/eg= github.com/go-json-experiment/json v0.0.0-20260623181947-01eb4420fa68/go.mod h1:tphK2c80bpPhMOI4v6bIc2xWywPfbqi1Z06+RcrMkDg= github.com/go-quicktest/qt v1.100.0 h1:I7iSLgIwNp0E0UnSvKJzs7ig0jg/Iq83zsZjtQNW7jY= @@ -43,115 +15,38 @@ github.com/golang/protobuf v1.5.2/go.mod h1:XVQd3VNwM+JqD3oG2Ue2ip4fOMUkwXdXDdiu github.com/google/go-cmp v0.5.5/go.mod h1:v8dTdLbMG2kIc/vJvl+f65V22dbkXbowE6jgT/gNBxE= github.com/google/go-cmp v0.5.9 h1:O2Tfq5qg4qc4AmwVlvv0oLiVAGB7enBSJ2x2DqQFi38= github.com/google/go-cmp v0.5.9/go.mod h1:17dUlkBOakJ0+DkrSSNjCkIjxS6bF9zb3elmeNGIjoY= -github.com/imdario/mergo v0.3.13 h1:lFzP57bqS/wsqKssCGmtLAb8A0wKjLGrve2q3PPVcBk= -github.com/imdario/mergo v0.3.13/go.mod h1:4lJ1jqUDcsbIECGy0RUJAXNIhg+6ocWgb1ALK2O4oXg= -github.com/jbenet/go-context v0.0.0-20150711004518-d14ea06fba99 h1:BQSFePA1RWJOlocH6Fxy8MmwDt+yVQYULKfN0RoTN8A= -github.com/jbenet/go-context v0.0.0-20150711004518-d14ea06fba99/go.mod h1:1lJo3i6rXxKeerYnT8Nvf0QmHCRC1n8sfWVwXF2Frvo= -github.com/jessevdk/go-flags v1.5.0/go.mod h1:Fw0T6WPc1dYxT4mKEZRfG5kJhaTDP9pj1c2EWnYs/m4= -github.com/kevinburke/ssh_config v1.2.0 h1:x584FjTGwHzMwvHx18PXxbBVzfnxogHaAReU4gf13a4= -github.com/kevinburke/ssh_config v1.2.0/go.mod h1:CT57kijsi8u/K/BOFA39wgDQJ9CxiF4nAY/ojJ6r6mM= -github.com/kr/pretty v0.1.0/go.mod h1:dAy3ld7l9f0ibDNOQOHHMYYIIbhfbHSm3C4ZsoJORNo= -github.com/kr/pretty v0.2.1/go.mod h1:ipq/a2n7PKx3OHsz4KJII5eveXtPO4qwEXGdVfWzfnI= github.com/kr/pretty v0.3.1 h1:flRD4NNwYAUpkphVc1HcthR4KEIFJ65n8Mw5qdRn3LE= github.com/kr/pretty v0.3.1/go.mod h1:hoEshYVHaxMs3cyo3Yncou5ZscifuDolrwPKZanG3xk= -github.com/kr/pty v1.1.1/go.mod h1:pFQYn66WHrOpPYNljwOMqo10TkYh1fy3cYio2l3bCsQ= -github.com/kr/text v0.1.0/go.mod h1:4Jbv+DJW3UT/LiOwJeYQe1efqtUx/iVham/4vfdArNI= github.com/kr/text v0.2.0 h1:5Nx0Ya0ZqY2ygV366QzturHI13Jq95ApcVaJBhpS+AY= github.com/kr/text v0.2.0/go.mod h1:eLer722TekiGuMkidMxC/pM04lWEeraHUUmBw8l2grE= -github.com/matryer/is v1.2.0 h1:92UTHpy8CDwaJ08GqLDzhhuixiBUUD1p3AU6PHddz4A= -github.com/matryer/is v1.2.0/go.mod h1:2fLPjFQM9rhQ15aVEtbuwhJinnOqrmgXPNdZsdwlWXA= -github.com/niemeyer/pretty v0.0.0-20200227124842-a10e7caefd8e/go.mod h1:zD1mROLANZcx1PVRCS0qkT7pwLkGfwJo4zjcN/Tysno= github.com/opencontainers/go-digest v1.0.0 h1:apOUWs51W5PlhuyGyz9FCeeBIOUDA/6nW8Oi/yOhh5U= github.com/opencontainers/go-digest v1.0.0/go.mod h1:0JzlMkj0TRzQZfJkVvzbP0HBR3IKzErnv2BNG4W4MAM= github.com/opencontainers/image-spec v1.1.0 h1:8SG7/vwALn54lVB/0yZ/MMwhFrPYtpEHQb2IpWsCzug= github.com/opencontainers/image-spec v1.1.0/go.mod h1:W4s4sFTMaBeK1BQLXbG4AdM2szdn85PY75RI83NrTrM= -github.com/pjbgf/sha1cd v0.2.3 h1:uKQP/7QOzNtKYH7UTohZLcjF5/55EnTw0jO/Ru4jZwI= -github.com/pjbgf/sha1cd v0.2.3/go.mod h1:HOK9QrgzdHpbc2Kzip0Q1yi3M2MFGPADtR6HjG65m5M= github.com/pkg/errors v0.9.1 h1:FEBLx1zS214owpjy7qsBeixbURkuhQAwrK5UwLGTwt4= github.com/pkg/errors v0.9.1/go.mod h1:bwawxfHBFNV+L2hUp1rHADufV3IMtnDRdf1r5NINEl0= github.com/pmezard/go-difflib v1.0.0 h1:4DBwDE0NGyQoBHbLQYPwSUPoCMWR5BEzIk/f1lZbAQM= github.com/pmezard/go-difflib v1.0.0/go.mod h1:iKH77koFhYxTK1pcRnkKkqfTogsbg7gZNVY4sRDYZ/4= github.com/rogpeppe/go-internal v1.10.1-0.20230524175051-ec119421bb97 h1:3RPlVWzZ/PDqmVuf/FKHARG5EMid/tl7cv54Sw/QRVY= github.com/rogpeppe/go-internal v1.10.1-0.20230524175051-ec119421bb97/go.mod h1:ddIwULY96R17DhadqLgMfk9H9tvdUzkipdSkR5nkCZA= -github.com/sergi/go-diff v1.1.0/go.mod h1:STckp+ISIX8hZLjrqAeVduY0gWCT9IjLuqbuNXdaHfM= -github.com/sergi/go-diff v1.2.0 h1:XU+rvMAioB0UC3q1MFrIQy4Vo5/4VsRDQQXHsEya6xQ= -github.com/sergi/go-diff v1.2.0/go.mod h1:STckp+ISIX8hZLjrqAeVduY0gWCT9IjLuqbuNXdaHfM= -github.com/sirupsen/logrus v1.7.0/go.mod h1:yWOB1SBYBC5VeMP7gHvWumXLIWorT60ONWic61uBYv0= github.com/sirupsen/logrus v1.9.0 h1:trlNQbNUG3OdDrDil03MCb1H2o9nJ1x4/5LYw7byDE0= github.com/sirupsen/logrus v1.9.0/go.mod h1:naHLuLoDiP4jHNo9R0sCBMtWGeIprob74mVsIT4qYEQ= -github.com/skeema/knownhosts v1.1.0 h1:Wvr9V0MxhjRbl3f9nMnKnFfiWTJmtECJ9Njkea3ysW0= -github.com/skeema/knownhosts v1.1.0/go.mod h1:sKFq3RD6/TKZkSWn8boUbDC7Qkgcv+8XXijpFO6roag= github.com/stretchr/objx v0.1.0/go.mod h1:HFkY916IF+rwdDfMAkV7OtwuqBVzrE8GR6GFx+wExME= -github.com/stretchr/testify v1.2.2/go.mod h1:a8OnRcib4nhh0OaRAV+Yts87kKdq0PP7pXfy6kDkUVs= -github.com/stretchr/testify v1.4.0/go.mod h1:j7eGeouHqKxXV5pUuKE4zz7dFj8WfuZ+81PSLYec5m4= github.com/stretchr/testify v1.7.0 h1:nwc3DEeHmmLAfoZucVR881uASk0Mfjw8xYJ99tb5CcY= github.com/stretchr/testify v1.7.0/go.mod h1:6Fq8oRcR53rry900zMqJjRRixrwX3KX962/h/Wwjteg= github.com/tianon/cuelabs-oci/ociregistry v0.0.0-20240329232705-b652d611e4b3 h1:kfAfFbiZ+2ZErqgqKtaMge1qeeE/0rnxuTl21G7fSwk= github.com/tianon/cuelabs-oci/ociregistry v0.0.0-20240329232705-b652d611e4b3/go.mod h1:pK23AUVXuNzzTpfMCA06sxZGeVQ/75FdVtW249de9Uo= -github.com/xanzy/ssh-agent v0.3.3 h1:+/15pJfg/RsTxqYcX6fHqOXZwwMP+2VyYWJeWM2qQFM= -github.com/xanzy/ssh-agent v0.3.3/go.mod h1:6dzNDKs0J9rVPHPhaGCukekBHKqfl+L3KghI1Bc68Uw= -github.com/yuin/goldmark v1.4.13/go.mod h1:6yULJ656Px+3vBD8DxQVa3kxgyrAnzto9xy5taEt/CY= -golang.org/x/crypto v0.0.0-20190308221718-c2843e01d9a2/go.mod h1:djNgcEr1/C05ACkg1iLfiJU5Ep61QUkGW8qpdssI0+w= -golang.org/x/crypto v0.0.0-20210921155107-089bfa567519/go.mod h1:GvvjBRRGRdwPK5ydBHafDWAxML/pGHZbMvKqRZ5+Abc= -golang.org/x/crypto v0.0.0-20220525230936-793ad666bf5e/go.mod h1:IxCIyHEi3zRg3s0A5j5BB6A9Jmi73HwBIUl50j+osU4= -golang.org/x/crypto v0.0.0-20220622213112-05595931fe9d/go.mod h1:IxCIyHEi3zRg3s0A5j5BB6A9Jmi73HwBIUl50j+osU4= -golang.org/x/crypto v0.0.0-20220826181053-bd7e27e6170d/go.mod h1:IxCIyHEi3zRg3s0A5j5BB6A9Jmi73HwBIUl50j+osU4= -golang.org/x/crypto v0.3.0/go.mod h1:hebNnKkNXi2UzZN1eVRvBB7co0a+JxK6XbPiWVs/3J4= -golang.org/x/crypto v0.14.0 h1:wBqGXzWJW6m1XrIKlAH0Hs1JJ7+9KBwnIO8v66Q9cHc= -golang.org/x/crypto v0.14.0/go.mod h1:MVFd36DqK4CsrnJYDkBA3VC4m2GkXAM0PvzMCn4JQf4= -golang.org/x/mod v0.6.0-dev.0.20220419223038-86c51ed26bb4/go.mod h1:jJ57K6gSWd91VN4djpZkiMVwK6gcyfeH4XE8wZrZaV4= -golang.org/x/mod v0.13.0 h1:I/DsJXRlw/8l/0c24sM9yb0T4z9liZTduXvdAWYiysY= -golang.org/x/mod v0.13.0/go.mod h1:hTbmBsO62+eylJbnUtE2MGJUyE7QWk4xUqPFrRgJ+7c= -golang.org/x/net v0.0.0-20190620200207-3b0461eec859/go.mod h1:z5CRVTTTmAJ677TzLLGU+0bjPO0LkuOLi4/5GtJWs/s= -golang.org/x/net v0.0.0-20210226172049-e18ecbb05110/go.mod h1:m0MpNAwzfU5UDzcl9v0D8zg8gWTRqZa9RBIspLL5mdg= -golang.org/x/net v0.0.0-20211112202133-69e39bad7dc2/go.mod h1:9nx3DQGgdP8bBQD5qxJ1jj9UTztislL4KSBs9R2vV5Y= -golang.org/x/net v0.0.0-20220722155237-a158d28d115b/go.mod h1:XRhObCWvk6IyKnWLug+ECip1KBveYUHfp+8e9klMJ9c= -golang.org/x/net v0.0.0-20220826154423-83b083e8dc8b/go.mod h1:YDH+HFinaLZZlnHAfSS6ZXJJ9M9t4Dl22yv3iI2vPwk= -golang.org/x/net v0.2.0/go.mod h1:KqCZLdyyvdV855qA2rE3GC2aiw5xGR5TEjj8smXukLY= golang.org/x/net v0.16.0 h1:7eBu7KsSvFDtSXUIDbh3aqlK4DPsZ1rByC8PFfBThos= golang.org/x/net v0.16.0/go.mod h1:NxSsAGuq816PNPmqtQdLE42eU2Fs7NoRIZrHJAlaCOE= -golang.org/x/sync v0.0.0-20190423024810-112230192c58/go.mod h1:RxMgew5VJxzue5/jJTE5uejpjVlOe/izrB70Jof72aM= -golang.org/x/sync v0.0.0-20220722155255-886fb9371eb4/go.mod h1:RxMgew5VJxzue5/jJTE5uejpjVlOe/izrB70Jof72aM= -golang.org/x/sync v0.4.0 h1:zxkM55ReGkDlKSM+Fu41A+zmbZuaPVbGMzvvdUPznYQ= -golang.org/x/sync v0.4.0/go.mod h1:FU7BRWz2tNW+3quACPkgCx/L+uEAv1htQ0V83Z9Rj+Y= -golang.org/x/sys v0.0.0-20190215142949-d0b11bdaac8a/go.mod h1:STP8DvDyc/dI5b8T5hshtkjS+E42TnysNCUPdjciGhY= -golang.org/x/sys v0.0.0-20191026070338-33540a1f6037/go.mod h1:h1NjWce9XRLGQEsW7wpKNCjG9DtNlClVuFLEZdDNbEs= -golang.org/x/sys v0.0.0-20200302150141-5c8b2ff67527/go.mod h1:h1NjWce9XRLGQEsW7wpKNCjG9DtNlClVuFLEZdDNbEs= -golang.org/x/sys v0.0.0-20201119102817-f84b799fce68/go.mod h1:h1NjWce9XRLGQEsW7wpKNCjG9DtNlClVuFLEZdDNbEs= -golang.org/x/sys v0.0.0-20210124154548-22da62e12c0c/go.mod h1:h1NjWce9XRLGQEsW7wpKNCjG9DtNlClVuFLEZdDNbEs= -golang.org/x/sys v0.0.0-20210320140829-1e4c9ba3b0c4/go.mod h1:h1NjWce9XRLGQEsW7wpKNCjG9DtNlClVuFLEZdDNbEs= -golang.org/x/sys v0.0.0-20210423082822-04245dca01da/go.mod h1:h1NjWce9XRLGQEsW7wpKNCjG9DtNlClVuFLEZdDNbEs= -golang.org/x/sys v0.0.0-20210615035016-665e8c7367d1/go.mod h1:oPkhp1MJrh7nUepCBck5+mAzfO9JrbApNNgaTdGDITg= -golang.org/x/sys v0.0.0-20211007075335-d3039528d8ac/go.mod h1:oPkhp1MJrh7nUepCBck5+mAzfO9JrbApNNgaTdGDITg= -golang.org/x/sys v0.0.0-20220520151302-bc2c85ada10a/go.mod h1:oPkhp1MJrh7nUepCBck5+mAzfO9JrbApNNgaTdGDITg= golang.org/x/sys v0.0.0-20220715151400-c0bba94af5f8/go.mod h1:oPkhp1MJrh7nUepCBck5+mAzfO9JrbApNNgaTdGDITg= -golang.org/x/sys v0.0.0-20220722155257-8c9f86f7a55f/go.mod h1:oPkhp1MJrh7nUepCBck5+mAzfO9JrbApNNgaTdGDITg= -golang.org/x/sys v0.0.0-20220728004956-3c1f35247d10/go.mod h1:oPkhp1MJrh7nUepCBck5+mAzfO9JrbApNNgaTdGDITg= -golang.org/x/sys v0.0.0-20220825204002-c680a09ffe64/go.mod h1:oPkhp1MJrh7nUepCBck5+mAzfO9JrbApNNgaTdGDITg= -golang.org/x/sys v0.2.0/go.mod h1:oPkhp1MJrh7nUepCBck5+mAzfO9JrbApNNgaTdGDITg= golang.org/x/sys v0.13.0 h1:Af8nKPmuFypiUBjVoU9V20FiaFXOcuZI21p0ycVYYGE= golang.org/x/sys v0.13.0/go.mod h1:oPkhp1MJrh7nUepCBck5+mAzfO9JrbApNNgaTdGDITg= -golang.org/x/term v0.0.0-20201126162022-7de9c90e9dd1/go.mod h1:bj7SfCRtBDWHUb9snDiAeCFNEtKQo2Wmx5Cou7ajbmo= -golang.org/x/term v0.0.0-20210927222741-03fcf44c2211/go.mod h1:jbD1KX2456YbFQfuXm/mYQcufACuNUgVhRMnK/tPxf8= -golang.org/x/term v0.0.0-20220722155259-a9ba230a4035/go.mod h1:jbD1KX2456YbFQfuXm/mYQcufACuNUgVhRMnK/tPxf8= -golang.org/x/term v0.2.0/go.mod h1:TVmDHMZPmdnySmBfhjOoOdhjzdE1h4u1VwSiw2l1Nuc= -golang.org/x/term v0.13.0 h1:bb+I9cTfFazGW51MZqBVmZy7+JEJMouUHTUSKVQLBek= -golang.org/x/term v0.13.0/go.mod h1:LTmsnFJwVN6bCy1rVCoS+qHT1HhALEFxKncY3WNNh4U= -golang.org/x/text v0.3.0/go.mod h1:NqM8EUOU14njkJ3fqMW+pc6Ldnwhi/IjpwHt7yyuwOQ= -golang.org/x/text v0.3.3/go.mod h1:5Zoc/QRtKVWzQhOtBMvqHzDpF6irO9z98xDceosuGiQ= -golang.org/x/text v0.3.6/go.mod h1:5Zoc/QRtKVWzQhOtBMvqHzDpF6irO9z98xDceosuGiQ= -golang.org/x/text v0.3.7/go.mod h1:u+2+/6zg+i71rQMx5EYifcz6MCKuco9NR6JIITiCfzQ= -golang.org/x/text v0.4.0/go.mod h1:mrYo+phRRbMaCq/xk9113O4dZlRixOauAjOtrjsXDZ8= golang.org/x/text v0.13.0 h1:ablQoSUd0tRdKxZewP80B+BaqeKJuVhuRxj/dkrun3k= golang.org/x/text v0.13.0/go.mod h1:TvPlkZtksWOMsz7fbANvkp4WM8x/WCo/om8BMLbz+aE= golang.org/x/time v0.5.0 h1:o7cqy6amK/52YcAKIPlM3a+Fpj35zvRj2TP+e1xFSfk= golang.org/x/time v0.5.0/go.mod h1:3BpzKBy/shNhVucY/MWOyx10tF3SFh9QdLuxbVysPQM= -golang.org/x/tools v0.0.0-20180917221912-90fa682c2a6e/go.mod h1:n7NCudcB/nEzxVGmLbDWY5pfWTLqBcC2KZ6jyYvM4mQ= -golang.org/x/tools v0.0.0-20191119224855-298f0cb1881e/go.mod h1:b+2E5dAYhXwXZwtnZ6UAqBI28+e2cm9otk0dWdXHAEo= -golang.org/x/tools v0.1.12/go.mod h1:hNGJHUnrk76NpqgfD5Aqm5Crs+Hm0VOH/i9J2+nxYbc= golang.org/x/tools v0.14.0 h1:jvNa2pY0M4r62jkRQ6RwEZZyPcymeL9XZMLBbV7U2nc= golang.org/x/tools v0.14.0/go.mod h1:uYBEerGOWcJyEORxN+Ek8+TT266gXkNlHdJBwexUsBg= -golang.org/x/xerrors v0.0.0-20190717185122-a985d3407aa7/go.mod h1:I/5z698sn9Ka8TeJc9MKroUUfqBBauWjQqLJ2OPfmY0= golang.org/x/xerrors v0.0.0-20191204190536-9bdfabe68543/go.mod h1:I/5z698sn9Ka8TeJc9MKroUUfqBBauWjQqLJ2OPfmY0= google.golang.org/genproto v0.0.0-20221207170731-23e4bf6bdc37 h1:jmIfw8+gSvXcZSgaFAGyInDXeWzUhvYH57G/5GKMn70= google.golang.org/genproto v0.0.0-20221207170731-23e4bf6bdc37/go.mod h1:RGgjbofJ8xD9Sq1VVhDM1Vok1vRONV+rg+CjzG4SZKM= @@ -162,16 +57,7 @@ google.golang.org/protobuf v1.26.0/go.mod h1:9q0QmTI4eRPtz6boOQmLYwt+qCgq0jsYwAQ google.golang.org/protobuf v1.28.1 h1:d0NfwRgPtno5B1Wa6L2DAG+KivqkdutMf1UhdNx175w= google.golang.org/protobuf v1.28.1/go.mod h1:HV8QOd/L58Z+nl8r43ehVNZIU/HEI6OcFqwMG9pJV4I= gopkg.in/check.v1 v0.0.0-20161208181325-20d25e280405/go.mod h1:Co6ibVJAznAaIkqp8huTwlJQCZ016jof/cbN4VW5Yz0= -gopkg.in/check.v1 v1.0.0-20190902080502-41f04d3bba15/go.mod h1:Co6ibVJAznAaIkqp8huTwlJQCZ016jof/cbN4VW5Yz0= -gopkg.in/check.v1 v1.0.0-20200227125254-8fa46927fb4f/go.mod h1:Co6ibVJAznAaIkqp8huTwlJQCZ016jof/cbN4VW5Yz0= -gopkg.in/check.v1 v1.0.0-20201130134442-10cb98267c6c h1:Hei/4ADfdWqJk1ZMxUNpqntNwaWcugrBjAiHlqqRiVk= -gopkg.in/check.v1 v1.0.0-20201130134442-10cb98267c6c/go.mod h1:JHkPIbrfpd72SG/EVd6muEfDQjcINNoR0C8j2r3qZ4Q= -gopkg.in/warnings.v0 v0.1.2 h1:wFXVbFY8DY5/xOe1ECiWdKCzZlxgshcYVNkBHstARME= -gopkg.in/warnings.v0 v0.1.2/go.mod h1:jksf8JmL6Qr/oQM2OXTHunEvvTAsrWBLb6OOjuVWRNI= -gopkg.in/yaml.v2 v2.2.2/go.mod h1:hI93XBmqTisBFMUTm0b8Fm+jr3Dg1NNxqwp+5A1VGuI= -gopkg.in/yaml.v2 v2.2.4/go.mod h1:hI93XBmqTisBFMUTm0b8Fm+jr3Dg1NNxqwp+5A1VGuI= gopkg.in/yaml.v3 v3.0.0-20200313102051-9f266ea9e77c/go.mod h1:K4uyk7z7BCEPqu6E+C64Yfv1cQ7kz7rIZviUmN+EgEM= -gopkg.in/yaml.v3 v3.0.0/go.mod h1:K4uyk7z7BCEPqu6E+C64Yfv1cQ7kz7rIZviUmN+EgEM= gopkg.in/yaml.v3 v3.0.1 h1:fxVm/GzAzEWqLHuvctI91KS9hhNmmWOoWu0XTYJS7CA= gopkg.in/yaml.v3 v3.0.1/go.mod h1:K4uyk7z7BCEPqu6E+C64Yfv1cQ7kz7rIZviUmN+EgEM= gotest.tools/v3 v3.0.3 h1:4AuOwCGf4lLR9u3YOe2awrHygurzhO/HeQ6laiA6Sx0=