From b8ceb3d7be37926ba4f90218d5c44aad0c45f16a Mon Sep 17 00:00:00 2001 From: Eugene Kalinin Date: Sat, 8 Aug 2026 22:22:04 +0300 Subject: [PATCH] feat(push): support --all-tags to push all tags `nerdctl push` accepts a bare repository name, but referenceutil.Parse normalizes it to ":latest", so only that single tag is pushed. Add the Docker-compatible `-a, --all-tags` flag, which pushes every local tag of the repository instead. Push is split into a dispatcher and pushSingle(): without --all-tags the dispatcher just delegates, with it the local tags are resolved through the `name~=^:` image filter (the same idiom nameFilterFor() uses for `nerdctl image ls`) and pushed one by one. The temporary images push creates for itself are skipped, so an interrupted push cannot leak a "-tmp-reduced-platform" tag into the registry, and the list is sorted because ImageService().List() guarantees no order. A tag or a digest in the reference is rejected, as docker does. The check looks at ExplicitTag rather than Tag: Parse() runs TagNameOnly(), so Tag is "latest" even for a bare repository name. A SOCI index is attached to the image manifest rather than to the tag, so it is now built once per distinct target digest. Pushing several tags of one image no longer makes each tag overwrite the index pushed by the previous one. Pushing more than once per process also uncovered a bug in the plain HTTP fallback. pushImageWithLocal builds a fresh in-memory tracker per push, but the fallback rebuilt the resolver through dockerconfigresolver.New, which silently substitutes the process-wide PushTracker. containerd's dockerPusher keys that tracker by content ref ("index-"), not by reference, and returns ErrAlreadyExists before issuing any request when the digest is already committed; remotes.push() treats that as success, so the manifest PUT that creates the tag never happens and the command still exits 0. Rebuild the resolver from the host options instead, reusing the resolver options assembled above so the fallback keeps the per-push tracker. The tests assert that the pushed tags are present in the registry rather than that they are the only ones: the listing is a superset, since a SOCI v1 index is attached through the referrers fallback tag ("sha256-") on registries without the referrers API. Closes #3751 Signed-off-by: Eugene Kalinin --- cmd/nerdctl/image/image_push.go | 7 ++ cmd/nerdctl/image/image_push_linux_test.go | 119 +++++++++++++++++++++ docs/command-reference.md | 3 +- pkg/api/types/image_types.go | 2 + pkg/cmd/image/push.go | 84 ++++++++++++++- 5 files changed, 209 insertions(+), 6 deletions(-) diff --git a/cmd/nerdctl/image/image_push.go b/cmd/nerdctl/image/image_push.go index 47104a4b7e5..f0535d9bce9 100644 --- a/cmd/nerdctl/image/image_push.go +++ b/cmd/nerdctl/image/image_push.go @@ -47,6 +47,8 @@ func PushCommand() *cobra.Command { cmd.Flags().Bool("all-platforms", false, "Push content for all platforms") // #endregion + cmd.Flags().BoolP("all-tags", "a", false, "Push all tags of an image to the repository") + cmd.Flags().Bool("estargz", false, "Convert the image into eStargz") cmd.Flags().Bool("ipfs-ensure-image", true, "Ensure the entire contents of the image is locally available before push") cmd.Flags().String("ipfs-address", "", "multiaddr of IPFS API (default uses $IPFS_PATH env variable if defined or local directory ~/.ipfs)") @@ -85,6 +87,10 @@ func pushOptions(cmd *cobra.Command) (types.ImagePushOptions, error) { if err != nil { return types.ImagePushOptions{}, err } + allTags, err := cmd.Flags().GetBool("all-tags") + if err != nil { + return types.ImagePushOptions{}, err + } estargz, err := cmd.Flags().GetBool("estargz") if err != nil { return types.ImagePushOptions{}, err @@ -119,6 +125,7 @@ func pushOptions(cmd *cobra.Command) (types.ImagePushOptions, error) { SociOptions: sociOptions, Platforms: platform, AllPlatforms: allPlatforms, + AllTags: allTags, Estargz: estargz, IpfsEnsureImage: ipfsEnsureImage, IpfsAddress: ipfsAddress, diff --git a/cmd/nerdctl/image/image_push_linux_test.go b/cmd/nerdctl/image/image_push_linux_test.go index c547341d012..f585c75c952 100644 --- a/cmd/nerdctl/image/image_push_linux_test.go +++ b/cmd/nerdctl/image/image_push_linux_test.go @@ -17,9 +17,11 @@ package image import ( + "encoding/json" "errors" "fmt" "net/http" + "slices" "testing" "gotest.tools/v3/assert" @@ -278,7 +280,124 @@ func TestPush(t *testing.T) { }, Expected: test.Expects(0, nil, nil), }, + { + Description: "all tags", + Require: require.Not(nerdtest.Docker), + Setup: func(data test.Data, helpers test.Helpers) { + helpers.Ensure("pull", "--quiet", testutil.CommonImage) + testImageRepo := fmt.Sprintf("%s:%d/%s", + registryNoAuthHTTPRandom.IP.String(), registryNoAuthHTTPRandom.Port, data.Identifier()) + data.Labels().Set("testImageRepo", testImageRepo) + helpers.Ensure("tag", testutil.CommonImage, testImageRepo+":v1") + helpers.Ensure("tag", testutil.CommonImage, testImageRepo+":v2") + }, + Cleanup: func(data test.Data, helpers test.Helpers) { + if data.Labels().Get("testImageRepo") != "" { + helpers.Anyhow("rmi", "-f", data.Labels().Get("testImageRepo")+":v1") + helpers.Anyhow("rmi", "-f", data.Labels().Get("testImageRepo")+":v2") + } + }, + Command: func(data test.Data, helpers test.Helpers) test.TestableCommand { + return helpers.Command("push", "--insecure-registry", "--all-tags", data.Labels().Get("testImageRepo")) + }, + Expected: func(data test.Data, helpers test.Helpers) *test.Expected { + return &test.Expected{ + Output: func(stdout string, t tig.T) { + assertRegistryHasTags(t, registryNoAuthHTTPRandom, data.Identifier(), "v1", "v2") + }, + } + }, + }, + { + Description: "all tags, with a tag", + Require: require.Not(nerdtest.Docker), + Setup: func(data test.Data, helpers test.Helpers) { + helpers.Ensure("pull", "--quiet", testutil.CommonImage) + testImageRef := fmt.Sprintf("%s:%d/%s:v1", + registryNoAuthHTTPRandom.IP.String(), registryNoAuthHTTPRandom.Port, data.Identifier()) + data.Labels().Set("testImageRef", testImageRef) + helpers.Ensure("tag", testutil.CommonImage, testImageRef) + }, + Cleanup: func(data test.Data, helpers test.Helpers) { + if data.Labels().Get("testImageRef") != "" { + helpers.Anyhow("rmi", "-f", data.Labels().Get("testImageRef")) + } + }, + Command: func(data test.Data, helpers test.Helpers) test.TestableCommand { + return helpers.Command("push", "--insecure-registry", "--all-tags", data.Labels().Get("testImageRef")) + }, + Expected: test.Expects(1, []error{errors.New("tag can't be used with --all-tags/-a")}, nil), + }, + { + Description: "all tags, no local tag", + Require: require.Not(nerdtest.Docker), + Command: func(data test.Data, helpers test.Helpers) test.TestableCommand { + testImageRepo := fmt.Sprintf("%s:%d/%s", + registryNoAuthHTTPRandom.IP.String(), registryNoAuthHTTPRandom.Port, data.Identifier()) + return helpers.Command("push", "--insecure-registry", "--all-tags", testImageRepo) + }, + Expected: test.Expects(1, []error{errors.New("an image does not exist locally with the tag")}, nil), + }, + { + Description: "all tags, soci", + Require: require.All( + nerdtest.Soci, + require.Not(nerdtest.Docker), + ), + Setup: func(data test.Data, helpers test.Helpers) { + helpers.Ensure("pull", "--quiet", testutil.UbuntuImage) + testImageRepo := fmt.Sprintf("%s:%d/%s", + registryNoAuthHTTPRandom.IP.String(), registryNoAuthHTTPRandom.Port, data.Identifier()) + data.Labels().Set("testImageRepo", testImageRepo) + helpers.Ensure("tag", testutil.UbuntuImage, testImageRepo+":v1") + helpers.Ensure("tag", testutil.UbuntuImage, testImageRepo+":v2") + }, + Cleanup: func(data test.Data, helpers test.Helpers) { + if data.Labels().Get("testImageRepo") != "" { + helpers.Anyhow("rmi", "-f", data.Labels().Get("testImageRepo")+":v1") + helpers.Anyhow("rmi", "-f", data.Labels().Get("testImageRepo")+":v2") + } + }, + Command: func(data test.Data, helpers test.Helpers) test.TestableCommand { + return helpers.Command("push", "--snapshotter=soci", "--insecure-registry", "--all-tags", "--soci-span-size=2097152", "--soci-min-layer-size=20971520", data.Labels().Get("testImageRepo")) + }, + Expected: func(data test.Data, helpers test.Helpers) *test.Expected { + return &test.Expected{ + Output: func(stdout string, t tig.T) { + assertRegistryHasTags(t, registryNoAuthHTTPRandom, data.Identifier(), "v1", "v2") + }, + } + }, + }, }, } testCase.Run(t) } + +// assertRegistryHasTags verifies the registry lists every tag of `want` for the repository `repo`. +// The listing legitimately holds more than the pushed tags: a SOCI v1 index is attached through the +// referrers fallback tag ("sha256-") on registries without the referrers API, and a re-run +// of the test hits a repository the previous run already populated. +func assertRegistryHasTags(t tig.T, reg *registry.Server, repo string, want ...string) { + t.Helper() + + tagsURL := fmt.Sprintf("http://%s:%d/v2/%s/tags/list", reg.IP.String(), reg.Port, repo) + resp, err := http.Get(tagsURL) + assert.NilError(t, err, "error making http request") + defer func() { + if resp.Body != nil { + _ = resp.Body.Close() + } + }() + assert.Equal(t, resp.StatusCode, http.StatusOK, "tag list should be available") + + var tagList struct { + Name string `json:"name"` + Tags []string `json:"tags"` + } + assert.NilError(t, json.NewDecoder(resp.Body).Decode(&tagList), "error decoding the tag list") + + for _, tag := range want { + assert.Assert(t, slices.Contains(tagList.Tags, tag), "expected tag %q in %v", tag, tagList.Tags) + } +} diff --git a/docs/command-reference.md b/docs/command-reference.md index 20239a0f3b0..db3c961ef51 100644 --- a/docs/command-reference.md +++ b/docs/command-reference.md @@ -899,6 +899,7 @@ Flags: - :nerd_face: `--platform=(amd64|arm64|...)`: Push content for a specific platform - :nerd_face: `--all-platforms`: Push content for all platforms +- :whale: `-a, --all-tags`: Push all tags of an image to the repository. `NAME` must not contain a tag. - :nerd_face: `--sign`: Sign the image (none|cosign|notation). See [`./cosign.md`](./cosign.md) and [`./notation.md`](./notation.md) for details. - :nerd_face: `--cosign-key`: Path to the private key file, KMS, URI or Kubernetes Secret for `--sign=cosign` - :nerd_face: `--notation-key-name`: Signing key name for a key previously added to notation's key list for `--sign=notation` @@ -908,7 +909,7 @@ Flags: - :nerd_face: `--soci-span-size`: Span size in bytes that soci index uses to segment layer data. Default is 4 MiB. - :nerd_face: `--soci-min-layer-size`: Minimum layer size in bytes to build zTOC for. Smaller layers won't have zTOC and not lazy pulled. Default is 10 MiB. -Unimplemented `docker push` flags: `--all-tags`, `--disable-content-trust` (default true) +Unimplemented `docker push` flags: `--disable-content-trust` (default true) ### :whale: nerdctl load diff --git a/pkg/api/types/image_types.go b/pkg/api/types/image_types.go index 4bea77dcf8b..503f2a3c776 100644 --- a/pkg/api/types/image_types.go +++ b/pkg/api/types/image_types.go @@ -205,6 +205,8 @@ type ImagePushOptions struct { Platforms []string // AllPlatforms convert content for all platforms AllPlatforms bool + // AllTags push all the tags of the repository named by the reference + AllTags bool // Estargz convert image to sStargz Estargz bool diff --git a/pkg/cmd/image/push.go b/pkg/cmd/image/push.go index 505e8eb7129..fdc55e7ab23 100644 --- a/pkg/cmd/image/push.go +++ b/pkg/cmd/image/push.go @@ -24,6 +24,9 @@ import ( "net/http" "os" "path/filepath" + "regexp" + "slices" + "strings" "github.com/opencontainers/go-digest" ocispec "github.com/opencontainers/image-spec/specs-go/v1" @@ -57,8 +60,74 @@ import ( "github.com/containerd/nerdctl/v2/pkg/snapshotterutil" ) +const ( + // Suffixes of the temporary images push creates for itself before uploading them. + tmpReducedPlatformSuffix = "-tmp-reduced-platform" + tmpEsgzSuffix = "-tmp-esgz" +) + // Push pushes an image specified by `rawRef`. +// With options.AllTags, `rawRef` must be a bare repository name, and every local tag of that +// repository is pushed. func Push(ctx context.Context, client *containerd.Client, rawRef string, options types.ImagePushOptions) error { + if !options.AllTags { + return pushSingle(ctx, client, rawRef, options, false) + } + + parsedReference, err := referenceutil.Parse(rawRef) + if err != nil { + return err + } + // ExplicitTag, not Tag: Parse normalizes a bare repository name to ":latest". + if parsedReference.ExplicitTag != "" || parsedReference.Digest != "" { + return errors.New("tag can't be used with --all-tags/-a") + } + if parsedReference.Protocol != "" { + return fmt.Errorf("--all-tags is not supported for %q references", parsedReference.Protocol) + } + + imgs, err := localTags(ctx, client, parsedReference.Name()) + if err != nil { + return err + } + if len(imgs) == 0 { + return fmt.Errorf("an image does not exist locally with the tag: %s", parsedReference.Name()) + } + + // A SOCI index is attached to the image manifest rather than to the tag, so it only needs to be + // built once per distinct target. Doing it per tag makes every tag overwrite the index pushed by + // the previous one: https://github.com/containerd/nerdctl/issues/3751 + indexed := make(map[digest.Digest]struct{}, len(imgs)) + for _, img := range imgs { + _, done := indexed[img.Target.Digest] + if err = pushSingle(ctx, client, img.Name, options, done); err != nil { + return err + } + indexed[img.Target.Digest] = struct{}{} + } + return nil +} + +// localTags returns the local images tagged under the repository `name`, sorted by name. +func localTags(ctx context.Context, client *containerd.Client, name string) ([]images.Image, error) { + // Same idiom as nameFilterFor() in cmd/nerdctl/image/image_list.go: a bare repository name + // matches every tag of that repository (pkg/ cannot import cmd/, hence the repeated filter). + imgs, err := client.ImageService().List(ctx, fmt.Sprintf("name~=^%s:", regexp.QuoteMeta(name))) + if err != nil { + return nil, err + } + // Drop the temporary images push creates for itself, which an interrupted push may have left behind. + imgs = slices.DeleteFunc(imgs, func(img images.Image) bool { + return strings.HasSuffix(img.Name, tmpReducedPlatformSuffix) || strings.HasSuffix(img.Name, tmpEsgzSuffix) + }) + // ImageService().List does not guarantee an order, and the order decides which tag gets indexed. + slices.SortFunc(imgs, func(a, b images.Image) int { + return strings.Compare(a.Name, b.Name) + }) + return imgs, nil +} + +func pushSingle(ctx context.Context, client *containerd.Client, rawRef string, options types.ImagePushOptions, skipSoci bool) error { parsedReference, err := referenceutil.Parse(rawRef) if err != nil { return err @@ -120,7 +189,7 @@ func Push(ctx context.Context, client *containerd.Client, rawRef string, options } pushRef := ref if !options.AllPlatforms { - pushRef = ref + "-tmp-reduced-platform" + pushRef = ref + tmpReducedPlatformSuffix // Push fails with "400 Bad Request" when the manifest is multi-platform but we do not locally have multi-platform blobs. // So we create a tmp reduced-platform image to avoid the error. // Ensure all the layers are here: https://github.com/containerd/nerdctl/issues/3425 @@ -140,7 +209,7 @@ func Push(ctx context.Context, client *containerd.Client, rawRef string, options } if options.Estargz { - pushRef = ref + "-tmp-esgz" + pushRef = ref + tmpEsgzSuffix esgzImg, err := nerdconverter.Convert(ctx, client, pushRef, ref, converter.WithPlatform(platMC), converter.WithLayerConvertFunc(eStargzConvertFunc())) if err != nil { return fmt.Errorf("failed to convert to eStargz: %v", err) @@ -185,7 +254,7 @@ func Push(ctx context.Context, client *containerd.Client, rawRef string, options options.SignOptions); err != nil { return err } - if options.GOptions.Snapshotter == "soci" { + if options.GOptions.Snapshotter == "soci" && !skipSoci { if err = snapshotterutil.CreateSociIndexV1(ref, options.GOptions, options.AllPlatforms, options.Platforms, options.SociOptions); err != nil { return err } @@ -281,11 +350,16 @@ func pushImageWithLocal(ctx context.Context, client *containerd.Client, parsedRe if options.GOptions.InsecureRegistry { log.G(ctx).WithError(err).Warnf("server %q does not seem to support HTTPS, falling back to plain HTTP", refDomain) dOpts = append(dOpts, dockerconfigresolver.WithPlainHTTP(true)) - resolver, err = dockerconfigresolver.New(ctx, refDomain, dOpts...) + // Rebuild the resolver rather than calling dockerconfigresolver.New, which would fall + // back to the process-wide dockerconfigresolver.PushTracker. That tracker is keyed by + // digest, not by reference, so a second push of an already-pushed digest short-circuits + // with ErrAlreadyExists and its tag is never written to the registry. + ho, err = dockerconfigresolver.NewHostOptions(ctx, refDomain, dOpts...) if err != nil { return err } - return pushFunc(resolver) + resolverOpts.Hosts = dockerconfig.ConfigureHosts(ctx, *ho) + return pushFunc(docker.NewResolver(resolverOpts)) } log.G(ctx).WithError(err).Errorf("server %q does not seem to support HTTPS", refDomain) log.G(ctx).Info("Hint: you may want to try --insecure-registry to allow plain HTTP (if you are in a trusted network)")