diff --git a/cmd/nerdctl/image/image_push.go b/cmd/nerdctl/image/image_push.go index 47104a4b7e5..f0535d9bce9 100644 --- a/cmd/nerdctl/image/image_push.go +++ b/cmd/nerdctl/image/image_push.go @@ -47,6 +47,8 @@ func PushCommand() *cobra.Command { cmd.Flags().Bool("all-platforms", false, "Push content for all platforms") // #endregion + cmd.Flags().BoolP("all-tags", "a", false, "Push all tags of an image to the repository") + cmd.Flags().Bool("estargz", false, "Convert the image into eStargz") cmd.Flags().Bool("ipfs-ensure-image", true, "Ensure the entire contents of the image is locally available before push") cmd.Flags().String("ipfs-address", "", "multiaddr of IPFS API (default uses $IPFS_PATH env variable if defined or local directory ~/.ipfs)") @@ -85,6 +87,10 @@ func pushOptions(cmd *cobra.Command) (types.ImagePushOptions, error) { if err != nil { return types.ImagePushOptions{}, err } + allTags, err := cmd.Flags().GetBool("all-tags") + if err != nil { + return types.ImagePushOptions{}, err + } estargz, err := cmd.Flags().GetBool("estargz") if err != nil { return types.ImagePushOptions{}, err @@ -119,6 +125,7 @@ func pushOptions(cmd *cobra.Command) (types.ImagePushOptions, error) { SociOptions: sociOptions, Platforms: platform, AllPlatforms: allPlatforms, + AllTags: allTags, Estargz: estargz, IpfsEnsureImage: ipfsEnsureImage, IpfsAddress: ipfsAddress, diff --git a/cmd/nerdctl/image/image_push_linux_test.go b/cmd/nerdctl/image/image_push_linux_test.go index c547341d012..f585c75c952 100644 --- a/cmd/nerdctl/image/image_push_linux_test.go +++ b/cmd/nerdctl/image/image_push_linux_test.go @@ -17,9 +17,11 @@ package image import ( + "encoding/json" "errors" "fmt" "net/http" + "slices" "testing" "gotest.tools/v3/assert" @@ -278,7 +280,124 @@ func TestPush(t *testing.T) { }, Expected: test.Expects(0, nil, nil), }, + { + Description: "all tags", + Require: require.Not(nerdtest.Docker), + Setup: func(data test.Data, helpers test.Helpers) { + helpers.Ensure("pull", "--quiet", testutil.CommonImage) + testImageRepo := fmt.Sprintf("%s:%d/%s", + registryNoAuthHTTPRandom.IP.String(), registryNoAuthHTTPRandom.Port, data.Identifier()) + data.Labels().Set("testImageRepo", testImageRepo) + helpers.Ensure("tag", testutil.CommonImage, testImageRepo+":v1") + helpers.Ensure("tag", testutil.CommonImage, testImageRepo+":v2") + }, + Cleanup: func(data test.Data, helpers test.Helpers) { + if data.Labels().Get("testImageRepo") != "" { + helpers.Anyhow("rmi", "-f", data.Labels().Get("testImageRepo")+":v1") + helpers.Anyhow("rmi", "-f", data.Labels().Get("testImageRepo")+":v2") + } + }, + Command: func(data test.Data, helpers test.Helpers) test.TestableCommand { + return helpers.Command("push", "--insecure-registry", "--all-tags", data.Labels().Get("testImageRepo")) + }, + Expected: func(data test.Data, helpers test.Helpers) *test.Expected { + return &test.Expected{ + Output: func(stdout string, t tig.T) { + assertRegistryHasTags(t, registryNoAuthHTTPRandom, data.Identifier(), "v1", "v2") + }, + } + }, + }, + { + Description: "all tags, with a tag", + Require: require.Not(nerdtest.Docker), + Setup: func(data test.Data, helpers test.Helpers) { + helpers.Ensure("pull", "--quiet", testutil.CommonImage) + testImageRef := fmt.Sprintf("%s:%d/%s:v1", + registryNoAuthHTTPRandom.IP.String(), registryNoAuthHTTPRandom.Port, data.Identifier()) + data.Labels().Set("testImageRef", testImageRef) + helpers.Ensure("tag", testutil.CommonImage, testImageRef) + }, + Cleanup: func(data test.Data, helpers test.Helpers) { + if data.Labels().Get("testImageRef") != "" { + helpers.Anyhow("rmi", "-f", data.Labels().Get("testImageRef")) + } + }, + Command: func(data test.Data, helpers test.Helpers) test.TestableCommand { + return helpers.Command("push", "--insecure-registry", "--all-tags", data.Labels().Get("testImageRef")) + }, + Expected: test.Expects(1, []error{errors.New("tag can't be used with --all-tags/-a")}, nil), + }, + { + Description: "all tags, no local tag", + Require: require.Not(nerdtest.Docker), + Command: func(data test.Data, helpers test.Helpers) test.TestableCommand { + testImageRepo := fmt.Sprintf("%s:%d/%s", + registryNoAuthHTTPRandom.IP.String(), registryNoAuthHTTPRandom.Port, data.Identifier()) + return helpers.Command("push", "--insecure-registry", "--all-tags", testImageRepo) + }, + Expected: test.Expects(1, []error{errors.New("an image does not exist locally with the tag")}, nil), + }, + { + Description: "all tags, soci", + Require: require.All( + nerdtest.Soci, + require.Not(nerdtest.Docker), + ), + Setup: func(data test.Data, helpers test.Helpers) { + helpers.Ensure("pull", "--quiet", testutil.UbuntuImage) + testImageRepo := fmt.Sprintf("%s:%d/%s", + registryNoAuthHTTPRandom.IP.String(), registryNoAuthHTTPRandom.Port, data.Identifier()) + data.Labels().Set("testImageRepo", testImageRepo) + helpers.Ensure("tag", testutil.UbuntuImage, testImageRepo+":v1") + helpers.Ensure("tag", testutil.UbuntuImage, testImageRepo+":v2") + }, + Cleanup: func(data test.Data, helpers test.Helpers) { + if data.Labels().Get("testImageRepo") != "" { + helpers.Anyhow("rmi", "-f", data.Labels().Get("testImageRepo")+":v1") + helpers.Anyhow("rmi", "-f", data.Labels().Get("testImageRepo")+":v2") + } + }, + Command: func(data test.Data, helpers test.Helpers) test.TestableCommand { + return helpers.Command("push", "--snapshotter=soci", "--insecure-registry", "--all-tags", "--soci-span-size=2097152", "--soci-min-layer-size=20971520", data.Labels().Get("testImageRepo")) + }, + Expected: func(data test.Data, helpers test.Helpers) *test.Expected { + return &test.Expected{ + Output: func(stdout string, t tig.T) { + assertRegistryHasTags(t, registryNoAuthHTTPRandom, data.Identifier(), "v1", "v2") + }, + } + }, + }, }, } testCase.Run(t) } + +// assertRegistryHasTags verifies the registry lists every tag of `want` for the repository `repo`. +// The listing legitimately holds more than the pushed tags: a SOCI v1 index is attached through the +// referrers fallback tag ("sha256-") on registries without the referrers API, and a re-run +// of the test hits a repository the previous run already populated. +func assertRegistryHasTags(t tig.T, reg *registry.Server, repo string, want ...string) { + t.Helper() + + tagsURL := fmt.Sprintf("http://%s:%d/v2/%s/tags/list", reg.IP.String(), reg.Port, repo) + resp, err := http.Get(tagsURL) + assert.NilError(t, err, "error making http request") + defer func() { + if resp.Body != nil { + _ = resp.Body.Close() + } + }() + assert.Equal(t, resp.StatusCode, http.StatusOK, "tag list should be available") + + var tagList struct { + Name string `json:"name"` + Tags []string `json:"tags"` + } + assert.NilError(t, json.NewDecoder(resp.Body).Decode(&tagList), "error decoding the tag list") + + for _, tag := range want { + assert.Assert(t, slices.Contains(tagList.Tags, tag), "expected tag %q in %v", tag, tagList.Tags) + } +} diff --git a/docs/command-reference.md b/docs/command-reference.md index 20239a0f3b0..db3c961ef51 100644 --- a/docs/command-reference.md +++ b/docs/command-reference.md @@ -899,6 +899,7 @@ Flags: - :nerd_face: `--platform=(amd64|arm64|...)`: Push content for a specific platform - :nerd_face: `--all-platforms`: Push content for all platforms +- :whale: `-a, --all-tags`: Push all tags of an image to the repository. `NAME` must not contain a tag. - :nerd_face: `--sign`: Sign the image (none|cosign|notation). See [`./cosign.md`](./cosign.md) and [`./notation.md`](./notation.md) for details. - :nerd_face: `--cosign-key`: Path to the private key file, KMS, URI or Kubernetes Secret for `--sign=cosign` - :nerd_face: `--notation-key-name`: Signing key name for a key previously added to notation's key list for `--sign=notation` @@ -908,7 +909,7 @@ Flags: - :nerd_face: `--soci-span-size`: Span size in bytes that soci index uses to segment layer data. Default is 4 MiB. - :nerd_face: `--soci-min-layer-size`: Minimum layer size in bytes to build zTOC for. Smaller layers won't have zTOC and not lazy pulled. Default is 10 MiB. -Unimplemented `docker push` flags: `--all-tags`, `--disable-content-trust` (default true) +Unimplemented `docker push` flags: `--disable-content-trust` (default true) ### :whale: nerdctl load diff --git a/pkg/api/types/image_types.go b/pkg/api/types/image_types.go index 4bea77dcf8b..503f2a3c776 100644 --- a/pkg/api/types/image_types.go +++ b/pkg/api/types/image_types.go @@ -205,6 +205,8 @@ type ImagePushOptions struct { Platforms []string // AllPlatforms convert content for all platforms AllPlatforms bool + // AllTags push all the tags of the repository named by the reference + AllTags bool // Estargz convert image to sStargz Estargz bool diff --git a/pkg/cmd/image/push.go b/pkg/cmd/image/push.go index 505e8eb7129..fdc55e7ab23 100644 --- a/pkg/cmd/image/push.go +++ b/pkg/cmd/image/push.go @@ -24,6 +24,9 @@ import ( "net/http" "os" "path/filepath" + "regexp" + "slices" + "strings" "github.com/opencontainers/go-digest" ocispec "github.com/opencontainers/image-spec/specs-go/v1" @@ -57,8 +60,74 @@ import ( "github.com/containerd/nerdctl/v2/pkg/snapshotterutil" ) +const ( + // Suffixes of the temporary images push creates for itself before uploading them. + tmpReducedPlatformSuffix = "-tmp-reduced-platform" + tmpEsgzSuffix = "-tmp-esgz" +) + // Push pushes an image specified by `rawRef`. +// With options.AllTags, `rawRef` must be a bare repository name, and every local tag of that +// repository is pushed. func Push(ctx context.Context, client *containerd.Client, rawRef string, options types.ImagePushOptions) error { + if !options.AllTags { + return pushSingle(ctx, client, rawRef, options, false) + } + + parsedReference, err := referenceutil.Parse(rawRef) + if err != nil { + return err + } + // ExplicitTag, not Tag: Parse normalizes a bare repository name to ":latest". + if parsedReference.ExplicitTag != "" || parsedReference.Digest != "" { + return errors.New("tag can't be used with --all-tags/-a") + } + if parsedReference.Protocol != "" { + return fmt.Errorf("--all-tags is not supported for %q references", parsedReference.Protocol) + } + + imgs, err := localTags(ctx, client, parsedReference.Name()) + if err != nil { + return err + } + if len(imgs) == 0 { + return fmt.Errorf("an image does not exist locally with the tag: %s", parsedReference.Name()) + } + + // A SOCI index is attached to the image manifest rather than to the tag, so it only needs to be + // built once per distinct target. Doing it per tag makes every tag overwrite the index pushed by + // the previous one: https://github.com/containerd/nerdctl/issues/3751 + indexed := make(map[digest.Digest]struct{}, len(imgs)) + for _, img := range imgs { + _, done := indexed[img.Target.Digest] + if err = pushSingle(ctx, client, img.Name, options, done); err != nil { + return err + } + indexed[img.Target.Digest] = struct{}{} + } + return nil +} + +// localTags returns the local images tagged under the repository `name`, sorted by name. +func localTags(ctx context.Context, client *containerd.Client, name string) ([]images.Image, error) { + // Same idiom as nameFilterFor() in cmd/nerdctl/image/image_list.go: a bare repository name + // matches every tag of that repository (pkg/ cannot import cmd/, hence the repeated filter). + imgs, err := client.ImageService().List(ctx, fmt.Sprintf("name~=^%s:", regexp.QuoteMeta(name))) + if err != nil { + return nil, err + } + // Drop the temporary images push creates for itself, which an interrupted push may have left behind. + imgs = slices.DeleteFunc(imgs, func(img images.Image) bool { + return strings.HasSuffix(img.Name, tmpReducedPlatformSuffix) || strings.HasSuffix(img.Name, tmpEsgzSuffix) + }) + // ImageService().List does not guarantee an order, and the order decides which tag gets indexed. + slices.SortFunc(imgs, func(a, b images.Image) int { + return strings.Compare(a.Name, b.Name) + }) + return imgs, nil +} + +func pushSingle(ctx context.Context, client *containerd.Client, rawRef string, options types.ImagePushOptions, skipSoci bool) error { parsedReference, err := referenceutil.Parse(rawRef) if err != nil { return err @@ -120,7 +189,7 @@ func Push(ctx context.Context, client *containerd.Client, rawRef string, options } pushRef := ref if !options.AllPlatforms { - pushRef = ref + "-tmp-reduced-platform" + pushRef = ref + tmpReducedPlatformSuffix // Push fails with "400 Bad Request" when the manifest is multi-platform but we do not locally have multi-platform blobs. // So we create a tmp reduced-platform image to avoid the error. // Ensure all the layers are here: https://github.com/containerd/nerdctl/issues/3425 @@ -140,7 +209,7 @@ func Push(ctx context.Context, client *containerd.Client, rawRef string, options } if options.Estargz { - pushRef = ref + "-tmp-esgz" + pushRef = ref + tmpEsgzSuffix esgzImg, err := nerdconverter.Convert(ctx, client, pushRef, ref, converter.WithPlatform(platMC), converter.WithLayerConvertFunc(eStargzConvertFunc())) if err != nil { return fmt.Errorf("failed to convert to eStargz: %v", err) @@ -185,7 +254,7 @@ func Push(ctx context.Context, client *containerd.Client, rawRef string, options options.SignOptions); err != nil { return err } - if options.GOptions.Snapshotter == "soci" { + if options.GOptions.Snapshotter == "soci" && !skipSoci { if err = snapshotterutil.CreateSociIndexV1(ref, options.GOptions, options.AllPlatforms, options.Platforms, options.SociOptions); err != nil { return err } @@ -281,11 +350,16 @@ func pushImageWithLocal(ctx context.Context, client *containerd.Client, parsedRe if options.GOptions.InsecureRegistry { log.G(ctx).WithError(err).Warnf("server %q does not seem to support HTTPS, falling back to plain HTTP", refDomain) dOpts = append(dOpts, dockerconfigresolver.WithPlainHTTP(true)) - resolver, err = dockerconfigresolver.New(ctx, refDomain, dOpts...) + // Rebuild the resolver rather than calling dockerconfigresolver.New, which would fall + // back to the process-wide dockerconfigresolver.PushTracker. That tracker is keyed by + // digest, not by reference, so a second push of an already-pushed digest short-circuits + // with ErrAlreadyExists and its tag is never written to the registry. + ho, err = dockerconfigresolver.NewHostOptions(ctx, refDomain, dOpts...) if err != nil { return err } - return pushFunc(resolver) + resolverOpts.Hosts = dockerconfig.ConfigureHosts(ctx, *ho) + return pushFunc(docker.NewResolver(resolverOpts)) } log.G(ctx).WithError(err).Errorf("server %q does not seem to support HTTPS", refDomain) log.G(ctx).Info("Hint: you may want to try --insecure-registry to allow plain HTTP (if you are in a trusted network)")