[WIP] Disable retries on the CI

.github/workflows/test.yml

@@ -99,12 +99,7 @@ jobs:
           docker run --privileged --rm tonistiigi/binfmt --install linux/arm64
           docker run --privileged --rm tonistiigi/binfmt --install linux/arm/v7
       - name: "Run integration tests"
-        uses: nick-fields/retry@v3
-        with:
-          timeout_minutes: 30
-          max_attempts: 2
-          retry_on: error
-          command: docker run -t --rm --privileged test-integration
+        run: docker run -t --rm --privileged test-integration
 
   test-integration-ipv6:
     runs-on: "ubuntu-${{ matrix.ubuntu }}"
@@ -155,12 +150,7 @@ jobs:
         # On the other side, using the host network is easier at configuration.
         # Besides, each job is running on a different instance, which means using host network here
         # is safe and has no side effects on others.
-        uses: nick-fields/retry@v3
-        with:
-          timeout_minutes: 30
-          max_attempts: 2
-          retry_on: error
-          command: docker run --network host -t --rm --privileged test-integration-ipv6
+        run: docker run --network host -t --rm --privileged test-integration-ipv6
 
   test-integration-rootless:
     runs-on: "ubuntu-${{ matrix.ubuntu }}"
@@ -230,12 +220,7 @@ jobs:
           fi
           echo "WORKAROUND_ISSUE_622=${WORKAROUND_ISSUE_622}" >> "$GITHUB_ENV"
       - name: "Test (network driver=slirp4netns, port driver=builtin)"
-        uses: nick-fields/retry@v3
-        with:
-          timeout_minutes: 30
-          max_attempts: 2
-          retry_on: error
-          command: docker run -t --rm --privileged -e WORKAROUND_ISSUE_622=${WORKAROUND_ISSUE_622} ${TEST_TARGET}
+        run: docker run -t --rm --privileged -e WORKAROUND_ISSUE_622=${WORKAROUND_ISSUE_622} ${TEST_TARGET}
 
   cross:
     runs-on: ubuntu-24.04
@@ -285,21 +270,11 @@ jobs:
         run: |
           sudo apt-get install -y expect
       - name: "Ensure that the integration test suite is compatible with Docker"
-        uses: nick-fields/retry@v3
-        with:
-          timeout_minutes: 30
-          max_attempts: 2
-          retry_on: error
-          # See https://github.com/containerd/nerdctl/blob/main/docs/testing/README.md#about-parallelization
-          command: go test -p 1 -timeout 20m -v -exec sudo ./cmd/nerdctl/... -args -test.target=docker -test.allow-kill-daemon
+        # See https://github.com/containerd/nerdctl/blob/main/docs/testing.md#about-parallelization
+        run: go test -p 1 -timeout 20m -v -exec sudo ./cmd/nerdctl/... -args -test.target=docker -test.allow-kill-daemon
       - name: "Ensure that the IPv6 integration test suite is compatible with Docker"
-        uses: nick-fields/retry@v3
-        with:
-          timeout_minutes: 30
-          max_attempts: 2
-          retry_on: error
-          # See https://github.com/containerd/nerdctl/blob/main/docs/testing/README.md#about-parallelization
-          command: go test -p 1 -timeout 20m -v -exec sudo ./cmd/nerdctl/... -args -test.target=docker -test.allow-kill-daemon -test.only-ipv6
+        # See https://github.com/containerd/nerdctl/blob/main/docs/testing.md#about-parallelization
+        run: go test -p 1 -timeout 20m -v -exec sudo ./cmd/nerdctl/... -args -test.target=docker -test.allow-kill-daemon -test.only-ipv6
 
   test-integration-windows:
     runs-on: windows-2022

Dockerfile

@@ -280,6 +280,7 @@ FROM base AS test-integration
 ARG DEBIAN_FRONTEND=noninteractive
 # `expect` package contains `unbuffer(1)`, which is used for emulating TTY for testing
 RUN apt-get update -qq && apt-get install -qq --no-install-recommends \
+    make \
     expect \
     git
 COPY --from=goversion /GOVERSION /GOVERSION
@@ -290,6 +291,7 @@ ARG GOTESTSUM_VERSION
 RUN GOBIN=/usr/local/bin go install gotest.tools/gotestsum@${GOTESTSUM_VERSION}
 COPY . /go/src/github.com/containerd/nerdctl
 WORKDIR /go/src/github.com/containerd/nerdctl
+RUN git config --global --add safe.directory /go/src/github.com/containerd/nerdctl
 VOLUME /tmp
 ENV CGO_ENABLED=0
 # copy cosign binary for integration test
@@ -318,8 +320,7 @@ RUN curl -o nydus-static.tgz -fsSL --proto '=https' --tlsv1.2 "https://github.co
   tar xzf nydus-static.tgz && \
   mv nydus-static/nydus-image nydus-static/nydusd nydus-static/nydusify /usr/bin/ && \
   rm nydus-static.tgz
-CMD ["gotestsum", "--format=testname", "--rerun-fails=2", "--packages=./cmd/nerdctl/...", \
-  "--", "-timeout=60m", "-p", "1", "-args", "-test.allow-kill-daemon"]
+CMD ["make", "test"]
 
 FROM test-integration AS test-integration-rootless
 # Install SSH for creating systemd user session.
@@ -342,17 +343,15 @@ RUN systemctl disable test-integration-ipfs-offline
 VOLUME /home/rootless/.local/share
 COPY ./Dockerfile.d/test-integration-rootless.sh /
 RUN chmod a+rx /test-integration-rootless.sh
-CMD ["/test-integration-rootless.sh", \
-  "gotestsum", "--format=testname", "--rerun-fails=2", "--packages=./cmd/nerdctl/...", \
-  "--", "-timeout=60m", "-p", "1", "-args", "-test.allow-kill-daemon"]
+CMD ["/test-integration-rootless.sh", "make", "test"]
 
 # test for CONTAINERD_ROOTLESS_ROOTLESSKIT_PORT_DRIVER=slirp4netns
 FROM test-integration-rootless AS test-integration-rootless-port-slirp4netns
 COPY ./Dockerfile.d/home_rootless_.config_systemd_user_containerd.service.d_port-slirp4netns.conf /home/rootless/.config/systemd/user/containerd.service.d/port-slirp4netns.conf
 RUN chown -R rootless:rootless /home/rootless/.config
 
 FROM test-integration AS test-integration-ipv6
-CMD ["gotestsum", "--format=testname", "--rerun-fails=2", "--packages=./cmd/nerdctl/...", \
+CMD ["gotestsum", "--format=testname", "--packages=./cmd/nerdctl/...", \
   "--", "-timeout=60m", "-p", "1", "-args", "-test.allow-kill-daemon", "-test.only-ipv6"]
 
 FROM base AS demo

Makefile

@@ -86,6 +86,14 @@ lint-yaml:
 lint-shell: $(call recursive_wildcard,$(MAKEFILE_DIR)/,*.sh)
 	shellcheck -a -x $^
 
+test: test-no-retry test-flaky
+
+test-flaky:
+	gotestsum --rerun-fails=2 --format=testname --packages=./cmd/nerdctl/builder,./cmd/nerdctl/compose,./cmd/nerdctl/container,./cmd/nerdctl/ipfs,./cmd/nerdctl/image -- -timeout=60m -p 1 -args -test.allow-kill-daemon
+
+test-no-retry:
+	go test $(shell go list ./cmd/nerdctl/... | grep -v nerdctl/builder | grep -v nerdctl/compose | grep -v nerdctl/container | grep -v nerdctl/ipfs | grep -v nerdctl/image) -timeout=60m -p 1 -args -test.allow-kill-daemon
+
 binaries: nerdctl
 
 install:

cmd/nerdctl/completion/completion_linux_test.go

@@ -20,53 +20,189 @@ import (
 	"testing"
 
 	"github.com/containerd/nerdctl/v2/pkg/testutil"
+	"github.com/containerd/nerdctl/v2/pkg/testutil/nerdtest"
+	"github.com/containerd/nerdctl/v2/pkg/testutil/test"
 )
 
 func TestCompletion(t *testing.T) {
-	testutil.DockerIncompatible(t)
-	base := testutil.NewBase(t)
-	const gsc = "__complete"
-	// cmd is executed with base.Args={"--namespace=nerdctl-test"}
-	base.Cmd(gsc, "--cgroup-manager", "").AssertOutContains("cgroupfs\n")
-	base.Cmd(gsc, "--snapshotter", "").AssertOutContains("native\n")
-	base.Cmd(gsc, "").AssertOutContains("run\t")
-	base.Cmd(gsc, "run", "-").AssertOutContains("--network\t")
-	base.Cmd(gsc, "run", "--n").AssertOutContains("--network\t")
-	base.Cmd(gsc, "run", "--ne").AssertOutContains("--network\t")
-	base.Cmd(gsc, "run", "--net", "").AssertOutContains("host\n")
-	base.Cmd(gsc, "run", "-it", "--net", "").AssertOutContains("host\n")
-	base.Cmd(gsc, "run", "-it", "--rm", "--net", "").AssertOutContains("host\n")
-	base.Cmd(gsc, "run", "--restart", "").AssertOutContains("always\n")
-	base.Cmd(gsc, "network", "rm", "").AssertOutNotContains("host\n") // host is unremovable
-	base.Cmd(gsc, "run", "--cap-add", "").AssertOutContains("sys_admin\n")
-	base.Cmd(gsc, "run", "--cap-add", "").AssertOutNotContains("CAP_SYS_ADMIN\n") // invalid form
+	nerdtest.Setup()
 
-	// Tests with an image
-	base.Cmd("pull", testutil.AlpineImage).AssertOK()
-	base.Cmd(gsc, "run", "-i", "").AssertOutContains(testutil.AlpineImage)
-	base.Cmd(gsc, "run", "-it", "").AssertOutContains(testutil.AlpineImage)
-	base.Cmd(gsc, "run", "-it", "--rm", "").AssertOutContains(testutil.AlpineImage)
+	testCase := &test.Case{
+		Description: "Base completion",
+		Require:     test.Not(nerdtest.Docker),
+		Setup: func(data test.Data, helpers test.Helpers) {
+			helpers.Ensure("pull", testutil.AlpineImage)
+			helpers.Ensure("network", "create", data.Identifier())
+			helpers.Ensure("volume", "create", data.Identifier())
+			data.Set("identifier", data.Identifier())
+		},
+		Cleanup: func(data test.Data, helpers test.Helpers) {
+			helpers.Anyhow("network", "rm", data.Identifier())
+			helpers.Anyhow("volume", "rm", data.Identifier())
+		},
+		SubTests: []*test.Case{
+			{
+				Description: "--cgroup-manager",
+				Command:     test.RunCommand("__complete", "--cgroup-manager", ""),
+				Expected:    test.Expects(0, nil, test.Contains("cgroupfs\n")),
+			},
+			{
+				Description: "--snapshotter",
+				Command:     test.RunCommand("__complete", "--snapshotter", ""),
+				Expected:    test.Expects(0, nil, test.Contains("native\n")),
+			},
+			{
+				Description: "empty",
+				Command:     test.RunCommand("__complete", ""),
+				Expected:    test.Expects(0, nil, test.Contains("run\t")),
+			},
+			{
+				Description: "run -",
+				Command:     test.RunCommand("__complete", "run", "-"),
+				Expected:    test.Expects(0, nil, test.Contains("--network\t")),
+			},
+			{
+				Description: "run --n",
+				Command:     test.RunCommand("__complete", "run", "--n"),
+				Expected:    test.Expects(0, nil, test.Contains("--network\t")),
+			},
+			{
+				Description: "run --ne",
+				Command:     test.RunCommand("__complete", "run", "--ne"),
+				Expected:    test.Expects(0, nil, test.Contains("--network\t")),
+			},
+			{
+				Description: "run --net",
+				Command:     test.RunCommand("__complete", "run", "--net", ""),
+				Expected: func(data test.Data, helpers test.Helpers) *test.Expected {
+					return &test.Expected{
+						Output: test.All(
+							test.Contains("host\n"),
+							test.Contains(data.Get("identifier")+"\n"),
+						),
+					}
+				},
+			},
+			{
+				Description: "run -it --net",
+				Command:     test.RunCommand("__complete", "run", "-it", "--net", ""),
+				Expected: func(data test.Data, helpers test.Helpers) *test.Expected {
+					return &test.Expected{
+						Output: test.All(
+							test.Contains("host\n"),
+							test.Contains(data.Get("identifier")+"\n"),
+						),
+					}
+				},
+			},
+			{
+				Description: "run -ti --rm --net",
+				Command:     test.RunCommand("__complete", "run", "-it", "--rm", "--net", ""),
+				Expected: func(data test.Data, helpers test.Helpers) *test.Expected {
+					return &test.Expected{
+						Output: test.All(
+							test.Contains("host\n"),
+							test.Contains(data.Get("identifier")+"\n"),
+						),
+					}
+				},
+			},
+			{
+				Description: "run --restart",
+				Command:     test.RunCommand("__complete", "run", "--restart", ""),
+				Expected:    test.Expects(0, nil, test.Contains("always\n")),
+			},
+			{
+				Description: "network --rm",
+				Command:     test.RunCommand("__complete", "network", "rm", ""),
+				Expected: func(data test.Data, helpers test.Helpers) *test.Expected {
+					return &test.Expected{
+						Output: test.All(
+							test.DoesNotContain("host\n"),
+							test.Contains(data.Get("identifier")+"\n"),
+						),
+					}
+				},
+			},
+			{
+				Description: "run --cap-add",
+				Command:     test.RunCommand("__complete", "run", "--cap-add", ""),
+				Expected: test.Expects(0, nil, test.All(
+					test.Contains("sys_admin\n"),
+					test.DoesNotContain("CAP_SYS_ADMIN\n"),
+				)),
+			},
+			{
+				Description: "volume inspect",
+				Command:     test.RunCommand("__complete", "volume", "inspect", ""),
+				Expected: func(data test.Data, helpers test.Helpers) *test.Expected {
+					return &test.Expected{
+						Output: test.Contains(data.Get("identifier") + "\n"),
+					}
+				},
+			},
+			{
+				Description: "volume rm",
+				Command:     test.RunCommand("__complete", "volume", "rm", ""),
+				Expected: func(data test.Data, helpers test.Helpers) *test.Expected {
+					return &test.Expected{
+						Output: test.Contains(data.Get("identifier") + "\n"),
+					}
+				},
+			},
+			{
+				Description: "no namespace --cgroup-manager",
+				Command: func(data test.Data, helpers test.Helpers) test.Command {
+					cmd := helpers.Command()
+					cmd.Clear()
+					cmd.WithBinary("nerdctl")
+					cmd.WithArgs("__complete", "--cgroup-manager", "")
+					return cmd
+				},
+				Expected: test.Expects(0, nil, test.Contains("cgroupfs\n")),
+			},
+			{
+				Description: "no namespace empty",
+				Command: func(data test.Data, helpers test.Helpers) test.Command {
+					return helpers.Command().Clear().WithBinary("nerdctl").WithArgs("__complete", "")
+				},
+				Expected: test.Expects(0, nil, test.Contains("run\t")),
+			},
+			{
+				Description: "namespace space empty",
+				Command: func(data test.Data, helpers test.Helpers) test.Command {
+					// mind {"--namespace=nerdctl-test"} vs {"--namespace", "nerdctl-test"}
+					return helpers.Command().Clear().WithBinary("nerdctl").
+						WithArgs("__complete", "--namespace", testutil.Namespace, "")
+				},
+				Expected: test.Expects(0, nil, test.Contains("run\t")),
+			},
+			{
+				Description: "run -i",
+				Command:     test.RunCommand("__complete", "run", "-i", ""),
+				Expected:    test.Expects(0, nil, test.Contains(testutil.AlpineImage)),
+			},
+			{
+				Description: "run -it",
+				Command:     test.RunCommand("__complete", "run", "-it", ""),
+				Expected:    test.Expects(0, nil, test.Contains(testutil.AlpineImage)),
+			},
+			{
+				Description: "run -it --rm",
+				Command:     test.RunCommand("__complete", "run", "-it", "--rm", ""),
+				Expected:    test.Expects(0, nil, test.Contains(testutil.AlpineImage)),
+			},
+			{
+				Description: "namespace run -i",
+				Command: func(data test.Data, helpers test.Helpers) test.Command {
+					// mind {"--namespace=nerdctl-test"} vs {"--namespace", "nerdctl-test"}
+					return helpers.Command().Clear().WithBinary("nerdctl").
+						WithArgs("__complete", "--namespace", testutil.Namespace, "run", "-i", "")
+				},
+				Expected: test.Expects(0, nil, test.Contains(testutil.AlpineImage+"\n")),
+			},
+		},
+	}
 
-	// Tests with a network
-	testNetworkName := "nerdctl-test-completion"
-	defer base.Cmd("network", "rm", testNetworkName).Run()
-	base.Cmd("network", "create", testNetworkName).AssertOK()
-	base.Cmd(gsc, "network", "rm", "").AssertOutContains(testNetworkName)
-	base.Cmd(gsc, "run", "--net", "").AssertOutContains(testNetworkName)
-
-	// Tests with a volume
-	testVolumekName := "nerdctl-test-completion"
-	defer base.Cmd("volume", "rm", testVolumekName).Run()
-	base.Cmd("volume", "create", testVolumekName).AssertOK()
-	base.Cmd(gsc, "volume", "inspect", "").AssertOutContains(testVolumekName)
-	base.Cmd(gsc, "volume", "rm", "").AssertOutContains(testVolumekName)
-
-	// Tests with raw base (without Args={"--namespace=nerdctl-test"})
-	rawBase := testutil.NewBase(t)
-	rawBase.Args = nil // unset "--namespace=nerdctl-test"
-	rawBase.Cmd(gsc, "--cgroup-manager", "").AssertOutContains("cgroupfs\n")
-	rawBase.Cmd(gsc, "").AssertOutContains("run\t")
-	// mind {"--namespace=nerdctl-test"} vs {"--namespace", "nerdctl-test"}
-	rawBase.Cmd(gsc, "--namespace", testutil.Namespace, "").AssertOutContains("run\t")
-	rawBase.Cmd(gsc, "--namespace", testutil.Namespace, "run", "-i", "").AssertOutContains(testutil.AlpineImage)
+	testCase.Run(t)
 }