From a4ec58e900e6cbe76835bd11462810a78a6dc1a1 Mon Sep 17 00:00:00 2001 From: Jimmi Dyson Date: Tue, 1 Aug 2023 17:51:47 +0100 Subject: [PATCH] test: Try to fix proxy flaky tests (#480) --- test/e2e/imagebundle/push_bundle_test.go | 512 ++++++++++++----------- 1 file changed, 261 insertions(+), 251 deletions(-) diff --git a/test/e2e/imagebundle/push_bundle_test.go b/test/e2e/imagebundle/push_bundle_test.go index d367a587..08bf72ff 100644 --- a/test/e2e/imagebundle/push_bundle_test.go +++ b/test/e2e/imagebundle/push_bundle_test.go @@ -40,326 +40,336 @@ var _ = Describe("Push Bundle", func() { tmpDir string ) - BeforeEach(func() { - tmpDir = GinkgoT().TempDir() + Context("Success", func() { + BeforeEach(func() { + tmpDir = GinkgoT().TempDir() - bundleFile = filepath.Join(tmpDir, "image-bundle.tar") + bundleFile = filepath.Join(tmpDir, "image-bundle.tar") - cmd = helpers.NewCommand( - GinkgoT(), - func(out output.Output) *cobra.Command { return pushbundle.NewCommand(out, "image-bundle") }, - ) - }) - - runTest := func( - registryHost string, - registryScheme string, - registryPath string, - registryInsecure bool, - ) { - httpProxy := os.Getenv("http_proxy") - httpsProxy := os.Getenv("https_proxy") - Expect(os.Unsetenv(httpProxy)).To(Succeed()) - helpers.CreateBundle( - GinkgoT(), - bundleFile, - filepath.Join("testdata", "create-success.yaml"), - ) - Expect(os.Setenv("http_proxy", httpProxy)).To(Succeed()) - Expect(os.Setenv("https_proxy", httpsProxy)).To(Succeed()) - - registryCACertFile := "" - registryCertFile := "" - registryKeyFile := "" - if registryHost != "127.0.0.1" && registryScheme != "http" { - tempCertDir := GinkgoT().TempDir() - registryCACertFile, _, registryCertFile, registryKeyFile = helpers.GenerateCertificateAndKeyWithIPSAN( + cmd = helpers.NewCommand( GinkgoT(), - tempCertDir, - net.ParseIP(registryHost), + func(out output.Output) *cobra.Command { return pushbundle.NewCommand(out, "image-bundle") }, ) - } - - port, err := freeport.GetFreePort() - Expect(err).NotTo(HaveOccurred()) - reg, err := registry.NewRegistry(registry.Config{ - StorageDirectory: filepath.Join(tmpDir, "registry"), - Host: registryHost, - Port: uint16(port), - TLS: registry.TLS{ - Certificate: registryCertFile, - Key: registryKeyFile, - }, }) - Expect(err).NotTo(HaveOccurred()) - done := make(chan struct{}) - go func() { - defer GinkgoRecover() + runTest := func( + registryHost string, + registryScheme string, + registryPath string, + registryInsecure bool, + ) { + registryCACertFile := "" + registryCertFile := "" + registryKeyFile := "" + if registryHost != "127.0.0.1" && registryScheme != "http" { + tempCertDir := GinkgoT().TempDir() + registryCACertFile, _, registryCertFile, registryKeyFile = helpers.GenerateCertificateAndKeyWithIPSAN( + GinkgoT(), + tempCertDir, + net.ParseIP(registryHost), + ) + } - Expect(reg.ListenAndServe()).To(Succeed()) + port, err := freeport.GetFreePort() + Expect(err).NotTo(HaveOccurred()) + reg, err := registry.NewRegistry(registry.Config{ + StorageDirectory: filepath.Join(tmpDir, "registry"), + Host: registryHost, + Port: uint16(port), + TLS: registry.TLS{ + Certificate: registryCertFile, + Key: registryKeyFile, + }, + }) + Expect(err).NotTo(HaveOccurred()) - close(done) - }() + done := make(chan struct{}) + go func() { + defer GinkgoRecover() - helpers.WaitForTCPPort(GinkgoT(), registryHost, port) + Expect(reg.ListenAndServe()).To(Succeed()) - registryHostWithOptionalScheme := fmt.Sprintf("%s:%d/%s", registryHost, port, strings.TrimLeft(registryPath, "/")) - if registryScheme != "" { - registryHostWithOptionalScheme = fmt.Sprintf( - "%s://%s", - registryScheme, - registryHostWithOptionalScheme, - ) - } + close(done) + }() - args := []string{ - "--image-bundle", bundleFile, - "--to-registry", registryHostWithOptionalScheme, - } - if registryInsecure { - args = append(args, "--to-registry-insecure-skip-tls-verify") - } else if registryCACertFile != "" { - args = append(args, "--to-registry-ca-cert-file", registryCACertFile) - } + helpers.WaitForTCPPort(GinkgoT(), registryHost, port) - cmd.SetArgs(args) + registryHostWithOptionalScheme := fmt.Sprintf("%s:%d/%s", registryHost, port, strings.TrimLeft(registryPath, "/")) + if registryScheme != "" { + registryHostWithOptionalScheme = fmt.Sprintf( + "%s://%s", + registryScheme, + registryHostWithOptionalScheme, + ) + } - Expect(cmd.Execute()).To(Succeed()) + args := []string{ + "--image-bundle", bundleFile, + "--to-registry", registryHostWithOptionalScheme, + } + if registryInsecure { + args = append(args, "--to-registry-insecure-skip-tls-verify") + } else if registryCACertFile != "" { + args = append(args, "--to-registry-ca-cert-file", registryCACertFile) + } - testRoundTripper, err := httputils.TLSConfiguredRoundTripper( - remote.DefaultTransport, - net.JoinHostPort(registryHost, strconv.Itoa(port)), - registryCACertFile != "", - registryCACertFile, - ) - Expect(err).NotTo(HaveOccurred()) - - helpers.ValidateImageIsAvailable( - GinkgoT(), - registryHost, - port, - registryPath, - "stefanprodan/podinfo", - "6.2.0", - []*v1.Platform{{ - OS: "linux", - Architecture: runtime.GOARCH, - }}, - remote.WithTransport(testRoundTripper), - ) + cmd.SetArgs(args) - Expect(reg.Shutdown(context.Background())).To((Succeed())) + Expect(cmd.Execute()).To(Succeed()) - Eventually(done).Should(BeClosed()) - } + testRoundTripper, err := httputils.TLSConfiguredRoundTripper( + remote.DefaultTransport, + net.JoinHostPort(registryHost, strconv.Itoa(port)), + registryCACertFile != "", + registryCACertFile, + ) + Expect(err).NotTo(HaveOccurred()) - DescribeTable("Success", - runTest, + helpers.ValidateImageIsAvailable( + GinkgoT(), + registryHost, + port, + registryPath, + "stefanprodan/podinfo", + "6.2.0", + []*v1.Platform{{ + OS: "linux", + Architecture: runtime.GOARCH, + }}, + remote.WithTransport(testRoundTripper), + ) - Entry("Without TLS", "127.0.0.1", "", "", true), + Expect(reg.Shutdown(context.Background())).To((Succeed())) - Entry("With TLS", helpers.GetFirstNonLoopbackIP(GinkgoT()).String(), "", "", false), + Eventually(done).Should(BeClosed()) + } - Entry("With Insecure TLS", helpers.GetFirstNonLoopbackIP(GinkgoT()).String(), "", "", true), + DescribeTable("Success", + func( + registryHost string, + registryScheme string, + registryPath string, + registryInsecure bool, + ) { + helpers.CreateBundle( + GinkgoT(), + bundleFile, + filepath.Join("testdata", "create-success.yaml"), + ) - Entry( - "With http registry", - helpers.GetFirstNonLoopbackIP(GinkgoT()).String(), - "http", - "", - true, - ), + runTest(registryHost, registryScheme, registryPath, registryInsecure) + }, - Entry( - "With http registry without TLS skip verify flag", - helpers.GetFirstNonLoopbackIP(GinkgoT()).String(), - "http", - "", - false, - ), + Entry("Without TLS", "127.0.0.1", "", "", true), - Entry("With Subpath", helpers.GetFirstNonLoopbackIP(GinkgoT()).String(), "", "/nested/path/for/registry", false), - ) + Entry("With TLS", helpers.GetFirstNonLoopbackIP(GinkgoT()).String(), "", "", false), - It("Bundle does not exist", func() { - cmd.SetArgs([]string{ - "--image-bundle", bundleFile, - "--to-registry", "127.0.0.1:0/charts", - "--to-registry-insecure-skip-tls-verify", - }) + Entry("With Insecure TLS", helpers.GetFirstNonLoopbackIP(GinkgoT()).String(), "", "", true), - Expect( - cmd.Execute(), - ).To(MatchError(fmt.Sprintf("did find any matching files for %q", bundleFile))) - }) + Entry( + "With http registry", + helpers.GetFirstNonLoopbackIP(GinkgoT()).String(), + "http", + "", + true, + ), - Context("With proxy", Serial, func() { - BeforeEach(func() { - proxy := goproxy.NewProxyHttpServer() - proxy.Verbose = true - proxy.Logger = GinkgoWriter + Entry( + "With http registry without TLS skip verify flag", + helpers.GetFirstNonLoopbackIP(GinkgoT()).String(), + "http", + "", + false, + ), - proxyServer := httptest.NewServer(proxy) - DeferCleanup(proxyServer.Close) + Entry("With Subpath", helpers.GetFirstNonLoopbackIP(GinkgoT()).String(), "", "/nested/path/for/registry", false), + ) - DeferCleanup(os.Setenv, "http_proxy", os.Getenv("http_proxy")) - DeferCleanup(os.Setenv, "https_proxy", os.Getenv("https_proxy")) - Expect(os.Setenv("http_proxy", proxyServer.URL)).To(Succeed()) - Expect(os.Setenv("https_proxy", proxyServer.URL)).To(Succeed()) - }) + It("Bundle does not exist", func() { + cmd.SetArgs([]string{ + "--image-bundle", bundleFile, + "--to-registry", "127.0.0.1:0/charts", + "--to-registry-insecure-skip-tls-verify", + }) - It("Success", func() { - runTest(helpers.GetFirstNonLoopbackIP(GinkgoT()).String(), "", "", false) + Expect( + cmd.Execute(), + ).To(MatchError(fmt.Sprintf("did find any matching files for %q", bundleFile))) }) - Context("With headers from Docker config", func() { + Context("With proxy", Serial, func() { BeforeEach(func() { - dockerConfigDir := GinkgoT().TempDir() - DeferCleanup(os.Setenv, "DOCKER_CONFIG", os.Getenv("DOCKER_CONFIG")) - Expect(os.Setenv("DOCKER_CONFIG", dockerConfigDir)).To(Succeed()) - err := os.WriteFile( - filepath.Join(dockerConfigDir, config.ConfigFileName), - []byte(`{ - "HttpHeaders": { - "MyHeader": "MyValue" - } - }`), - 0o644, + helpers.CreateBundle( + GinkgoT(), + bundleFile, + filepath.Join("testdata", "create-success.yaml"), ) - Expect(err).ToNot(HaveOccurred()) + }) + + JustBeforeEach(func() { + proxy := goproxy.NewProxyHttpServer() + proxy.Verbose = true + proxy.Logger = GinkgoWriter + + proxyServer := httptest.NewServer(proxy) + DeferCleanup(proxyServer.Close) + + GinkgoT().Setenv("http_proxy", proxyServer.URL) + GinkgoT().Setenv("https_proxy", proxyServer.URL) }) It("Success", func() { runTest(helpers.GetFirstNonLoopbackIP(GinkgoT()).String(), "", "", false) }) + + Context("With headers from Docker config", func() { + BeforeEach(func() { + dockerConfigDir := GinkgoT().TempDir() + DeferCleanup(os.Setenv, "DOCKER_CONFIG", os.Getenv("DOCKER_CONFIG")) + Expect(os.Setenv("DOCKER_CONFIG", dockerConfigDir)).To(Succeed()) + err := os.WriteFile( + filepath.Join(dockerConfigDir, config.ConfigFileName), + []byte(`{ + "HttpHeaders": { + "MyHeader": "MyValue" + } + }`), + 0o644, + ) + Expect(err).ToNot(HaveOccurred()) + }) + + It("Success", func() { + runTest(helpers.GetFirstNonLoopbackIP(GinkgoT()).String(), "", "", false) + }) + }) }) - }) - Context("On existing tag", Ordered, func() { - var ( - registryAddress string - outputBuf *bytes.Buffer - ) + Context("On existing tag", Ordered, func() { + var ( + registryAddress string + outputBuf *bytes.Buffer + ) - BeforeAll(func() { - port, err := freeport.GetFreePort() - Expect(err).NotTo(HaveOccurred()) - reg, err := registry.NewRegistry(registry.Config{ - StorageDirectory: filepath.Join(tmpDir, "registry"), - Host: "127.0.0.1", - Port: uint16(port), - }) - Expect(err).NotTo(HaveOccurred()) - registryAddress = fmt.Sprintf("http://127.0.0.1:%d", port) + BeforeAll(func() { + port, err := freeport.GetFreePort() + Expect(err).NotTo(HaveOccurred()) + reg, err := registry.NewRegistry(registry.Config{ + StorageDirectory: filepath.Join(tmpDir, "registry"), + Host: "127.0.0.1", + Port: uint16(port), + }) + Expect(err).NotTo(HaveOccurred()) + registryAddress = fmt.Sprintf("http://127.0.0.1:%d", port) - done := make(chan struct{}) - go func() { - defer GinkgoRecover() + done := make(chan struct{}) + go func() { + defer GinkgoRecover() - Expect(reg.ListenAndServe()).To(Succeed()) + Expect(reg.ListenAndServe()).To(Succeed()) - close(done) - }() + close(done) + }() - DeferCleanup(func() { - Expect(reg.Shutdown(context.Background())).To((Succeed())) + DeferCleanup(func() { + Expect(reg.Shutdown(context.Background())).To((Succeed())) - Eventually(done).Should(BeClosed()) - }) + Eventually(done).Should(BeClosed()) + }) - helpers.WaitForTCPPort(GinkgoT(), "127.0.0.1", port) - }) + helpers.WaitForTCPPort(GinkgoT(), "127.0.0.1", port) + }) - BeforeEach(func() { - helpers.CreateBundle( - GinkgoT(), - bundleFile, - filepath.Join("testdata", "create-success.yaml"), - ) + BeforeEach(func() { + helpers.CreateBundle( + GinkgoT(), + bundleFile, + filepath.Join("testdata", "create-success.yaml"), + ) - DeferCleanup(GinkgoWriter.ClearTeeWriters) - outputBuf = bytes.NewBuffer(nil) - GinkgoWriter.TeeTo(outputBuf) - }) + DeferCleanup(GinkgoWriter.ClearTeeWriters) + outputBuf = bytes.NewBuffer(nil) + GinkgoWriter.TeeTo(outputBuf) + }) - It("Successful push with explicit --on-existing-tag=skip flag even though doesn't exist yet", func() { - args := []string{ - "--image-bundle", bundleFile, - "--to-registry", registryAddress, - "--to-registry-insecure-skip-tls-verify", - "--on-existing-tag=skip", - "--image-push-concurrency=4", - } + It("Successful push with explicit --on-existing-tag=skip flag even though doesn't exist yet", func() { + args := []string{ + "--image-bundle", bundleFile, + "--to-registry", registryAddress, + "--to-registry-insecure-skip-tls-verify", + "--on-existing-tag=skip", + "--image-push-concurrency=4", + } - cmd.SetArgs(args) + cmd.SetArgs(args) - Expect(cmd.Execute()).To(Succeed()) + Expect(cmd.Execute()).To(Succeed()) - Expect(outputBuf.String()).To(ContainSubstring("✓")) - }) + Expect(outputBuf.String()).To(ContainSubstring("✓")) + }) - It("Successful push without on-existing-tag flag (default to overwrite)", func() { - args := []string{ - "--image-bundle", bundleFile, - "--to-registry", registryAddress, - "--to-registry-insecure-skip-tls-verify", - "--image-push-concurrency=4", - } + It("Successful push without on-existing-tag flag (default to overwrite)", func() { + args := []string{ + "--image-bundle", bundleFile, + "--to-registry", registryAddress, + "--to-registry-insecure-skip-tls-verify", + "--image-push-concurrency=4", + } - cmd.SetArgs(args) + cmd.SetArgs(args) - Expect(cmd.Execute()).To(Succeed()) + Expect(cmd.Execute()).To(Succeed()) - Expect(outputBuf.String()).To(ContainSubstring("✓")) - }) + Expect(outputBuf.String()).To(ContainSubstring("✓")) + }) - It("Successful push with explicit --on-existing-tag=overwrite", func() { - args := []string{ - "--image-bundle", bundleFile, - "--to-registry", registryAddress, - "--to-registry-insecure-skip-tls-verify", - "--on-existing-tag=overwrite", - "--image-push-concurrency=4", - } + It("Successful push with explicit --on-existing-tag=overwrite", func() { + args := []string{ + "--image-bundle", bundleFile, + "--to-registry", registryAddress, + "--to-registry-insecure-skip-tls-verify", + "--on-existing-tag=overwrite", + "--image-push-concurrency=4", + } - cmd.SetArgs(args) + cmd.SetArgs(args) - Expect(cmd.Execute()).To(Succeed()) + Expect(cmd.Execute()).To(Succeed()) - Expect(outputBuf.String()).To(ContainSubstring("✓")) - }) + Expect(outputBuf.String()).To(ContainSubstring("✓")) + }) - It("Successful push with explicit --on-existing-tag=skip", func() { - args := []string{ - "--image-bundle", bundleFile, - "--to-registry", registryAddress, - "--to-registry-insecure-skip-tls-verify", - "--on-existing-tag=skip", - "--image-push-concurrency=4", - } + It("Successful push with explicit --on-existing-tag=skip", func() { + args := []string{ + "--image-bundle", bundleFile, + "--to-registry", registryAddress, + "--to-registry-insecure-skip-tls-verify", + "--on-existing-tag=skip", + "--image-push-concurrency=4", + } - cmd.SetArgs(args) + cmd.SetArgs(args) - Expect(cmd.Execute()).To(Succeed()) + Expect(cmd.Execute()).To(Succeed()) - Expect(outputBuf.String()).To(ContainSubstring("✓")) - }) + Expect(outputBuf.String()).To(ContainSubstring("✓")) + }) - It("Failed push with explicit --on-existing-tag=error", func() { - args := []string{ - "--image-bundle", bundleFile, - "--to-registry", registryAddress, - "--to-registry-insecure-skip-tls-verify", - "--on-existing-tag=error", - "--image-push-concurrency=4", - } + It("Failed push with explicit --on-existing-tag=error", func() { + args := []string{ + "--image-bundle", bundleFile, + "--to-registry", registryAddress, + "--to-registry-insecure-skip-tls-verify", + "--on-existing-tag=error", + "--image-push-concurrency=4", + } - cmd.SetArgs(args) + cmd.SetArgs(args) - Expect(cmd.Execute()).To(HaveOccurred()) + Expect(cmd.Execute()).To(HaveOccurred()) - Expect(outputBuf.String()).To(ContainSubstring("✗")) + Expect(outputBuf.String()).To(ContainSubstring("✗")) + }) }) }) })