From 9597383eccc64b85e0ec8aeb1d1d10d0ae746ca3 Mon Sep 17 00:00:00 2001 From: Patrick Ohly Date: Mon, 20 Apr 2020 09:41:45 +0200 Subject: [PATCH 1/2] test: add newline after command output This fixes broken output like this one: Nmap done: 1 IP address (1 host up) scanned in 0.24 secondsSTEP:delete the pod --- test/e2e/pod/exec.go | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/test/e2e/pod/exec.go b/test/e2e/pod/exec.go index 748a0fd588..2ea0de751b 100644 --- a/test/e2e/pod/exec.go +++ b/test/e2e/pod/exec.go @@ -56,7 +56,7 @@ func RunInPod(f *framework.Framework, rootdir string, items []string, command st stdout, stderr, err := f.ExecWithOptions(options) framework.ExpectNoError(err, "command failed in namespace %s, pod/container %s/%s:\nstderr:\n%s\nstdout:%s\n", namespace, pod, container, stderr, stdout) - fmt.Fprintf(GinkgoWriter, "stderr:\n%s\nstdout:\n%s", + fmt.Fprintf(GinkgoWriter, "stderr:\n%s\nstdout:\n%s\n", stderr, stdout) return stdout, stderr From 40250fee449b799b1086bad1d1d03b661581628e Mon Sep 17 00:00:00 2001 From: Patrick Ohly Date: Mon, 20 Apr 2020 09:42:35 +0200 Subject: [PATCH 2/2] test: retry TLS checking if no ports found The test has been flaky, sometimes failing with no open ports found. Perhaps that was because networking hadn't been fully configured yet when running nmap, so now the test is repeated until at least some open port is found. --- test/e2e/tls/tls.go | 185 +++++++++++++++++++++++--------------------- 1 file changed, 96 insertions(+), 89 deletions(-) diff --git a/test/e2e/tls/tls.go b/test/e2e/tls/tls.go index 908d2c0961..e72239da19 100644 --- a/test/e2e/tls/tls.go +++ b/test/e2e/tls/tls.go @@ -85,102 +85,106 @@ func checkTLS(f *framework.Framework, server string) { podErr := e2epod.WaitForPodRunningInNamespace(f.ClientSet, createdPod) framework.ExpectNoError(podErr, "running pod") - By("scanning ports") - // We have to patch nmap because of https://github.com/nmap/nmap/issues/1187#issuecomment-587031079. - output, _ := pmempod.RunInPod(f, os.Getenv("REPO_ROOT")+"/test/e2e/tls", []string{"nmap-ssl-enum-ciphers.patch"}, + // Install and patch nmap. + pmempod.RunInPod(f, os.Getenv("REPO_ROOT")+"/test/e2e/tls", []string{"nmap-ssl-enum-ciphers.patch"}, strings.Join([]string{ fmt.Sprintf("https_proxy=%s swupd bundle-add nmap patch >&2", os.Getenv("HTTPS_PROXY")), "patch /usr/share/nmap/scripts/ssl-enum-ciphers.nse &1", server), }, " && "), ns, pod.Name, containerName) - // Now analyze all ports and the ciphers found for them. - // The output will be something like this: - // Nmap scan report for pmem-csi-controller-0.pmem-csi-controller.default (10.44.0.1) - // Host is up (0.00013s latency). - // rDNS record for 10.44.0.1: pmem-csi-controller-0.pmem-csi-controller.default.svc.cluster.local - // Not shown: 997 closed ports - // PORT STATE SERVICE - // 8000/tcp open http-alt - // | ssl-enum-ciphers: - // | TLSv1.0: - // | ciphers: - // | TLS_ECDHE_ECDSA_WITH_AES_128_CBC_SHA (ecdh_x25519) - A - // | TLS_ECDHE_ECDSA_WITH_AES_256_CBC_SHA (ecdh_x25519) - A - // | compressors: - // | NULL - // | cipher preference: server - // | TLSv1.1: - // | ciphers: - // | TLS_ECDHE_ECDSA_WITH_AES_128_CBC_SHA (ecdh_x25519) - A - // | TLS_ECDHE_ECDSA_WITH_AES_256_CBC_SHA (ecdh_x25519) - A - // | compressors: - // | NULL - // | cipher preference: server - // | TLSv1.2: - // | ciphers: - // | TLS_ECDHE_ECDSA_WITH_AES_128_GCM_SHA256 (ecdh_x25519) - A - // | TLS_ECDHE_ECDSA_WITH_AES_256_GCM_SHA384 (ecdh_x25519) - A - // | TLS_ECDHE_ECDSA_WITH_CHACHA20_POLY1305_SHA256 (ecdh_x25519) - A - // | TLS_ECDHE_ECDSA_WITH_AES_128_CBC_SHA (ecdh_x25519) - A - // | TLS_ECDHE_ECDSA_WITH_AES_256_CBC_SHA (ecdh_x25519) - A - // | compressors: - // | NULL - // | cipher preference: server - // |_ least strength: A - // 10000/tcp open snet-sensor-mgmt - // | ssl-enum-ciphers: - // | TLSv1.2: - // | ciphers: - // | TLS_ECDHE_ECDSA_WITH_AES_128_CBC_SHA256 (ecdh_x25519) - A - // | TLS_ECDHE_ECDSA_WITH_AES_128_GCM_SHA256 (ecdh_x25519) - A - // | TLS_ECDHE_ECDSA_WITH_AES_256_GCM_SHA384 (ecdh_x25519) - A - // | TLS_ECDHE_ECDSA_WITH_CHACHA20_POLY1305_SHA256 (ecdh_x25519) - A - // | TLS_ECDHE_ECDSA_WITH_RC4_128_SHA (ecdh_x25519) - C - // | compressors: - // | NULL - // | cipher preference: client - // | warnings: - // | Broken cipher RC4 is deprecated by RFC 7465 - // |_ least strength: C - // 10002/tcp open documentum - // MAC Address: D2:82:BC:59:C9:CC (Unknown) - // - // Nmap done: 1 IP address (1 host up) scanned in 0.34 seconds - - // We need the full strings if the comparison below fails. - old := format.TruncatedDiff - defer func() { - format.TruncatedDiff = old - }() - format.TruncatedDiff = false - - re := regexp.MustCompile(`(?m)^([[:digit:]]+)/.* open .*\n((?:^\|.*\n)*)`) - ports := re.FindAllStringSubmatch(output, -1) - Expect(ports).NotTo(BeEmpty(), "ports found") - for _, entry := range ports { - port, ciphers := entry[1], entry[2] - if port == "10002" { - // The socat debugging port. Can be ignored. - continue - } - // All other ports must use TLS, with exactly the - // ciphers that we want enabled. All of them should be rated A. - // - // The exact output depends on: - // - the version of nmap (locked on release branches by fixing the Clear Linux - // release, varies on development branches) - // - the version of Go that is being used for building PMEM-CSI (locked - // in our Dockerfile) - // - the generated keys and thus the deployment method (the - // current list is for "make start" and keys created with - // test/setup-ca.sh, which in turn uses cfssl as installed - // by test/test.make, at least in the CI) + Eventually(func() int { + By("scanning ports") + // We have to patch nmap because of https://github.com/nmap/nmap/issues/1187#issuecomment-587031079. + output, _ := pmempod.RunInPod(f, os.Getenv("REPO_ROOT")+"/test/e2e/tls", []string{"nmap-ssl-enum-ciphers.patch"}, + fmt.Sprintf("nmap --script +ssl-enum-ciphers --open -Pn %s 2>&1", server), + ns, pod.Name, containerName) + + // Now analyze all ports and the ciphers found for them. + // The output will be something like this: + // Nmap scan report for pmem-csi-controller-0.pmem-csi-controller.default (10.44.0.1) + // Host is up (0.00013s latency). + // rDNS record for 10.44.0.1: pmem-csi-controller-0.pmem-csi-controller.default.svc.cluster.local + // Not shown: 997 closed ports + // PORT STATE SERVICE + // 8000/tcp open http-alt + // | ssl-enum-ciphers: + // | TLSv1.0: + // | ciphers: + // | TLS_ECDHE_ECDSA_WITH_AES_128_CBC_SHA (ecdh_x25519) - A + // | TLS_ECDHE_ECDSA_WITH_AES_256_CBC_SHA (ecdh_x25519) - A + // | compressors: + // | NULL + // | cipher preference: server + // | TLSv1.1: + // | ciphers: + // | TLS_ECDHE_ECDSA_WITH_AES_128_CBC_SHA (ecdh_x25519) - A + // | TLS_ECDHE_ECDSA_WITH_AES_256_CBC_SHA (ecdh_x25519) - A + // | compressors: + // | NULL + // | cipher preference: server + // | TLSv1.2: + // | ciphers: + // | TLS_ECDHE_ECDSA_WITH_AES_128_GCM_SHA256 (ecdh_x25519) - A + // | TLS_ECDHE_ECDSA_WITH_AES_256_GCM_SHA384 (ecdh_x25519) - A + // | TLS_ECDHE_ECDSA_WITH_CHACHA20_POLY1305_SHA256 (ecdh_x25519) - A + // | TLS_ECDHE_ECDSA_WITH_AES_128_CBC_SHA (ecdh_x25519) - A + // | TLS_ECDHE_ECDSA_WITH_AES_256_CBC_SHA (ecdh_x25519) - A + // | compressors: + // | NULL + // | cipher preference: server + // |_ least strength: A + // 10000/tcp open snet-sensor-mgmt + // | ssl-enum-ciphers: + // | TLSv1.2: + // | ciphers: + // | TLS_ECDHE_ECDSA_WITH_AES_128_CBC_SHA256 (ecdh_x25519) - A + // | TLS_ECDHE_ECDSA_WITH_AES_128_GCM_SHA256 (ecdh_x25519) - A + // | TLS_ECDHE_ECDSA_WITH_AES_256_GCM_SHA384 (ecdh_x25519) - A + // | TLS_ECDHE_ECDSA_WITH_CHACHA20_POLY1305_SHA256 (ecdh_x25519) - A + // | TLS_ECDHE_ECDSA_WITH_RC4_128_SHA (ecdh_x25519) - C + // | compressors: + // | NULL + // | cipher preference: client + // | warnings: + // | Broken cipher RC4 is deprecated by RFC 7465 + // |_ least strength: C + // 10002/tcp open documentum + // MAC Address: D2:82:BC:59:C9:CC (Unknown) // - // This list may have to be adapted when changing either of these. - Expect(ciphers).To(Equal(`| ssl-enum-ciphers: + // Nmap done: 1 IP address (1 host up) scanned in 0.34 seconds + + // We need the full strings if the comparison below fails. + old := format.TruncatedDiff + defer func() { + format.TruncatedDiff = old + }() + format.TruncatedDiff = false + + re := regexp.MustCompile(`(?m)^([[:digit:]]+)/.* open .*\n((?:^\|.*\n)*)`) + ports := re.FindAllStringSubmatch(output, -1) + for _, entry := range ports { + port, ciphers := entry[1], entry[2] + if port == "10002" { + // The socat debugging port. Can be ignored. + continue + } + // All other ports must use TLS, with exactly the + // ciphers that we want enabled. All of them should be rated A. + // + // The exact output depends on: + // - the version of nmap (locked on release branches by fixing the Clear Linux + // release, varies on development branches) + // - the version of Go that is being used for building PMEM-CSI (locked + // in our Dockerfile) + // - the generated keys and thus the deployment method (the + // current list is for "make start" and keys created with + // test/setup-ca.sh, which in turn uses cfssl as installed + // by test/test.make, at least in the CI) + // + // This list may have to be adapted when changing either of these. + Expect(ciphers).To(Equal(`| ssl-enum-ciphers: | TLSv1.2: | ciphers: | TLS_ECDHE_ECDSA_WITH_AES_128_CBC_SHA256 (ecdh_x25519) - A @@ -192,5 +196,8 @@ func checkTLS(f *framework.Framework, server string) { | cipher preference: client |_ least strength: A `), "ciphers for port %s in %s", port, server) - } + } + + return len(ports) + }, "1m", "5s").Should(BeNumerically(">", 0), "no open ports found, networking down?") }