build: collect coverage information in Jenkins

The support for this in the pmem-csi-driver-test still worked, we just have to
enable it before running tests. Running "make kustomize" with a different base
deployment achieves that. It modifies the current tree, but that is okay for
usage in Jenkins where the directory is going to be deleted.

Jenkins can consume Cobertura XML when the right set of plugins are
installed. How to do that was mostly based on trial-and-error, upstream
documentation was vague.

Jenkinsfile

@@ -68,6 +68,10 @@ pipeline {
         TEST_ETCD_TMPFS = "${WORKSPACE}/_work/${env.CLUSTER}/data/pmem-csi-${env.CLUSTER}-master/etcd-tmpfs"
         TEST_ETCD_VOLUME = "${env.TEST_ETCD_TMPFS}/etcd-volume"
         TEST_ETCD_VOLUME_SIZE = "1073741824" // 1GB
+
+        // Tests that will get skipped when collecting coverage information.
+        // The operator tests fail because the reference files get modified.
+        COVERAGE_SKIP = "[email protected]"
     }
 
     stages {
@@ -183,7 +187,19 @@ pipeline {
                 stage('1.22') {
                     steps {
                         // Skip production, i.e. run testing.
-                        TestInVM("", "fedora", "", "1.22", "Top.Level..[[:alpha:]]*-production[[:space:]]")
+                        TestInVM("", "", "fedora", "", "1.22", "Top.Level..[[:alpha:]]*-production[[:space:]]", "")
+                    }
+                }
+                stage('coverage-1.22') {
+                    when {
+		        beforeAgent true
+		        not { changeRequest() }
+                    }
+                    agent {
+                        label "pmem-csi"
+                    }
+                    steps {
+                        TestInVM("fedora-coverage-1.22", "coverage-", "fedora", "", "1.22", "", "${env.COVERAGE_SKIP}")
                     }
                 }
 
@@ -197,7 +213,7 @@ pipeline {
                         label "pmem-csi"
                     }
                     steps {
-                        TestInVM("fedora-1.21", "fedora", "", "1.21", "")
+                        TestInVM("fedora-1.21", "", "fedora", "", "1.21", "", "")
                     }
                 }
                 stage('1.20') {
@@ -209,7 +225,7 @@ pipeline {
                         label "pmem-csi"
                     }
                     steps {
-                        TestInVM("fedora-1.20", "fedora", "", "1.20", "")
+                        TestInVM("fedora-1.20", "", "fedora", "", "1.20", "", "")
                     }
                 }
                 stage('1.19') {
@@ -218,7 +234,19 @@ pipeline {
                     }
                     steps {
                         // Skip testing, i.e. run production.
-                        TestInVM("fedora-1.19", "fedora", "", "1.19",  "Top.Level..[[:alpha:]]*-testing[[:space:]]")
+                        TestInVM("fedora-1.19", "", "fedora", "", "1.19",  "Top.Level..[[:alpha:]]*-testing[[:space:]]", "")
+                    }
+                }
+                stage('coverage-1.19') {
+                    when {
+		        beforeAgent true
+		        not { changeRequest() }
+                    }
+                    agent {
+                        label "pmem-csi"
+                    }
+                    steps {
+                        TestInVM("fedora-coverage-1.19", "coverage-", "fedora", "", "1.19", "", "${env.COVERAGE_SKIP}")
                     }
                 }
             }
@@ -436,10 +464,13 @@ void RestoreEnv() {
         done"
 }
 
-void TestInVM(worker, distro, distroVersion, kubernetesVersion, skipIfPR) {
+void TestInVM(worker, coverage, distro, distroVersion, kubernetesVersion, skipIfPR, skipAlways) {
     if (worker) {
         RestoreEnv()
     }
+    if (coverage) {
+        sh "${RunInBuilder()} -e CLUSTER=${env.CLUSTER} ${env.BUILD_CONTAINER} make kustomize KUSTOMIZE_WITH_COVERAGE=true"
+    }
     try { timeout(unit: "HOURS", time: TestTimeoutHours()) {
         /*
         We have to run "make start" in the current directory
@@ -463,7 +494,7 @@ void TestInVM(worker, distro, distroVersion, kubernetesVersion, skipIfPR) {
         so for now we disable VMX with -vmx.
         */
         sh "#!/bin/bash\n \
-           echo Note: job output is filtered, see joblog-${BUILD_TAG}-test-${kubernetesVersion}.log artifact for full output. && \
+           echo Note: job output is filtered, see joblog-${BUILD_TAG}-test-${coverage}${kubernetesVersion}.log artifact for full output. && \
            set -o pipefail && \
            ( \
            loggers=; \
@@ -508,10 +539,10 @@ void TestInVM(worker, distro, distroVersion, kubernetesVersion, skipIfPR) {
                                  done | sed -e \"s/^/\$hostname: /\" ) & \
                                loggers=\"\$loggers \$!\"; \
                            done && \
-                           testrun=\$(echo '${distro}-${distroVersion}-${kubernetesVersion}' | sed -e s/--*/-/g | tr . _ ) && \
+                           testrun=\$(echo '${distro}-${distroVersion}-${coverage}${kubernetesVersion}' | sed -e s/--*/-/g | tr . _ ) && \
                            make test_e2e TEST_E2E_REPORT_DIR=${WORKSPACE}/build/reports.tmp/\$testrun \
-                                         TEST_E2E_SKIP=\$(if [ \"${env.CHANGE_ID}\" ] && [ \"${env.CHANGE_ID}\" != null ]; then echo \\\\[Slow\\\\]@${skipIfPR}; fi) \
-                           ') 2>&1 | tee joblog-${BUILD_TAG}-test-${kubernetesVersion}.log | grep --line-buffered -E -e 'checking for test|Passed|FAIL:|^ERROR' \
+                                         TEST_E2E_SKIP=${skipAlways}@\$(if [ \"${env.CHANGE_ID}\" ] && [ \"${env.CHANGE_ID}\" != null ]; then echo \\\\[Slow\\\\]@${skipIfPR}; fi) \
+                           ') 2>&1 | tee joblog-${BUILD_TAG}-test-${coverage}${kubernetesVersion}.log | grep --line-buffered -E -e 'checking for test|Passed|FAIL:|^ERROR' \
            "
     } } finally {
         echo "Writing cluster state and kubelet logs into files."
@@ -545,6 +576,30 @@ void TestInVM(worker, distro, distroVersion, kubernetesVersion, skipIfPR) {
            done'''
         archiveArtifacts('**/joblog-*')
         junit 'build/reports/**/*.xml'
+
+        if (coverage) {
+            // https://stackoverflow.com/questions/36918370/cobertura-code-coverage-report-for-jenkins-pipeline-jobs
+            // https://www.jenkins.io/doc/pipeline/steps/cobertura/
+            sh "${RunInBuilder()} -e CLUSTER=${env.CLUSTER} ${env.BUILD_CONTAINER} make _work/coverage/coverage.txt _work/coverage/coverage.xml"
+            sh "cat _work/coverage/coverage.txt"
+            // The tag ensures that reports from different jobs get merged.
+            // https://plugins.jenkins.io/code-coverage-api/#plugin-content-reports-combining-support
+            // works, no source code:   publishCoverage adapters: [cobertura(coberturaReportFile: '_work/coverage/coverage.xml')], tag: 't'
+            // Simplify relative paths ("github.com/intel/pmem-csi/...").
+            // To view source code, Jenkins users must be logged in (https://stackoverflow.com/a/59951809).
+            sh "sed -e 's;filename=\"github.com/intel/pmem-csi/;filename=\";g' _work/coverage/coverage.xml >coverage.xml"
+
+            // The relationship between "Code Coverage API Plugin" and "Cobertura" plugin is not clear.
+            // With just "Code Coverage API Plugin" installed, this here works, but doesn't show source code
+            // (old UI?):
+            // publishCoverage adapters: [cobertura(path: 'coverage.xml')], tag: 't'
+
+            // When both are installed, this here works (note the different coberura parameter!)
+            // and shows source code.
+            publishCoverage adapters: [cobertura(coberturaReportFile: 'coverage.xml')], tag: 't'
+
+            // There is also a "coberturaAdapter". That one hasn't been tested.
+        }
     }
 }
 

Makefile

@@ -159,14 +159,21 @@ KUSTOMIZE_INPUT := $(shell [ ! -d deploy/kustomize ] || find deploy/kustomize -t
 # Output files and their corresponding kustomization, in the format <target .yaml>=<kustomization directory>
 KUSTOMIZE :=
 
+# Setting this to any non-empty value before "make kustomize" will enable
+# collection of coverage profiles in all deployments.
+KUSTOMIZE_WITH_COVERAGE =
+ifneq "$(KUSTOMIZE_WITH_COVERAGE)" ""
+KUSTOMIZE_COVERAGE_SUFFIX = -coverage
+endif
+
 # For each supported Kubernetes version, we provide four different flavors.
 # The "testing" flavor of the generated files contains both
 # the loglevel changes but does not enable coverage data collection.
 KUSTOMIZE_KUBERNETES_OUTPUT = \
-    deploy/kubernetes-X.XX/pmem-csi-direct.yaml=deploy/kustomize/kubernetes-base-direct \
-    deploy/kubernetes-X.XX/pmem-csi-lvm.yaml=deploy/kustomize/kubernetes-base-lvm \
-    deploy/kubernetes-X.XX/pmem-csi-direct-testing.yaml=deploy/kustomize/kubernetes-base-direct-testing \
-    deploy/kubernetes-X.XX/pmem-csi-lvm-testing.yaml=deploy/kustomize/kubernetes-base-lvm-testing \
+    deploy/kubernetes-X.XX/pmem-csi-direct.yaml=deploy/kustomize/kubernetes-base-direct$(KUSTOMIZE_COVERAGE_SUFFIX) \
+    deploy/kubernetes-X.XX/pmem-csi-lvm.yaml=deploy/kustomize/kubernetes-base-lvm$(KUSTOMIZE_COVERAGE_SUFFIX) \
+    deploy/kubernetes-X.XX/pmem-csi-direct-testing.yaml=deploy/kustomize/kubernetes-base-direct-testing$(KUSTOMIZE_COVERAGE_SUFFIX) \
+    deploy/kubernetes-X.XX/pmem-csi-lvm-testing.yaml=deploy/kustomize/kubernetes-base-lvm-testing$(KUSTOMIZE_COVERAGE_SUFFIX) \
 
 # Kubernetes versions derived from kubernetes-base.
 #

deploy/kustomize/kubernetes-1.21-direct-testing-coverage/kustomization.yaml

@@ -0,0 +1,13 @@
+bases:
+- ../kubernetes-base-direct-testing-coverage
+
+patches:
+- ../patches/driverinfo-storage-capacity-patch.yaml
+
+patchesJson6902:
+- target:
+    group: apps
+    version: v1
+    kind: DaemonSet
+    name: pmem-csi-intel-com-node
+  path: ../patches/external-provisioner-storage-capacity-patch.yaml

deploy/kustomize/kubernetes-1.21-lvm-testing-coverage/kustomization.yaml

@@ -0,0 +1,13 @@
+bases:
+- ../kubernetes-base-lvm-testing-coverage
+
+patches:
+- ../patches/driverinfo-storage-capacity-patch.yaml
+
+patchesJson6902:
+- target:
+    group: apps
+    version: v1
+    kind: DaemonSet
+    name: pmem-csi-intel-com-node
+  path: ../patches/external-provisioner-storage-capacity-patch.yaml

deploy/kustomize/kubernetes-base-direct-coverage/kustomization.yaml

@@ -1,5 +1,5 @@
 bases:
-- ../kubernetes-base-direct-testing/
+- ../kubernetes-base-direct/
 
 patchesJson6902:
 - target:

deploy/kustomize/kubernetes-base-direct-testing-coverage/kustomization.yaml

@@ -0,0 +1,21 @@
+bases:
+- ../kubernetes-base-direct-testing/
+
+patchesJson6902:
+- target:
+    group: apps
+    version: v1
+    kind: Deployment
+    name: pmem-csi-intel-com-controller
+  path: ../testing/controller-coverage-patch.yaml
+
+- target:
+    group: apps
+    version: v1
+    kind: DaemonSet
+    name: pmem-csi-intel-com-node
+  path: ../testing/node-coverage-patch.yaml
+
+images:
+- name: intel/pmem-csi-driver
+  newName: intel/pmem-csi-driver-test

deploy/kustomize/kubernetes-base-lvm-coverage/kustomization.yaml

@@ -1,5 +1,5 @@
 bases:
-- ../kubernetes-base-lvm-testing/
+- ../kubernetes-base-lvm/
 
 patchesJson6902:
 - target:

deploy/kustomize/kubernetes-base-lvm-testing-coverage/kustomization.yaml

@@ -0,0 +1,21 @@
+bases:
+- ../kubernetes-base-lvm-testing/
+
+patchesJson6902:
+- target:
+    group: apps
+    version: v1
+    kind: Deployment
+    name: pmem-csi-intel-com-controller
+  path: ../testing/controller-coverage-patch.yaml
+
+- target:
+    group: apps
+    version: v1
+    kind: DaemonSet
+    name: pmem-csi-intel-com-node
+  path: ../testing/node-coverage-patch.yaml
+
+images:
+- name: intel/pmem-csi-driver
+  newName: intel/pmem-csi-driver-test

test/e2e/gotests/gotests.go

@@ -46,7 +46,7 @@ func runGoTest(f *framework.Framework, pkg string) {
 	root := os.Getenv("REPO_ROOT")
 	var err error
 
-	build := exec.Command("/bin/sh", "-c", os.Getenv("TEST_CMD")+" -c -o _work/test.test "+pkg)
+	build := exec.Command("/bin/sh", "-c", os.Getenv("TEST_CMD")+" --cover -covermode=atomic -c -o _work/test.test "+pkg)
 	build.Stdout = GinkgoWriter
 	build.Stderr = GinkgoWriter
 	build.Dir = root
@@ -60,11 +60,28 @@ func runGoTest(f *framework.Framework, pkg string) {
 	Expect(pods.Items).NotTo(BeEmpty(), "have PMEM-CSI pods")
 	pmem := pods.Items[0]
 
+	coverDir := "/var/lib/pmem-csi-coverage"
+	coverFile := strings.ReplaceAll(pkg, "/", "-")
+	if strings.HasPrefix(coverFile, ".") {
+		coverFile = coverFile[1:]
+	}
+
 	By(fmt.Sprintf("Running in PMEM-CSI pod %s", pmem.Name))
 	pod.RunInPod(f, root,
 		[]string{"_work/test.test"},
-		"if /tmp/test.test -h 2>&1 | grep -q ginkgo; then "+
-			"TEST_WORK=_work REPO_ROOT=. /tmp/test.test -ginkgo.v; else "+
-			"TEST_WORK=_work REPO_ROOT=. /tmp/test.test; fi",
+		""+
+			"if [ -d "+coverDir+" ]; then "+
+			"  if /tmp/test.test -h 2>&1 | grep -q ginkgo; then "+
+			"     TEST_WORK=_work REPO_ROOT=. /tmp/test.test -test.coverprofile="+coverDir+"/gotest"+coverFile+".out -ginkgo.v; "+
+			"  else "+
+			"     TEST_WORK=_work REPO_ROOT=. /tmp/test.test -test.coverprofile="+coverDir+"/gotest"+coverFile+".out;"+
+			"  fi;"+
+			"else "+
+			"  if /tmp/test.test -h 2>&1 | grep -q ginkgo; then "+
+			"     TEST_WORK=_work REPO_ROOT=. /tmp/test.test -ginkgo.v; "+
+			"  else "+
+			"     TEST_WORK=_work REPO_ROOT=. /tmp/test.test;"+
+			"  fi;"+
+			"fi",
 		pmem.Namespace, pmem.Name, "pmem-driver")
 }

test/test.make

@@ -185,46 +185,54 @@ _work/.setupcfssl-stamp:
 # Build gocovmerge at a certain revision. Depends on go >= 1.11
 # because we use module support.
 GOCOVMERGE_VERSION=b5bfa59ec0adc420475f97f89b58045c721d761c
-_work/gocovmerge-$(GOCOVMERGE_VERSION):
-	tmpdir=`mktemp -d` && \
-	trap 'rm -r $$tmpdir' EXIT && \
-	cd $$tmpdir && \
-	echo "module foo" >go.mod && \
-	go get github.com/wadey/gocovmerge@$(GOCOVMERGE_VERSION) && \
-	go build -o $(abspath $@) github.com/wadey/gocovmerge
-	ln -sf $(@F) _work/gocovmerge
-
-# This is a special target that runs unit and E2E testing and
-# combines the various cover profiles into one. To re-run testing,
-# remove the file or use "make coverage".
+_work/gocovmerge-$(GOCOVMERGE_VERSION): check-go-version-$(GO_BINARY)
+	GOBIN=`pwd`/_work go install github.com/wadey/gocovmerge@$(GOCOVMERGE_VERSION)
+	ln -sf gocovmerge  $@
+
+GOCOVER_COBERTURA_VERSION=aaee18c8195c3f2d90e5ef80ca918d265463842a
+_work/gocover-cobertura-$(GOCOVER_COBERTURA_VERSION): check-go-version-$(GO_BINARY)
+	GOBIN=`pwd`/_work go install github.com/t-yuki/gocover-cobertura@$(GOCOVER_COBERTURA_VERSION)
+	ln -sf gocover-cobertura $@
+
+# This is a special target that collects the coverage information from E2E
+# testing (which includes Go unit tests) and combines it into one file.
 #
-# We remove all pmem-csi-driver coverage files
-# before testing, restart the driver, and then collect all
-# files, including the ones written earlier by init containers.
-_work/coverage.out: _work/gocovmerge-$(GOCOVMERGE_VERSION)
-	$(MAKE) start
-	@ echo "removing old pmem-csi-driver coverage information from all nodes"
-	@ for ssh in _work/$(CLUSTER)/ssh.*; do for i in $$($$ssh ls /var/lib/pmem-csi-coverage/pmem-csi-driver* 2>/dev/null); do (set -x; $$ssh rm $$i); done; done
-	@ rm -rf _work/coverage
-	@ mkdir _work/coverage
-	@ go clean -testcache
-	$(subst go test,go test -coverprofile=$(abspath _work/coverage/unit.out) -covermode=atomic,$(RUN_TESTS))
-	$(RUN_E2E)
-	@ echo "killing pmem-csi-driver to flush coverage data"
-	@ for ssh in _work/$(CLUSTER)/ssh.*; do (set -x; $$ssh killall pmem-csi-driver); done
-	@ echo "waiting for all pods to restart"
-	@ while _work/$(CLUSTER)/ssh.0 kubectl get --no-headers pods | grep -q -v Running; do sleep 5; done
+# Beware that "make kustomize KUSTOMIZE_WITH_COVERAGE=true" must have been run
+# before starting tests. This replaces YAML files in "deploy" with versions
+# that support coverage.
+#
+# To ensure that the results of the current driver instance are included, it
+# gets removed first.
+.PHONY: _work/coverage.out
+_work/coverage/coverage.out: _work/gocovmerge-$(GOCOVMERGE_VERSION)
+	@ echo "removing PMEM-CSI to flush coverage data"
+	test/delete-deployment.sh
 	@ echo "collecting coverage data"
-	@ for ssh in _work/$(CLUSTER)/ssh.*; do for i in $$($$ssh ls /var/lib/pmem-csi-coverage/ 2>/dev/null); do (set -x; $$ssh cat /var/lib/pmem-csi-coverage/$$i) >_work/coverage/$$(echo $$ssh | sed -e 's;.*/ssh\.;;').$$i; done; done
-	$< _work/coverage/* >$@
+	@ rm -rf ${@D}
+	@ mkdir -p ${@D}
+	@ node=0; while true; do ssh=_work/$(CLUSTER)/ssh.$$node; if ! [ -e $$ssh ]; then break; fi; for i in $$($$ssh ls /var/lib/pmem-csi-coverage/ 2>/dev/null); do in=/var/lib/pmem-csi-coverage/$$i; out=${@D}/node-$$node.$$i; (set -x; $$ssh sudo cat $$in | tee $$out >/dev/null); done; node=$$((node + 1)); done
+	$< ${@D}/* >$@
 
-_work/coverage.html: _work/coverage.out
+_work/coverage/coverage.html: _work/coverage/coverage.out check-go-version-$(GO_BINARY)
 	$(GO) tool cover -html $< -o $@
 
-_work/coverage.txt: _work/coverage.out check-go-version-$(GO_BINARY)
+_work/coverage/coverage.txt: _work/coverage/coverage.out check-go-version-$(GO_BINARY)
 	$(GO) tool cover -func $< -o $@
 
+# Convert to Cobertura XML for Jenkins.
+_work/coverage/coverage.xml: _work/gocover-cobertura-$(GOCOVER_COBERTURA_VERSION) _work/coverage/coverage.out
+	$<  <_work/coverage/coverage.out >$@
+
+# This runs all steps for coverage profile collection.
+# The Jenkinsfile has its own rules for this.
 .PHONY: coverage
 coverage:
-	@ rm -rf _work/coverage.out
+	@ rm -rf _work/coverage
+	$(MAKE) kustomize KUSTOMIZE_WITH_COVERAGE=true"
+	$(MAKE) start
+	@ echo "removing old PMEM-CSI installation"
+	test/delete-deployment.sh
+	@ echo "removing old pmem-csi-driver coverage information from all nodes"
+	@ for ssh in _work/$(CLUSTER)/ssh.*; do $$ssh sudo rm -f /var/lib/pmem-csi-coverage/pmem-csi-driver* || exit 1; done
+	$(RUN_E2E)
 	$(MAKE) _work/coverage.txt _work/coverage.html