further improve conformance test binary

Signed-off-by: Tim Ramlot <[email protected]>

Makefile

@@ -165,18 +165,20 @@ $(BINDIR)/conformance.test: | $(NEEDS_GINKGO)
 test: test-unit-deps | $(NEEDS_GO) $(NEEDS_GOTESTSUM) ## Run unit tests.
 	$(GOTESTSUM) ./... -coverprofile cover.out
 
-# $(GOTESTSUM) ./internal/testsetups/simple/e2e/... -coverprofile cover.out -timeout 5m
-
 .PHONY: test-e2e
 test-e2e: test-e2e-deps | $(NEEDS_GOTESTSUM) $(NEEDS_GINKGO) $(BINDIR)/conformance.test ## Run e2e tests. This creates a Kind cluster, installs dependencies, deploys the issuer-lib and runs the E2E tests.
-
+	$(GOTESTSUM) ./internal/testsetups/simple/e2e/... -coverprofile cover.out -timeout 5m
 
-	kubectl apply -f internal/testsetups/simple/example/simple-cluster-issuer.yaml
+	kubectl create ns cm-conformance-test || true
+	kubectl -n cm-conformance-test apply -f internal/testsetups/simple/example/simple-issuer.yaml
+	kubectl -n cm-conformance-test apply -f internal/testsetups/simple/example/simple-cluster-issuer.yaml
 
 	$(GINKGO) -procs=10 run $(BINDIR)/conformance.test -- \
+		--namespace=cm-conformance-test \
+		--cm-issuers=testing.cert-manager.io/SimpleIssuer/simple-issuer \
 		--cm-issuers=testing.cert-manager.io/SimpleClusterIssuer/simple-cluster-issuer \
 		--k8s-issuers=simpleclusterissuers.testing.cert-manager.io/simple-cluster-issuer \
-		--unsupported-features=SaveCAToSecret \
+		--unsupported-features=SaveCAToSecret
 
 ##@ Build
 

api/v1alpha1/issuer_interface.go

@@ -35,7 +35,7 @@ type Issuer interface {
 	// issuer type for a Kubernetes CertificateSigningRequest resource based
 	// on the issuerName field. The value should be formatted as follows:
 	// "<issuer resource (plural)>.<issuer group>". For example, the value
-	// "simpleclusterissuers.issuer.cert-manager.io" will match all CSRs
-	// with an issuerName set to eg. "simpleclusterissuers.issuer.cert-manager.io/issuer1".
+	// "simpleclusterissuers.testing.cert-manager.io" will match all CSRs
+	// with an issuerName set to eg. "simpleclusterissuers.testing.cert-manager.io/issuer1".
 	GetIssuerTypeIdentifier() string
 }

conformance/certificates/suite.go

@@ -44,6 +44,10 @@ type Suite struct {
 	// created with this issuer reference.
 	IssuerRef cmmeta.ObjectReference
 
+	// Namespace is the namespace in which the Certificate resources will be
+	// created.
+	Namespace string
+
 	// DomainSuffix is a suffix used on all domain requests.
 	// This is useful when the issuer being tested requires special
 	// configuration for a set of domains in order for certificates to be
@@ -72,6 +76,10 @@ func (s *Suite) complete(f *framework.Framework) {
 		Fail("IssuerRef must be set")
 	}
 
+	if s.Namespace == "" {
+		Fail("Namespace must be set")
+	}
+
 	if s.DomainSuffix == "" {
 		s.DomainSuffix = "example.com"
 	}

conformance/certificates/tests.go

@@ -33,6 +33,7 @@ import (
 	metav1 "k8s.io/apimachinery/pkg/apis/meta/v1"
 	"k8s.io/apimachinery/pkg/types"
 	"k8s.io/client-go/util/retry"
+	"sigs.k8s.io/controller-runtime/pkg/client"
 
 	"conformance/framework"
 	"conformance/framework/helper/featureset"
@@ -49,7 +50,16 @@ import (
 // automatically called.
 func (s *Suite) Define() {
 	Describe("with issuer type "+s.Name, func() {
-		f := framework.NewFramework("certificates", s.KubeClientConfig)
+		f := framework.NewFramework(
+			"certificates",
+			s.KubeClientConfig,
+			s.Namespace,
+			[]client.Object{
+				&cmapi.Certificate{},
+				&cmapi.CertificateRequest{},
+				&corev1.Secret{},
+			},
+		)
 
 		sharedIPAddress := "127.0.0.1"
 
@@ -371,14 +381,26 @@ func (s *Suite) Define() {
 
 		defineTest := func(test testCase) {
 			s.it(f, test.name, func(ctx context.Context, issuerRef cmmeta.ObjectReference) {
+				randomTestID := e2eutil.RandStringRunes(10)
 				certificate := &cmapi.Certificate{
 					ObjectMeta: metav1.ObjectMeta{
-						Name:      "testcert",
-						Namespace: f.Namespace.Name,
+						Name:      "e2e-conformance-" + randomTestID,
+						Namespace: f.Namespace,
+						Labels: map[string]string{
+							f.CleanupLabel: "true",
+						},
+						Annotations: map[string]string{
+							"conformance.cert-manager.io/test-name": s.Name + " " + test.name,
+						},
 					},
 					Spec: cmapi.CertificateSpec{
-						SecretName: "testcert-tls",
+						SecretName: "e2e-conformance-tls-" + randomTestID,
 						IssuerRef:  issuerRef,
+						SecretTemplate: &cmapi.CertificateSecretTemplate{
+							Labels: map[string]string{
+								f.CleanupLabel: "true",
+							},
+						},
 					},
 				}
 
@@ -407,15 +429,27 @@ func (s *Suite) Define() {
 		}
 
 		s.it(f, "should issue another certificate with the same private key if the existing certificate and CertificateRequest are deleted", func(ctx context.Context, issuerRef cmmeta.ObjectReference) {
+			randomTestID := e2eutil.RandStringRunes(10)
 			testCertificate := &cmapi.Certificate{
 				ObjectMeta: metav1.ObjectMeta{
-					Name:      "testcert",
-					Namespace: f.Namespace.Name,
+					Name:      "e2e-conformance-" + randomTestID,
+					Namespace: f.Namespace,
+					Labels: map[string]string{
+						f.CleanupLabel: "true",
+					},
+					Annotations: map[string]string{
+						"conformance.cert-manager.io/test-name": s.Name + " should issue another certificate with the same private key if the existing certificate and CertificateRequest are deleted",
+					},
 				},
 				Spec: cmapi.CertificateSpec{
-					SecretName: "testcert-tls",
+					SecretName: "e2e-conformance-tls-" + randomTestID,
 					DNSNames:   []string{e2eutil.RandomSubdomain(s.DomainSuffix)},
 					IssuerRef:  issuerRef,
+					SecretTemplate: &cmapi.CertificateSecretTemplate{
+						Labels: map[string]string{
+							f.CleanupLabel: "true",
+						},
+					},
 				},
 			}
 			By("Creating a Certificate")
@@ -431,7 +465,7 @@ func (s *Suite) Define() {
 			Expect(err).NotTo(HaveOccurred())
 
 			By("Deleting existing certificate data in Secret")
-			sec, err := f.KubeClientSet.CoreV1().Secrets(f.Namespace.Name).
+			sec, err := f.KubeClientSet.CoreV1().Secrets(f.Namespace).
 				Get(ctx, testCertificate.Spec.SecretName, metav1.GetOptions{})
 			Expect(err).NotTo(HaveOccurred(), "failed to get secret containing signed certificate key pair data")
 
@@ -442,11 +476,11 @@ func (s *Suite) Define() {
 
 			sec.Data[corev1.TLSCertKey] = []byte{}
 
-			_, err = f.KubeClientSet.CoreV1().Secrets(f.Namespace.Name).Update(ctx, sec, metav1.UpdateOptions{})
+			_, err = f.KubeClientSet.CoreV1().Secrets(f.Namespace).Update(ctx, sec, metav1.UpdateOptions{})
 			Expect(err).NotTo(HaveOccurred(), "failed to update secret by deleting the signed certificate data")
 
 			By("Waiting for the Certificate to re-issue a certificate")
-			sec, err = f.Helper().WaitForSecretCertificateData(ctx, sec.Name, f.Namespace.Name, time.Minute*8)
+			sec, err = f.Helper().WaitForSecretCertificateData(ctx, sec.Name, f.Namespace, time.Minute*8)
 			Expect(err).NotTo(HaveOccurred(), "failed to wait for secret to have a valid 2nd certificate")
 
 			crtPEM2 := sec.Data[corev1.TLSCertKey]
@@ -463,15 +497,27 @@ func (s *Suite) Define() {
 		}, featureset.ReusePrivateKeyFeature, featureset.OnlySAN)
 
 		s.it(f, "should allow updating an existing certificate with a new DNS Name", func(ctx context.Context, issuerRef cmmeta.ObjectReference) {
+			randomTestID := e2eutil.RandStringRunes(10)
 			testCertificate := &cmapi.Certificate{
 				ObjectMeta: metav1.ObjectMeta{
-					Name:      "testcert",
-					Namespace: f.Namespace.Name,
+					Name:      "e2e-conformance-" + randomTestID,
+					Namespace: f.Namespace,
+					Labels: map[string]string{
+						f.CleanupLabel: "true",
+					},
+					Annotations: map[string]string{
+						"conformance.cert-manager.io/test-name": s.Name + " should allow updating an existing certificate with a new DNS Name",
+					},
 				},
 				Spec: cmapi.CertificateSpec{
-					SecretName: "testcert-tls",
+					SecretName: "e2e-conformance-tls-" + randomTestID,
 					DNSNames:   []string{e2eutil.RandomSubdomain(s.DomainSuffix)},
 					IssuerRef:  issuerRef,
+					SecretTemplate: &cmapi.CertificateSecretTemplate{
+						Labels: map[string]string{
+							f.CleanupLabel: "true",
+						},
+					},
 				},
 			}
 			validations := validation.CertificateSetForUnsupportedFeatureSet(s.UnsupportedFeatures)

conformance/certificatesigningrequests/tests.go

@@ -29,6 +29,7 @@ import (
 	corev1 "k8s.io/api/core/v1"
 	metav1 "k8s.io/apimachinery/pkg/apis/meta/v1"
 	"k8s.io/utils/pointer"
+	"sigs.k8s.io/controller-runtime/pkg/client"
 
 	"conformance/framework"
 	"conformance/framework/helper/featureset"
@@ -49,7 +50,14 @@ import (
 // they are not active, these tests will fail.
 func (s *Suite) Define() {
 	Describe("CertificateSigningRequest with issuer type "+s.Name, func() {
-		f := framework.NewFramework("certificatesigningrequests", s.KubeClientConfig)
+		f := framework.NewFramework(
+			"certificatesigningrequests",
+			s.KubeClientConfig,
+			"",
+			[]client.Object{
+				&certificatesv1.CertificateSigningRequest{},
+			},
+		)
 
 		sharedIPAddress := "127.0.0.1"
 		sharedURI, err := url.Parse("spiffe://cluster.local/ns/sandbox/sa/foo")
@@ -435,17 +443,33 @@ func (s *Suite) Define() {
 			},
 		}
 
+		addAnnotation := func(annotations map[string]string, key, value string) map[string]string {
+			if annotations == nil {
+				annotations = map[string]string{}
+			}
+			annotations[key] = value
+			return annotations
+		}
+
 		defineTest := func(test testCase) {
 			s.it(f, test.name, func(ctx context.Context, signerName string) {
 				// Generate request CSR
 				csr, key, err := gen.CSR(test.keyAlgo, test.csrModifiers...)
 				Expect(err).NotTo(HaveOccurred())
 
 				// Create CertificateSigningRequest
+				randomTestID := e2eutil.RandStringRunes(10)
 				kubeCSR := &certificatesv1.CertificateSigningRequest{
 					ObjectMeta: metav1.ObjectMeta{
-						GenerateName: "e2e-conformance-",
-						Annotations:  test.kubeCSRAnnotations,
+						Name: "e2e-conformance-" + randomTestID,
+						Labels: map[string]string{
+							f.CleanupLabel: "true",
+						},
+						Annotations: addAnnotation(
+							test.kubeCSRAnnotations,
+							"conformance.cert-manager.io/test-name",
+							s.Name+" "+test.name,
+						),
 					},
 					Spec: certificatesv1.CertificateSigningRequestSpec{
 						Request:           csr,
@@ -458,9 +482,6 @@ func (s *Suite) Define() {
 				// Create the request, and delete at the end of the test
 				By("Creating a CertificateSigningRequest")
 				Expect(f.CRClient.Create(ctx, kubeCSR)).NotTo(HaveOccurred())
-				DeferCleanup(func(ctx context.Context) {
-					Expect(f.CRClient.Delete(ctx, kubeCSR)).NotTo(HaveOccurred())
-				})
 
 				// Approve the request for testing, so that cert-manager may sign the
 				// request.

conformance/conformance_test.go

@@ -26,12 +26,13 @@ func (i *arrayFlags) Set(value string) error {
 	return nil
 }
 
+var namespace string
 var unsupportedFeatures arrayFlags
-
 var cmIssuerReferences arrayFlags
 var k8sIssuerReferences arrayFlags
 
 func init() {
+	flag.StringVar(&namespace, "namespace", "", "list of issuer references to use for conformance tests")
 	flag.Var(&unsupportedFeatures, "unsupported-features", "list of features that are not supported by this invocation of the test suite")
 	flag.Var(&cmIssuerReferences, "cm-issuers", "list of issuer references to use for conformance tests")
 	flag.Var(&k8sIssuerReferences, "k8s-issuers", "list of issuer references to use for conformance tests")
@@ -71,6 +72,7 @@ func TestConformance(t *testing.T) {
 		(&certificates.Suite{
 			KubeClientConfig:    restConfig,
 			Name:                ref,
+			Namespace:           namespace,
 			IssuerRef:           parseCMReference(g, ref),
 			UnsupportedFeatures: unsupportedFeatureSet,
 		}).Define()