create binary for conformance tests

Signed-off-by: Tim Ramlot <[email protected]>

Makefile

@@ -157,15 +157,26 @@ test-e2e-deps: TEST_MODE := E2E
 test-e2e-deps: DOCKER_REGISTRY := kind.local
 test-e2e-deps: e2e-setup docker-build test-e2e-envs install
 
+$(BINDIR)/conformance.test: | $(NEEDS_GINKGO)
+	$(GINKGO) build ./conformance/ --trimpath --cover --require-suite
+	mv ./conformance/conformance.test $@
+
 .PHONY: test
 test: test-unit-deps | $(NEEDS_GO) $(NEEDS_GOTESTSUM) ## Run unit tests.
 	$(GOTESTSUM) ./... -coverprofile cover.out
 
+# $(GOTESTSUM) ./internal/testsetups/simple/e2e/... -coverprofile cover.out -timeout 5m
+
 .PHONY: test-e2e
-test-e2e: test-e2e-deps | $(NEEDS_GOTESTSUM) $(NEEDS_GINKGO) ## Run e2e tests. This creates a Kind cluster, installs dependencies, deploys the issuer-lib and runs the E2E tests.
-	$(GOTESTSUM) ./internal/testsetups/simple/e2e/... -coverprofile cover.out -timeout 5m
+test-e2e: test-e2e-deps | $(NEEDS_GOTESTSUM) $(NEEDS_GINKGO) $(BINDIR)/conformance.test ## Run e2e tests. This creates a Kind cluster, installs dependencies, deploys the issuer-lib and runs the E2E tests.
+
+
+	kubectl apply -f internal/testsetups/simple/example/simple-cluster-issuer.yaml
 
-	$(GINKGO) ./internal/testsetups/simple/e2e/conformance/...
+	$(GINKGO) -procs=10 run $(BINDIR)/conformance.test -- \
+		--cm-issuers=testing.cert-manager.io/SimpleClusterIssuer/simple-cluster-issuer \
+		--k8s-issuers=simpleclusterissuers.testing.cert-manager.io/simple-cluster-issuer \
+		--unsupported-features=SaveCAToSecret \
 
 ##@ Build
 

conformance/certificates/suite.go

@@ -22,8 +22,8 @@ import (
 	cmmeta "github.com/cert-manager/cert-manager/pkg/apis/meta/v1"
 	"k8s.io/client-go/rest"
 
-	"github.com/cert-manager/issuer-lib/conformance/framework"
-	"github.com/cert-manager/issuer-lib/conformance/framework/helper/featureset"
+	"conformance/framework"
+	"conformance/framework/helper/featureset"
 
 	. "github.com/onsi/ginkgo/v2"
 )
@@ -39,18 +39,10 @@ type Suite struct {
 	// This field must be provided.
 	Name string
 
-	// CreateIssuerFunc is a function that provisions a new issuer resource and
-	// returns an ObjectReference to that Issuer that will be used as the
-	// IssuerRef on Certificate resources that this suite creates.
-	// This field must be provided.
-	CreateIssuerFunc func(*framework.Framework, context.Context) cmmeta.ObjectReference
-
-	// DeleteIssuerFunc is a function that is run after the test has completed
-	// in order to clean up resources created for a test (e.g. the resources
-	// created in CreateIssuerFunc).
-	// This function will be run regardless whether the test passes or fails.
-	// If not specified, this function will be skipped.
-	DeleteIssuerFunc func(*framework.Framework, context.Context, cmmeta.ObjectReference)
+	// IssuerRef is reference to the issuer resource that this test suite will
+	// test against. All Certificate resources created by this suite will be
+	// created with this issuer reference.
+	IssuerRef cmmeta.ObjectReference
 
 	// DomainSuffix is a suffix used on all domain requests.
 	// This is useful when the issuer being tested requires special
@@ -76,8 +68,8 @@ func (s *Suite) complete(f *framework.Framework) {
 		Fail("Name must be set")
 	}
 
-	if s.CreateIssuerFunc == nil {
-		Fail("CreateIssuerFunc must be set")
+	if s.IssuerRef != (cmmeta.ObjectReference{}) && s.IssuerRef.Name == "" {
+		Fail("IssuerRef must be set")
 	}
 
 	if s.DomainSuffix == "" {
@@ -92,20 +84,12 @@ func (s *Suite) complete(f *framework.Framework) {
 }
 
 // it is called by the tests to in Define() to setup and run the test
-func (s *Suite) it(f *framework.Framework, name string, fn func(cmmeta.ObjectReference), requiredFeatures ...featureset.Feature) {
+func (s *Suite) it(f *framework.Framework, name string, fn func(context.Context, cmmeta.ObjectReference), requiredFeatures ...featureset.Feature) {
 	if !s.checkFeatures(requiredFeatures...) {
 		return
 	}
 	It(name, func(ctx context.Context) {
-		By("Creating an issuer resource")
-		issuerRef := s.CreateIssuerFunc(f, ctx)
-		defer func() {
-			if s.DeleteIssuerFunc != nil {
-				By("Cleaning up the issuer resource")
-				s.DeleteIssuerFunc(f, ctx, issuerRef)
-			}
-		}()
-		fn(issuerRef)
+		fn(ctx, s.IssuerRef)
 	})
 }
 

conformance/certificates/tests.go

@@ -34,11 +34,11 @@ import (
 	"k8s.io/apimachinery/pkg/types"
 	"k8s.io/client-go/util/retry"
 
-	"github.com/cert-manager/issuer-lib/conformance/framework"
-	"github.com/cert-manager/issuer-lib/conformance/framework/helper/featureset"
-	"github.com/cert-manager/issuer-lib/conformance/framework/helper/validation"
-	"github.com/cert-manager/issuer-lib/conformance/framework/helper/validation/certificates"
-	e2eutil "github.com/cert-manager/issuer-lib/conformance/util"
+	"conformance/framework"
+	"conformance/framework/helper/featureset"
+	"conformance/framework/helper/validation"
+	"conformance/framework/helper/validation/certificates"
+	e2eutil "conformance/util"
 
 	. "github.com/onsi/ginkgo/v2"
 	. "github.com/onsi/gomega"
@@ -49,7 +49,6 @@ import (
 // automatically called.
 func (s *Suite) Define() {
 	Describe("with issuer type "+s.Name, func() {
-		ctx := context.Background()
 		f := framework.NewFramework("certificates", s.KubeClientConfig)
 
 		sharedIPAddress := "127.0.0.1"
@@ -371,7 +370,7 @@ func (s *Suite) Define() {
 		}
 
 		defineTest := func(test testCase) {
-			s.it(f, test.name, func(issuerRef cmmeta.ObjectReference) {
+			s.it(f, test.name, func(ctx context.Context, issuerRef cmmeta.ObjectReference) {
 				certificate := &cmapi.Certificate{
 					ObjectMeta: metav1.ObjectMeta{
 						Name:      "testcert",
@@ -393,7 +392,7 @@ func (s *Suite) Define() {
 				Expect(err).NotTo(HaveOccurred())
 
 				By("Waiting for the Certificate to be issued...")
-				certificate, err = f.Helper().WaitForCertificateReadyAndDoneIssuing(ctx, certificate, time.Minute*8)
+				certificate, err = f.Helper().WaitForCertificateReadyAndDoneIssuing(ctx, certificate.Name, certificate.Namespace, certificate.Generation, time.Minute*8)
 				Expect(err).NotTo(HaveOccurred())
 
 				By("Validating the issued Certificate...")
@@ -407,7 +406,7 @@ func (s *Suite) Define() {
 			defineTest(tc)
 		}
 
-		s.it(f, "should issue another certificate with the same private key if the existing certificate and CertificateRequest are deleted", func(issuerRef cmmeta.ObjectReference) {
+		s.it(f, "should issue another certificate with the same private key if the existing certificate and CertificateRequest are deleted", func(ctx context.Context, issuerRef cmmeta.ObjectReference) {
 			testCertificate := &cmapi.Certificate{
 				ObjectMeta: metav1.ObjectMeta{
 					Name:      "testcert",
@@ -424,7 +423,7 @@ func (s *Suite) Define() {
 			Expect(err).NotTo(HaveOccurred())
 
 			By("Waiting for the Certificate to be issued...")
-			testCertificate, err = f.Helper().WaitForCertificateReadyAndDoneIssuing(ctx, testCertificate, time.Minute*8)
+			testCertificate, err = f.Helper().WaitForCertificateReadyAndDoneIssuing(ctx, testCertificate.Name, testCertificate.Namespace, testCertificate.Generation, time.Minute*8)
 			Expect(err).NotTo(HaveOccurred())
 
 			By("Validating the issued Certificate...")
@@ -447,7 +446,7 @@ func (s *Suite) Define() {
 			Expect(err).NotTo(HaveOccurred(), "failed to update secret by deleting the signed certificate data")
 
 			By("Waiting for the Certificate to re-issue a certificate")
-			sec, err = f.Helper().WaitForSecretCertificateData(ctx, f.Namespace.Name, sec.Name, time.Minute*8)
+			sec, err = f.Helper().WaitForSecretCertificateData(ctx, sec.Name, f.Namespace.Name, time.Minute*8)
 			Expect(err).NotTo(HaveOccurred(), "failed to wait for secret to have a valid 2nd certificate")
 
 			crtPEM2 := sec.Data[corev1.TLSCertKey]
@@ -463,7 +462,7 @@ func (s *Suite) Define() {
 			}
 		}, featureset.ReusePrivateKeyFeature, featureset.OnlySAN)
 
-		s.it(f, "should allow updating an existing certificate with a new DNS Name", func(issuerRef cmmeta.ObjectReference) {
+		s.it(f, "should allow updating an existing certificate with a new DNS Name", func(ctx context.Context, issuerRef cmmeta.ObjectReference) {
 			testCertificate := &cmapi.Certificate{
 				ObjectMeta: metav1.ObjectMeta{
 					Name:      "testcert",
@@ -482,7 +481,7 @@ func (s *Suite) Define() {
 			Expect(err).NotTo(HaveOccurred())
 
 			By("Waiting for the Certificate to be ready")
-			testCertificate, err = f.Helper().WaitForCertificateReadyAndDoneIssuing(ctx, testCertificate, time.Minute*8)
+			testCertificate, err = f.Helper().WaitForCertificateReadyAndDoneIssuing(ctx, testCertificate.Name, testCertificate.Namespace, testCertificate.Generation, time.Minute*8)
 			Expect(err).NotTo(HaveOccurred())
 
 			By("Sanity-check the issued Certificate")
@@ -507,7 +506,7 @@ func (s *Suite) Define() {
 			Expect(err).NotTo(HaveOccurred())
 
 			By("Waiting for the Certificate Ready condition to be updated")
-			testCertificate, err = f.Helper().WaitForCertificateReadyAndDoneIssuing(ctx, testCertificate, time.Minute*8)
+			testCertificate, err = f.Helper().WaitForCertificateReadyAndDoneIssuing(ctx, testCertificate.Name, testCertificate.Namespace, testCertificate.Generation, time.Minute*8)
 			Expect(err).NotTo(HaveOccurred())
 
 			By("Sanity-check the issued Certificate")

conformance/certificatesigningrequests/suite.go

@@ -18,13 +18,11 @@ package certificatesigningrequests
 
 import (
 	"context"
-	"crypto"
 
-	certificatesv1 "k8s.io/api/certificates/v1"
 	"k8s.io/client-go/rest"
 
-	"github.com/cert-manager/issuer-lib/conformance/framework"
-	"github.com/cert-manager/issuer-lib/conformance/framework/helper/featureset"
+	"conformance/framework"
+	"conformance/framework/helper/featureset"
 
 	. "github.com/onsi/ginkgo/v2"
 )
@@ -40,32 +38,10 @@ type Suite struct {
 	// This field must be provided.
 	Name string
 
-	// CreateIssuerFunc is a function that provisions a new issuer resource and
-	// returns an SignerName to that Issuer that will be used as the SignerName
-	// on CertificateSigningRequest resources that this suite creates.
-	// This field must be provided.
-	CreateIssuerFunc func(*framework.Framework, context.Context) string
-
-	// DeleteIssuerFunc is a function that is run after the test has completed
-	// in order to clean up resources created for a test (e.g. the resources
-	// created in CreateIssuerFunc).
-	// This function will be run regardless whether the test passes or fails.
-	// If not specified, this function will be skipped.
-	DeleteIssuerFunc func(*framework.Framework, context.Context, string)
-
-	// ProvisionFunc is a function that is run every test just before the
-	// CertificateSigningRequest is created within a test. This is used to
-	// provision or create any resources that are required by the Issuer to sign
-	// the CertificateSigningRequest. This could be for example to annotate the
-	// CertificateSigningRequest, or create a resource like a Secret needed for
-	// signing.
-	// If not specified, this function will be skipped.
-	ProvisionFunc func(*framework.Framework, context.Context, *certificatesv1.CertificateSigningRequest, crypto.Signer)
-
-	// DeProvisionFunc is run after every test. This is to be used to remove and
-	// clean-up any resources which may have been created by ProvisionFunc.
-	// If not specified, this function will be skipped.
-	DeProvisionFunc func(*framework.Framework, context.Context, *certificatesv1.CertificateSigningRequest)
+	// SignerName is the name of the signer that the conformance suite will test
+	// against. All CertificateSigningRequest resources created by this suite
+	// will be created with this signer name.
+	SignerName string
 
 	// DomainSuffix is a suffix used on all domain requests.
 	// This is useful when the issuer being tested requires special
@@ -91,8 +67,8 @@ func (s *Suite) complete(f *framework.Framework) {
 		Fail("Name must be set")
 	}
 
-	if s.CreateIssuerFunc == nil {
-		Fail("CreateIssuerFunc must be set")
+	if s.SignerName == "" {
+		Fail("SignerName must be set")
 	}
 
 	if s.DomainSuffix == "" {
@@ -112,15 +88,7 @@ func (s *Suite) it(f *framework.Framework, name string, fn func(context.Context,
 		return
 	}
 	It(name, func(ctx context.Context) {
-		By("Creating an issuer resource")
-		signerName := s.CreateIssuerFunc(f, ctx)
-		defer func() {
-			if s.DeleteIssuerFunc != nil {
-				By("Cleaning up the issuer resource")
-				s.DeleteIssuerFunc(f, ctx, signerName)
-			}
-		}()
-		fn(ctx, signerName)
+		fn(ctx, s.SignerName)
 	})
 }
 

conformance/certificatesigningrequests/tests.go

@@ -30,11 +30,11 @@ import (
 	metav1 "k8s.io/apimachinery/pkg/apis/meta/v1"
 	"k8s.io/utils/pointer"
 
-	"github.com/cert-manager/issuer-lib/conformance/framework"
-	"github.com/cert-manager/issuer-lib/conformance/framework/helper/featureset"
-	"github.com/cert-manager/issuer-lib/conformance/framework/helper/validation"
-	"github.com/cert-manager/issuer-lib/conformance/framework/helper/validation/certificatesigningrequests"
-	e2eutil "github.com/cert-manager/issuer-lib/conformance/util"
+	"conformance/framework"
+	"conformance/framework/helper/featureset"
+	"conformance/framework/helper/validation"
+	"conformance/framework/helper/validation/certificatesigningrequests"
+	e2eutil "conformance/util"
 
 	. "github.com/onsi/ginkgo/v2"
 	. "github.com/onsi/gomega"
@@ -455,26 +455,12 @@ func (s *Suite) Define() {
 					},
 				}
 
-				// Provision any resources needed for the request, or modify the
-				// request based on Issuer requirements
-				if s.ProvisionFunc != nil {
-					s.ProvisionFunc(f, ctx, kubeCSR, key)
-				}
-				// Ensure related resources are cleaned up at the end of the test
-				if s.DeProvisionFunc != nil {
-					defer s.DeProvisionFunc(f, ctx, kubeCSR)
-				}
-
 				// Create the request, and delete at the end of the test
 				By("Creating a CertificateSigningRequest")
 				Expect(f.CRClient.Create(ctx, kubeCSR)).NotTo(HaveOccurred())
-				defer func() {
-					// Create a new context with a timeout to prevent the deletion of the
-					// CertificateSigningRequest from blocking test completion.
-					deleteCtx, cancel := context.WithTimeout(context.Background(), time.Second*30)
-					defer cancel()
-					Expect(f.CRClient.Delete(deleteCtx, kubeCSR)).NotTo(HaveOccurred())
-				}()
+				DeferCleanup(func(ctx context.Context) {
+					Expect(f.CRClient.Delete(ctx, kubeCSR)).NotTo(HaveOccurred())
+				})
 
 				// Approve the request for testing, so that cert-manager may sign the
 				// request.