use Knative certificates for Serving encryption

cmd/activator/main.go

@@ -167,8 +167,11 @@ func main() {
 	// At this moment activator with TLS does not disable HTTP.
 	// See also https://github.com/knative/serving/issues/12808.
 	if tlsEnabled {
-		logger.Info("Knative Internal TLS is enabled")
-		certCache = certificate.NewCertCache(ctx)
+		logger.Info("Knative system-internal-tls is enabled")
+		certCache, err = certificate.NewCertCache(ctx)
+		if err != nil {
+			logger.Fatalw("Failed to create certificate cache", zap.Error(err))
+		}
 		transport = pkgnet.NewProxyAutoTLSTransport(env.MaxIdleProxyConns, env.MaxIdleProxyConnsPerHost, certCache.TLSContext())
 	}
 

cmd/controller/main.go

@@ -20,7 +20,6 @@ import (
 	// The set of controllers this controller process runs.
 	"flag"
 
-	certificate "knative.dev/networking/pkg/certificates/reconciler"
 	"knative.dev/pkg/reconciler"
 	"knative.dev/pkg/signals"
 	"knative.dev/serving/pkg/reconciler/configuration"
@@ -32,18 +31,11 @@ import (
 	"knative.dev/serving/pkg/reconciler/serverlessservice"
 	"knative.dev/serving/pkg/reconciler/service"
 
-	// This defines the shared main for injected controllers.
-	filteredFactory "knative.dev/pkg/client/injection/kube/informers/factory/filtered"
 	"knative.dev/pkg/injection"
 	"knative.dev/pkg/injection/sharedmain"
-	"knative.dev/serving/pkg/networking"
 	"knative.dev/serving/pkg/reconciler/domainmapping"
 )
 
-const (
-	secretLabelNamePostfix = "-ctrl"
-)
-
 var ctors = []injection.ControllerConstructor{
 	configuration.NewController,
 	labeler.NewController,
@@ -53,7 +45,6 @@ var ctors = []injection.ControllerConstructor{
 	service.NewController,
 	gc.NewController,
 	nscert.NewController,
-	certificate.NewControllerFactory(networking.ServingCertName),
 	domainmapping.NewController,
 }
 
@@ -62,7 +53,5 @@ func main() {
 		"reconciliation-timeout", reconciler.DefaultTimeout,
 		"The amount of time to give each reconciliation of a resource to complete before its context is canceled.")
 
-	labelName := networking.ServingCertName + secretLabelNamePostfix
-	ctx := filteredFactory.WithSelectors(signals.NewContext(), labelName)
-	sharedmain.MainWithContext(ctx, "controller", ctors...)
+	sharedmain.MainWithContext(signals.NewContext(), "controller", ctors...)
 }

go.mod

@@ -33,9 +33,9 @@ require (
 	k8s.io/kube-openapi v0.0.0-20230501164219-8b0f38b5fd1f
 	k8s.io/utils v0.0.0-20230209194617-a36077c30491
 	knative.dev/caching v0.0.0-20231101191025-c6425778e5ba
-	knative.dev/hack v0.0.0-20231102183416-0d99b7ee9d63
+	knative.dev/hack v0.0.0-20231107173840-883479423aaa
 	knative.dev/networking v0.0.0-20231103063604-18529fd26a8b
-	knative.dev/pkg v0.0.0-20231106073528-acf0a2d2dea9
+	knative.dev/pkg v0.0.0-20231107094615-5c9b7a8d8265
 	sigs.k8s.io/yaml v1.4.0
 )
 
@@ -156,3 +156,5 @@ require (
 
 // TODO: https://github.com/knative/serving/issues/14597
 replace github.com/gorilla/websocket => github.com/gorilla/websocket v1.5.0
+
+replace knative.dev/networking => /Users/rlehmann/code/knative/networking

go.sum

@@ -928,12 +928,10 @@ k8s.io/utils v0.0.0-20230209194617-a36077c30491 h1:r0BAOLElQnnFhE/ApUsg3iHdVYYPB
 k8s.io/utils v0.0.0-20230209194617-a36077c30491/go.mod h1:OLgZIPagt7ERELqWJFomSt595RzquPNLL48iOWgYOg0=
 knative.dev/caching v0.0.0-20231101191025-c6425778e5ba h1:wMfEfoiu+yfpKG79k9MUhY4ww8p3YiuqQOt0QfgqMoE=
 knative.dev/caching v0.0.0-20231101191025-c6425778e5ba/go.mod h1:36UniA7tdm8pj9dPAt44dneazLbcQqraNSftnOUYRks=
-knative.dev/hack v0.0.0-20231102183416-0d99b7ee9d63 h1:QNbIbsep8jreNCXj3EFrsw0RtsrmMZw7yxYyUG5AILs=
-knative.dev/hack v0.0.0-20231102183416-0d99b7ee9d63/go.mod h1:yk2OjGDsbEnQjfxdm0/HJKS2WqTLEFg/N6nUs6Rqx3Q=
-knative.dev/networking v0.0.0-20231103063604-18529fd26a8b h1:bimYPtsVmXBGjA0dARsoyNFKtREIVceVScnKdsHmqjo=
-knative.dev/networking v0.0.0-20231103063604-18529fd26a8b/go.mod h1:XKuKrS5QCQ3LkPeOYBZqyUxseBIBLpujxttejIBjoCk=
-knative.dev/pkg v0.0.0-20231106073528-acf0a2d2dea9 h1:9JQb0ETlOXxtXb6zeJuvjcdsI5BQ83Rv1PEzIzC8B8U=
-knative.dev/pkg v0.0.0-20231106073528-acf0a2d2dea9/go.mod h1:jCNm6Y66ArM8uDqpHwRwW7LuMFn79Q3KIn5L1R/WfI4=
+knative.dev/hack v0.0.0-20231107173840-883479423aaa h1:XVFqW2a8xvyo231TbVSD3i+580pVej9LbTSmYex6d1g=
+knative.dev/hack v0.0.0-20231107173840-883479423aaa/go.mod h1:yk2OjGDsbEnQjfxdm0/HJKS2WqTLEFg/N6nUs6Rqx3Q=
+knative.dev/pkg v0.0.0-20231107094615-5c9b7a8d8265 h1:wFDUSmvSQF48tUCIUIFKMOxq9jpV+vXf5l+RZYxYyt4=
+knative.dev/pkg v0.0.0-20231107094615-5c9b7a8d8265/go.mod h1:P3m1Mg/FJjmr9oFHfWcoUbJLSlBi/hgwakobPxyqrZ4=
 pgregory.net/rapid v1.1.0 h1:CMa0sjHSru3puNx+J0MIAuiiEV4N0qj8/cMWGBBCsjw=
 rsc.io/binaryregexp v0.2.0/go.mod h1:qTv7/COck+e2FymRvadv62gMdZztPaShugOCi3I+8D8=
 rsc.io/quote/v3 v3.1.0/go.mod h1:yEA65RcK8LyAZtP9Kv3t0HmxON59tX3rD+tICJqUlj0=

pkg/activator/certificate/cache.go

@@ -21,6 +21,7 @@ import (
 	"crypto/tls"
 	"crypto/x509"
 	"encoding/pem"
+	"fmt"
 	"sync"
 
 	"go.uber.org/zap"
@@ -47,8 +48,8 @@ type CertCache struct {
 	certificatesMux sync.RWMutex
 }
 
-// NewCertCache starts secretInformer.
-func NewCertCache(ctx context.Context) *CertCache {
+// NewCertCache creates and starts the certificate cache that watches Activators certificate.
+func NewCertCache(ctx context.Context) (*CertCache, error) {
 	secretInformer := secretinformer.Get(ctx)
 
 	cr := &CertCache{
@@ -58,8 +59,8 @@ func NewCertCache(ctx context.Context) *CertCache {
 
 	secret, err := cr.secretInformer.Lister().Secrets(system.Namespace()).Get(netcfg.ServingRoutingCertName)
 	if err != nil {
-		cr.logger.Warnw("failed to get secret", zap.Error(err))
-		return nil
+		return nil, fmt.Errorf("failed to get activator certificate, secret %s/%s was not found: %w. Enabling system-internal-tls requires the secret to be present and populated with a valid certificate and CA",
+			system.Namespace(), netcfg.ServingRoutingCertName, err)
 	}
 
 	cr.updateCache(secret)
@@ -72,7 +73,7 @@ func NewCertCache(ctx context.Context) *CertCache {
 		},
 	})
 
-	return cr
+	return cr, nil
 }
 
 func (cr *CertCache) handleCertificateAdd(added interface{}) {

pkg/apis/serving/v1/route_lifecycle.go

@@ -195,11 +195,6 @@ const (
 	// RouteConditionCertificateProvisioned condition when it is set to True
 	// because external-domain-tls was not enabled.
 	ExternalDomainTLSNotEnabledMessage = "external-domain-tls is not enabled"
-
-	// TLSNotEnabledForClusterLocalMessage is the message which is set on the
-	// RouteConditionCertificateProvisioned condition when it is set to True
-	// because the domain is cluster-local.
-	TLSNotEnabledForClusterLocalMessage = "TLS is not enabled for cluster-local"
 )
 
 // MarkTLSNotEnabled sets RouteConditionCertificateProvisioned to true when

pkg/queue/certificate/watcher_test.go

@@ -17,9 +17,15 @@ limitations under the License.
 package certificate
 
 import (
+	"bytes"
 	"context"
+	"crypto/rand"
+	"crypto/rsa"
 	"crypto/tls"
 	"crypto/x509"
+	"crypto/x509/pkix"
+	"encoding/pem"
+	"math/big"
 	"os"
 	"testing"
 	"time"
@@ -94,26 +100,47 @@ func TestCertificateRotation(t *testing.T) {
 }
 
 func createAndSaveCertificate(san, dir string) error {
-	ca, err := certificates.CreateCACerts(1 * time.Hour)
-	if err != nil {
-		return err
+	cert := &x509.Certificate{
+		SerialNumber: big.NewInt(2019),
+		Subject: pkix.Name{
+			Organization: []string{"Knative"},
+		},
+		NotBefore:             time.Now(),
+		NotAfter:              time.Now().AddDate(10, 0, 0),
+		IsCA:                  false,
+		ExtKeyUsage:           []x509.ExtKeyUsage{x509.ExtKeyUsageClientAuth, x509.ExtKeyUsageServerAuth},
+		KeyUsage:              x509.KeyUsageDigitalSignature | x509.KeyUsageCertSign,
+		BasicConstraintsValid: true,
 	}
+	cert.DNSNames = []string{san}
 
-	caCert, caKey, err := ca.Parse()
+	pk, err := rsa.GenerateKey(rand.Reader, 4096)
 	if err != nil {
 		return err
 	}
 
-	cert, err := certificates.CreateCert(caKey, caCert, 1*time.Hour, san)
+	certBytes, err := x509.CreateCertificate(rand.Reader, cert, cert, &pk.PublicKey, pk)
 	if err != nil {
 		return err
 	}
 
-	if err := os.WriteFile(dir+"/"+certificates.CertName, cert.CertBytes(), 0644); err != nil {
+	caPEM := new(bytes.Buffer)
+	pem.Encode(caPEM, &pem.Block{
+		Type:  "CERTIFICATE",
+		Bytes: certBytes,
+	})
+
+	caPrivKeyPEM := new(bytes.Buffer)
+	pem.Encode(caPrivKeyPEM, &pem.Block{
+		Type:  "RSA PRIVATE KEY",
+		Bytes: x509.MarshalPKCS1PrivateKey(pk),
+	})
+
+	if err := os.WriteFile(dir+"/"+certificates.CertName, caPEM.Bytes(), 0644); err != nil {
 		return err
 	}
 
-	return os.WriteFile(dir+"/"+certificates.PrivateKeyName, cert.PrivateKeyBytes(), 0644)
+	return os.WriteFile(dir+"/"+certificates.PrivateKeyName, caPrivKeyPEM.Bytes(), 0644)
 }
 
 func getSAN(c *tls.Certificate) (string, error) {

pkg/reconciler/accessor/networking/certificate.go

@@ -63,10 +63,15 @@ func ReconcileCertificate(ctx context.Context, owner kmeta.Accessor, desired *v1
 		return nil, kaccessor.NewAccessorError(
 			fmt.Errorf("owner: %s with Type %T does not own Certificate: %q", owner.GetName(), owner, cert.Name),
 			kaccessor.NotOwnResource)
-	} else if !equality.Semantic.DeepEqual(cert.Spec, desired.Spec) {
+	} else if !equality.Semantic.DeepEqual(cert.Spec, desired.Spec) ||
+		!equality.Semantic.DeepEqual(cert.Annotations, desired.Annotations) ||
+		!equality.Semantic.DeepEqual(cert.Labels, desired.Labels) {
+
 		// Don't modify the informers copy
 		existing := cert.DeepCopy()
 		existing.Spec = desired.Spec
+		existing.Annotations = desired.Annotations
+		existing.Labels = desired.Labels
 		cert, err = certAccessor.GetNetworkingClient().NetworkingV1alpha1().Certificates(existing.Namespace).Update(ctx, existing, metav1.UpdateOptions{})
 		if err != nil {
 			recorder.Eventf(owner, corev1.EventTypeWarning, "UpdateFailed",

pkg/reconciler/domainmapping/reconciler.go

@@ -228,7 +228,7 @@ func (r *Reconciler) tls(ctx context.Context, dm *v1beta1.DomainMapping) ([]netv
 	}
 	if cert.IsReady() {
 		dm.Status.MarkCertificateReady(cert.Name)
-		return []netv1alpha1.IngressTLS{routeresources.MakeIngressTLS(cert, desiredCert.Spec.DNSNames)}, nil, nil
+		return []netv1alpha1.IngressTLS{routeresources.MakeIngressTLS(cert, desiredCert.Spec.DNSNames, netv1alpha1.IngressVisibilityExternalIP)}, nil, nil
 	}
 	if config.FromContext(ctx).Network.HTTPProtocol == netcfg.HTTPEnabled {
 		// When httpProtocol is enabled, downgrade http scheme.

pkg/reconciler/domainmapping/resources/certificate_test.go

@@ -22,6 +22,7 @@ import (
 	"github.com/google/go-cmp/cmp"
 	corev1 "k8s.io/api/core/v1"
 	metav1 "k8s.io/apimachinery/pkg/apis/meta/v1"
+	"knative.dev/networking/pkg/config"
 
 	"knative.dev/networking/pkg/apis/networking"
 	networkingv1alpha1 "knative.dev/networking/pkg/apis/networking/v1alpha1"
@@ -58,7 +59,8 @@ func TestMakeCertificate(t *testing.T) {
 				Namespace:   "the-namespace",
 				Annotations: map[string]string{networking.CertificateClassAnnotationKey: certClass},
 				Labels: map[string]string{
-					serving.DomainMappingUIDLabelKey: "mapping.com",
+					serving.DomainMappingUIDLabelKey:   "mapping.com",
+					networking.CertificateTypeLabelKey: string(config.CertificateExternalDomain),
 				},
 			},
 			Spec: networkingv1alpha1.CertificateSpec{
@@ -96,7 +98,8 @@ func TestMakeCertificate(t *testing.T) {
 					"others":                                 "kept",
 				},
 				Labels: map[string]string{
-					serving.DomainMappingUIDLabelKey: "mapping.com",
+					serving.DomainMappingUIDLabelKey:   "mapping.com",
+					networking.CertificateTypeLabelKey: string(config.CertificateExternalDomain),
 				},
 			},
 			Spec: networkingv1alpha1.CertificateSpec{

pkg/reconciler/domainmapping/table_test.go

@@ -967,6 +967,7 @@ func TestReconcileTLSEnabled(t *testing.T) {
 					},
 					Labels: map[string]string{
 						serving.DomainMappingUIDLabelKey: "becomes.ready.run",
+						netapi.CertificateTypeLabelKey:   string(netcfg.CertificateExternalDomain),
 					},
 				},
 				Spec: netv1alpha1.CertificateSpec{
@@ -997,6 +998,7 @@ func TestReconcileTLSEnabled(t *testing.T) {
 					Hosts:           []string{"becomes.ready.run"},
 					SecretName:      "becomes.ready.run",
 					SecretNamespace: "default",
+					Visibility:      netv1alpha1.IngressVisibilityExternalIP,
 				})),
 		}},
 		WantPatches: []clientgotesting.PatchActionImpl{
@@ -1114,6 +1116,7 @@ func TestReconcileTLSEnabled(t *testing.T) {
 					},
 					Labels: map[string]string{
 						serving.DomainMappingUIDLabelKey: "challenged.com",
+						netapi.CertificateTypeLabelKey:   string(netcfg.CertificateExternalDomain),
 					},
 				},
 				Spec: netv1alpha1.CertificateSpec{

pkg/reconciler/revision/controller.go

@@ -26,6 +26,8 @@ import (
 	"golang.org/x/time/rate"
 	cachingclient "knative.dev/caching/pkg/client/injection/client"
 	imageinformer "knative.dev/caching/pkg/client/injection/informers/caching/v1alpha1/image"
+	networkingclient "knative.dev/networking/pkg/client/injection/client"
+	certificateinformer "knative.dev/networking/pkg/client/injection/informers/networking/v1alpha1/certificate"
 	"knative.dev/pkg/changeset"
 	kubeclient "knative.dev/pkg/client/injection/kube/client"
 	deploymentinformer "knative.dev/pkg/client/injection/kube/informers/apps/v1/deployment"
@@ -73,15 +75,18 @@ func newControllerWithOptions(
 	deploymentInformer := deploymentinformer.Get(ctx)
 	imageInformer := imageinformer.Get(ctx)
 	paInformer := painformer.Get(ctx)
+	certificateInformer := certificateinformer.Get(ctx)
 
 	c := &Reconciler{
-		kubeclient:    kubeclient.Get(ctx),
-		client:        servingclient.Get(ctx),
-		cachingclient: cachingclient.Get(ctx),
+		kubeclient:       kubeclient.Get(ctx),
+		client:           servingclient.Get(ctx),
+		networkingclient: networkingclient.Get(ctx),
+		cachingclient:    cachingclient.Get(ctx),
 
 		podAutoscalerLister: paInformer.Lister(),
 		imageLister:         imageInformer.Lister(),
 		deploymentLister:    deploymentInformer.Lister(),
+		certificateLister:   certificateInformer.Lister(),
 	}
 
 	impl := revisionreconciler.NewImpl(ctx, c, func(impl *controller.Impl) controller.Options {