feat: support a CA bundle for the control plane connection

Makefile

@@ -30,7 +30,7 @@ KIND_NAME ?= apisix-ingress-cluster
 KIND_NODE_IMAGE ?= kindest/node:v1.32.2@sha256:f226345927d7e348497136874b6d207e0b32cc52154ad8323129352923a3142f
 
 DASHBOARD_VERSION ?= dev
-ADC_VERSION ?= 0.27.1
+ADC_VERSION ?= 0.29.0
 
 DIR := $(shell pwd)
 

api/adc/types.go

@@ -807,6 +807,11 @@ type Config struct {
 	TlsVerify   bool
 	BackendType string
 
+	// CaCert is a PEM-encoded CA certificate (or bundle) used to verify the
+	// control plane, in place of the system trust store. Only meaningful when
+	// TlsVerify is true.
+	CaCert string
+
 	// BypassCache makes the ADC server drop the in-memory baseline it holds for this
 	// cacheKey and re-derive it from the data plane before computing the diff. It is a
 	// per-request flag set on the sync path, not part of the translated configuration.
@@ -820,10 +825,12 @@ func (c Config) MarshalJSON() ([]byte, error) {
 		Name        string   `json:"name"`
 		ServerAddrs []string `json:"serverAddrs"`
 		TlsVerify   bool     `json:"tlsVerify"`
+		HasCaCert   bool     `json:"hasCaCert"`
 	}{
 		Name:        c.Name,
 		ServerAddrs: c.ServerAddrs,
 		TlsVerify:   c.TlsVerify,
+		HasCaCert:   c.CaCert != "",
 	})
 }
 

api/v1alpha1/gatewayproxy_types.go

@@ -136,11 +136,29 @@ type ControlPlaneProvider struct {
 	// +optional
 	TlsVerify *bool `json:"tlsVerify,omitempty"`
 
+	// CaCert specifies the CA certificate used to verify the control plane's TLS
+	// certificate, in place of the system trust store.
+	// Set it when the control plane uses a self-signed or private CA certificate.
+	// It has no effect when tlsVerify is false.
+	// +optional
+	CaCert *ControlPlaneCaCert `json:"caCert,omitempty"`
+
 	// Auth specifies the authentication configuration.
 	// +kubebuilder:validation:Required
 	Auth ControlPlaneAuth `json:"auth"`
 }
 
+// ControlPlaneCaCert defines the CA certificate used to verify the control plane.
+//
+// Only an inline value is supported today. A valueFrom source can be added
+// later without breaking existing resources.
+type ControlPlaneCaCert struct {
+	// Value sets the PEM-encoded CA certificate (or bundle) explicitly.
+	// +kubebuilder:validation:Required
+	// +kubebuilder:validation:XValidation:rule="self.contains('-----BEGIN CERTIFICATE-----')",message="value must be a PEM-encoded certificate"
+	Value string `json:"value"`
+}
+
 type ProviderService struct {
 	// Name is the name of the provider.
 	Name string `json:"name"`

config/crd-nocel/apisix.apache.org_v2.yaml

@@ -2273,6 +2273,20 @@ spec:
                         required:
                         - type
                         type: object
+                      caCert:
+                        description: |-
+                          CaCert specifies the CA certificate used to verify the control plane's TLS
+                          certificate, in place of the system trust store.
+                          Set it when the control plane uses a self-signed or private CA certificate.
+                          It has no effect when tlsVerify is false.
+                        properties:
+                          value:
+                            description: Value sets the PEM-encoded CA certificate
+                              (or bundle) explicitly.
+                            type: string
+                        required:
+                        - value
+                        type: object
                       endpoints:
                         description: Endpoints specifies the list of control plane
                           endpoints.

config/crd/bases/apisix.apache.org_gatewayproxies.yaml

@@ -120,6 +120,23 @@ spec:
                         - message: adminKey must be specified when type is AdminKey
                           rule: 'self.type == ''AdminKey'' ? has(self.adminKey) :
                             true'
+                      caCert:
+                        description: |-
+                          CaCert specifies the CA certificate used to verify the control plane's TLS
+                          certificate, in place of the system trust store.
+                          Set it when the control plane uses a self-signed or private CA certificate.
+                          It has no effect when tlsVerify is false.
+                        properties:
+                          value:
+                            description: Value sets the PEM-encoded CA certificate
+                              (or bundle) explicitly.
+                            type: string
+                            x-kubernetes-validations:
+                            - message: value must be a PEM-encoded certificate
+                              rule: self.contains('-----BEGIN CERTIFICATE-----')
+                        required:
+                        - value
+                        type: object
                       endpoints:
                         description: Endpoints specifies the list of control plane
                           endpoints.

docs/en/latest/reference/api-reference.md

@@ -297,6 +297,23 @@ ControlPlaneAuth defines the authentication configuration for control plane.
 | `adminKey` _[AdminKeyAuth](#adminkeyauth)_ | AdminKey specifies the admin key authentication configuration. |
 
 
+_Appears in:_
+- [ControlPlaneProvider](#controlplaneprovider)
+
+#### ControlPlaneCaCert
+
+
+ControlPlaneCaCert defines the CA certificate used to verify the control plane.<br /><br />
+Only an inline value is supported today. A valueFrom source can be added
+later without breaking existing resources.
+
+
+
+| Field | Description |
+| --- | --- |
+| `value` _string_ | Value sets the PEM-encoded CA certificate (or bundle) explicitly. |
+
+
 _Appears in:_
 - [ControlPlaneProvider](#controlplaneprovider)
 
@@ -313,6 +330,7 @@ ControlPlaneProvider defines configuration for control plane provider.
 | `endpoints` _string array_ | Endpoints specifies the list of control plane endpoints. |
 | `service` _[ProviderService](#providerservice)_ |  |
 | `tlsVerify` _boolean_ | TlsVerify specifies whether to verify the TLS certificate of the control plane. |
+| `caCert` _[ControlPlaneCaCert](#controlplanecacert)_ | CaCert specifies the CA certificate used to verify the control plane's TLS certificate, in place of the system trust store. Set it when the control plane uses a self-signed or private CA certificate. It has no effect when tlsVerify is false. |
 | `auth` _[ControlPlaneAuth](#controlplaneauth)_ | Auth specifies the authentication configuration. |
 
 

internal/adc/client/executor.go

@@ -84,7 +84,11 @@ type ADCServerOpts struct {
 	LabelSelector       map[string]string `json:"labelSelector,omitempty"`
 	IncludeResourceType []string          `json:"includeResourceType,omitempty"`
 	TlsSkipVerify       *bool             `json:"tlsSkipVerify,omitempty"`
-	CacheKey            string            `json:"cacheKey"`
+	// CaCert is the PEM-encoded CA certificate (or bundle) the ADC server verifies
+	// the control plane against. Older ADC servers ignore it, and omitempty keeps
+	// requests without a CA bundle byte for byte what they were.
+	CaCert   string `json:"caCert,omitempty"`
+	CacheKey string `json:"cacheKey"`
 	// BypassCache is only accepted by the /sync task of ADC >= 0.27.0. Both ADC task
 	// schemas reject unknown fields, so omitempty is what keeps every other request --
 	// /validate, and every sync that is not recovering from a rejection -- byte for byte
@@ -103,6 +107,7 @@ func (r ADCServerRequest) MarshalLog() any {
 		"labelSelector":       r.Task.Opts.LabelSelector,
 		"includeResourceType": r.Task.Opts.IncludeResourceType,
 		"tlsSkipVerify":       r.Task.Opts.TlsSkipVerify,
+		"hasCaCert":           r.Task.Opts.CaCert != "",
 		"cacheKey":            r.Task.Opts.CacheKey,
 		"config":              r.Task.Config.MarshalLog(),
 	}
@@ -244,7 +249,7 @@ func (e *HTTPADCExecutor) runHTTPSyncForSingleServer(ctx context.Context, server
 	}
 
 	// Build HTTP request
-	req, err := e.buildHTTPRequest(ctx, serverAddr, config, labels, types, resources, http.MethodPut, pathSync)
+	req, err := e.buildHTTPRequest(ctx, serverAddr, config, labels, types, resources, pathSync)
 	if err != nil {
 		return fmt.Errorf("failed to build HTTP request: %w", err)
 	}
@@ -278,7 +283,7 @@ func (e *HTTPADCExecutor) runHTTPValidateForSingleServer(ctx context.Context, se
 		return fmt.Errorf("failed to load resources from file %s: %w", filePath, err)
 	}
 
-	req, err := e.buildHTTPRequest(ctx, serverAddr, config, labels, types, resources, http.MethodPut, pathValidate)
+	req, err := e.buildHTTPRequest(ctx, serverAddr, config, labels, types, resources, pathValidate)
 	if err != nil {
 		return fmt.Errorf("failed to build validate request: %w", err)
 	}
@@ -349,7 +354,7 @@ func (e *HTTPADCExecutor) loadResourcesFromFile(filePath string) (*adctypes.Reso
 }
 
 // buildHTTPRequest builds the HTTP request for ADC Server
-func (e *HTTPADCExecutor) buildHTTPRequest(ctx context.Context, serverAddr string, config adctypes.Config, labels map[string]string, types []string, resources *adctypes.Resources, method string, path string) (*http.Request, error) {
+func (e *HTTPADCExecutor) buildHTTPRequest(ctx context.Context, serverAddr string, config adctypes.Config, labels map[string]string, types []string, resources *adctypes.Resources, path string) (*http.Request, error) {
 	// Prepare request body
 	tlsVerify := config.TlsVerify
 	bypassCache := path == pathSync && config.BypassCache
@@ -362,6 +367,7 @@ func (e *HTTPADCExecutor) buildHTTPRequest(ctx context.Context, serverAddr strin
 				LabelSelector:       labels,
 				IncludeResourceType: types,
 				TlsSkipVerify:       ptr.To(!tlsVerify),
+				CaCert:              config.CaCert,
 				CacheKey:            config.Name,
 				BypassCache:         bypassCache,
 			},
@@ -385,10 +391,11 @@ func (e *HTTPADCExecutor) buildHTTPRequest(ctx context.Context, serverAddr strin
 		"labelSelector", labels,
 		"includeResourceType", types,
 		"tlsSkipVerify", !tlsVerify,
+		"hasCaCert", config.CaCert != "",
 	)
 
 	// Create HTTP request
-	req, err := http.NewRequestWithContext(ctx, method, e.serverURL+path, bytes.NewBuffer(jsonData))
+	req, err := http.NewRequestWithContext(ctx, http.MethodPut, e.serverURL+path, bytes.NewBuffer(jsonData))
 	if err != nil {
 		return nil, fmt.Errorf("failed to create HTTP request: %w", err)
 	}

internal/adc/client/executor_test.go

@@ -22,7 +22,6 @@ import (
 	"encoding/json"
 	"errors"
 	"io"
-	"net/http"
 	"testing"
 
 	"github.com/go-logr/logr"
@@ -41,7 +40,7 @@ func TestHTTPADCExecutorBuildHTTPRequestBypassCache(t *testing.T) {
 
 	build := func(config adctypes.Config, path string) (ADCServerOpts, string) {
 		req, err := e.buildHTTPRequest(context.Background(), "http://apisix:9180", config, nil, nil,
-			&adctypes.Resources{}, http.MethodPut, path)
+			&adctypes.Resources{}, path)
 		require.NoError(t, err)
 		body, err := io.ReadAll(req.Body)
 		require.NoError(t, err)
@@ -70,6 +69,37 @@ func TestHTTPADCExecutorBuildHTTPRequestBypassCache(t *testing.T) {
 	assert.NotContains(t, raw, "bypassCache")
 }
 
+func TestHTTPADCExecutorBuildHTTPRequestCaCert(t *testing.T) {
+	e := &HTTPADCExecutor{
+		serverURL: "http://127.0.0.1:3000",
+		log:       logr.Discard(),
+	}
+
+	build := func(config adctypes.Config) (ADCServerOpts, string) {
+		req, err := e.buildHTTPRequest(context.Background(), "https://apisix:9180", config, nil, nil,
+			&adctypes.Resources{}, pathSync)
+		require.NoError(t, err)
+		body, err := io.ReadAll(req.Body)
+		require.NoError(t, err)
+		var parsed ADCServerRequest
+		require.NoError(t, json.Unmarshal(body, &parsed))
+		return parsed.Task.Opts, string(body)
+	}
+
+	// Without a CA bundle the request stays what an ADC server that predates caCert
+	// already accepts.
+	opts, raw := build(adctypes.Config{Name: "GatewayProxy/ns/name", TlsVerify: true})
+	assert.Empty(t, opts.CaCert)
+	assert.NotContains(t, raw, "caCert")
+
+	const caCert = "-----BEGIN CERTIFICATE-----\nMIIB\n-----END CERTIFICATE-----"
+	opts, raw = build(adctypes.Config{Name: "GatewayProxy/ns/name", TlsVerify: true, CaCert: caCert})
+	assert.Equal(t, caCert, opts.CaCert)
+	assert.Contains(t, raw, "caCert")
+	// verification stays on, otherwise the bundle would be pointless
+	assert.Equal(t, false, *opts.TlsSkipVerify)
+}
+
 // confVersionError is what a push carrying a conf_version older than the data plane's
 // comes back as, once the ADC server has relayed the rejection to us.
 func confVersionError() error {

internal/adc/translator/gatewayproxy.go

@@ -18,6 +18,8 @@
 package translator
 
 import (
+	"crypto/x509"
+	"encoding/pem"
 	"fmt"
 	"net"
 	"strconv"
@@ -56,6 +58,17 @@ func (t *Translator) TranslateGatewayProxyToConfig(tctx *provider.TranslateConte
 		cfg.TlsVerify = *cp.TlsVerify
 	}
 
+	if cp.CaCert != nil && cp.CaCert.Value != "" {
+		// reject unusable CA material here rather than at connect time
+		if err := validateCaCert(cp.CaCert.Value); err != nil {
+			return nil, err
+		}
+		if !cfg.TlsVerify {
+			t.Log.Info("caCert is ignored because tlsVerify is disabled", "gatewayproxy", utils.NamespacedNameKind(gatewayProxy))
+		}
+		cfg.CaCert = cp.CaCert.Value
+	}
+
 	if cp.Auth.Type == v1alpha1.AuthTypeAdminKey && cp.Auth.AdminKey != nil {
 		if cp.Auth.AdminKey.ValueFrom != nil && cp.Auth.AdminKey.ValueFrom.SecretKeyRef != nil {
 			secretRef := cp.Auth.AdminKey.ValueFrom.SecretKeyRef
@@ -142,3 +155,28 @@ func (t *Translator) TranslateGatewayProxyToConfig(tctx *provider.TranslateConte
 
 	return &cfg, nil
 }
+
+// validateCaCert parses every certificate in the bundle. x509.CertPool skips
+// blocks it cannot decode, so a bundle whose second certificate is broken would
+// otherwise reach the ADC server and fail there instead.
+func validateCaCert(caCert string) error {
+	var count int
+	for rest := []byte(caCert); len(rest) > 0; {
+		var block *pem.Block
+		block, rest = pem.Decode(rest)
+		if block == nil {
+			break
+		}
+		if block.Type != "CERTIFICATE" {
+			return fmt.Errorf("invalid caCert: expected a CERTIFICATE block, got %s", block.Type)
+		}
+		if _, err := x509.ParseCertificate(block.Bytes); err != nil {
+			return fmt.Errorf("invalid caCert: %w", err)
+		}
+		count++
+	}
+	if count == 0 {
+		return errors.New("invalid caCert: no PEM-encoded certificate found")
+	}
+	return nil
+}