diff --git a/Makefile b/Makefile
index 39a1f7ae..73cf9284 100644
--- a/Makefile
+++ b/Makefile
@@ -30,7 +30,7 @@ KIND_NAME ?= apisix-ingress-cluster
KIND_NODE_IMAGE ?= kindest/node:v1.32.2@sha256:f226345927d7e348497136874b6d207e0b32cc52154ad8323129352923a3142f
DASHBOARD_VERSION ?= dev
-ADC_VERSION ?= 0.27.1
+ADC_VERSION ?= 0.29.0
DIR := $(shell pwd)
diff --git a/api/adc/types.go b/api/adc/types.go
index 4ff25a71..f7adae1c 100644
--- a/api/adc/types.go
+++ b/api/adc/types.go
@@ -807,6 +807,11 @@ type Config struct {
TlsVerify bool
BackendType string
+ // CaCert is a PEM-encoded CA certificate (or bundle) used to verify the
+ // control plane, in place of the system trust store. Only meaningful when
+ // TlsVerify is true.
+ CaCert string
+
// BypassCache makes the ADC server drop the in-memory baseline it holds for this
// cacheKey and re-derive it from the data plane before computing the diff. It is a
// per-request flag set on the sync path, not part of the translated configuration.
@@ -820,10 +825,12 @@ func (c Config) MarshalJSON() ([]byte, error) {
Name string `json:"name"`
ServerAddrs []string `json:"serverAddrs"`
TlsVerify bool `json:"tlsVerify"`
+ HasCaCert bool `json:"hasCaCert"`
}{
Name: c.Name,
ServerAddrs: c.ServerAddrs,
TlsVerify: c.TlsVerify,
+ HasCaCert: c.CaCert != "",
})
}
diff --git a/api/v1alpha1/gatewayproxy_types.go b/api/v1alpha1/gatewayproxy_types.go
index 680fa8a9..f9a5aa8e 100644
--- a/api/v1alpha1/gatewayproxy_types.go
+++ b/api/v1alpha1/gatewayproxy_types.go
@@ -136,11 +136,29 @@ type ControlPlaneProvider struct {
// +optional
TlsVerify *bool `json:"tlsVerify,omitempty"`
+ // CaCert specifies the CA certificate used to verify the control plane's TLS
+ // certificate, in place of the system trust store.
+ // Set it when the control plane uses a self-signed or private CA certificate.
+ // It has no effect when tlsVerify is false.
+ // +optional
+ CaCert *ControlPlaneCaCert `json:"caCert,omitempty"`
+
// Auth specifies the authentication configuration.
// +kubebuilder:validation:Required
Auth ControlPlaneAuth `json:"auth"`
}
+// ControlPlaneCaCert defines the CA certificate used to verify the control plane.
+//
+// Only an inline value is supported today. A valueFrom source can be added
+// later without breaking existing resources.
+type ControlPlaneCaCert struct {
+ // Value sets the PEM-encoded CA certificate (or bundle) explicitly.
+ // +kubebuilder:validation:Required
+ // +kubebuilder:validation:XValidation:rule="self.contains('-----BEGIN CERTIFICATE-----')",message="value must be a PEM-encoded certificate"
+ Value string `json:"value"`
+}
+
type ProviderService struct {
// Name is the name of the provider.
Name string `json:"name"`
diff --git a/api/v1alpha1/zz_generated.deepcopy.go b/api/v1alpha1/zz_generated.deepcopy.go
index 6d89538d..4974d7f1 100644
--- a/api/v1alpha1/zz_generated.deepcopy.go
+++ b/api/v1alpha1/zz_generated.deepcopy.go
@@ -384,6 +384,21 @@ func (in *ControlPlaneAuth) DeepCopy() *ControlPlaneAuth {
return out
}
+// DeepCopyInto is an autogenerated deepcopy function, copying the receiver, writing into out. in must be non-nil.
+func (in *ControlPlaneCaCert) DeepCopyInto(out *ControlPlaneCaCert) {
+ *out = *in
+}
+
+// DeepCopy is an autogenerated deepcopy function, copying the receiver, creating a new ControlPlaneCaCert.
+func (in *ControlPlaneCaCert) DeepCopy() *ControlPlaneCaCert {
+ if in == nil {
+ return nil
+ }
+ out := new(ControlPlaneCaCert)
+ in.DeepCopyInto(out)
+ return out
+}
+
// DeepCopyInto is an autogenerated deepcopy function, copying the receiver, writing into out. in must be non-nil.
func (in *ControlPlaneProvider) DeepCopyInto(out *ControlPlaneProvider) {
*out = *in
@@ -402,6 +417,11 @@ func (in *ControlPlaneProvider) DeepCopyInto(out *ControlPlaneProvider) {
*out = new(bool)
**out = **in
}
+ if in.CaCert != nil {
+ in, out := &in.CaCert, &out.CaCert
+ *out = new(ControlPlaneCaCert)
+ **out = **in
+ }
in.Auth.DeepCopyInto(&out.Auth)
}
diff --git a/config/crd-nocel/apisix.apache.org_v2.yaml b/config/crd-nocel/apisix.apache.org_v2.yaml
index 9e4cc186..337fe997 100644
--- a/config/crd-nocel/apisix.apache.org_v2.yaml
+++ b/config/crd-nocel/apisix.apache.org_v2.yaml
@@ -2273,6 +2273,20 @@ spec:
required:
- type
type: object
+ caCert:
+ description: |-
+ CaCert specifies the CA certificate used to verify the control plane's TLS
+ certificate, in place of the system trust store.
+ Set it when the control plane uses a self-signed or private CA certificate.
+ It has no effect when tlsVerify is false.
+ properties:
+ value:
+ description: Value sets the PEM-encoded CA certificate
+ (or bundle) explicitly.
+ type: string
+ required:
+ - value
+ type: object
endpoints:
description: Endpoints specifies the list of control plane
endpoints.
diff --git a/config/crd/bases/apisix.apache.org_gatewayproxies.yaml b/config/crd/bases/apisix.apache.org_gatewayproxies.yaml
index 23a7ed50..b2865255 100644
--- a/config/crd/bases/apisix.apache.org_gatewayproxies.yaml
+++ b/config/crd/bases/apisix.apache.org_gatewayproxies.yaml
@@ -120,6 +120,23 @@ spec:
- message: adminKey must be specified when type is AdminKey
rule: 'self.type == ''AdminKey'' ? has(self.adminKey) :
true'
+ caCert:
+ description: |-
+ CaCert specifies the CA certificate used to verify the control plane's TLS
+ certificate, in place of the system trust store.
+ Set it when the control plane uses a self-signed or private CA certificate.
+ It has no effect when tlsVerify is false.
+ properties:
+ value:
+ description: Value sets the PEM-encoded CA certificate
+ (or bundle) explicitly.
+ type: string
+ x-kubernetes-validations:
+ - message: value must be a PEM-encoded certificate
+ rule: self.contains('-----BEGIN CERTIFICATE-----')
+ required:
+ - value
+ type: object
endpoints:
description: Endpoints specifies the list of control plane
endpoints.
diff --git a/docs/en/latest/reference/api-reference.md b/docs/en/latest/reference/api-reference.md
index 8def827e..f973f85f 100644
--- a/docs/en/latest/reference/api-reference.md
+++ b/docs/en/latest/reference/api-reference.md
@@ -297,6 +297,23 @@ ControlPlaneAuth defines the authentication configuration for control plane.
| `adminKey` _[AdminKeyAuth](#adminkeyauth)_ | AdminKey specifies the admin key authentication configuration. |
+_Appears in:_
+- [ControlPlaneProvider](#controlplaneprovider)
+
+#### ControlPlaneCaCert
+
+
+ControlPlaneCaCert defines the CA certificate used to verify the control plane.
+Only an inline value is supported today. A valueFrom source can be added
+later without breaking existing resources.
+
+
+
+| Field | Description |
+| --- | --- |
+| `value` _string_ | Value sets the PEM-encoded CA certificate (or bundle) explicitly. |
+
+
_Appears in:_
- [ControlPlaneProvider](#controlplaneprovider)
@@ -313,6 +330,7 @@ ControlPlaneProvider defines configuration for control plane provider.
| `endpoints` _string array_ | Endpoints specifies the list of control plane endpoints. |
| `service` _[ProviderService](#providerservice)_ | |
| `tlsVerify` _boolean_ | TlsVerify specifies whether to verify the TLS certificate of the control plane. |
+| `caCert` _[ControlPlaneCaCert](#controlplanecacert)_ | CaCert specifies the CA certificate used to verify the control plane's TLS certificate, in place of the system trust store. Set it when the control plane uses a self-signed or private CA certificate. It has no effect when tlsVerify is false. |
| `auth` _[ControlPlaneAuth](#controlplaneauth)_ | Auth specifies the authentication configuration. |
diff --git a/internal/adc/client/executor.go b/internal/adc/client/executor.go
index 2fe32009..92feb7d9 100644
--- a/internal/adc/client/executor.go
+++ b/internal/adc/client/executor.go
@@ -84,7 +84,11 @@ type ADCServerOpts struct {
LabelSelector map[string]string `json:"labelSelector,omitempty"`
IncludeResourceType []string `json:"includeResourceType,omitempty"`
TlsSkipVerify *bool `json:"tlsSkipVerify,omitempty"`
- CacheKey string `json:"cacheKey"`
+ // CaCert is the PEM-encoded CA certificate (or bundle) the ADC server verifies
+ // the control plane against. Older ADC servers ignore it, and omitempty keeps
+ // requests without a CA bundle byte for byte what they were.
+ CaCert string `json:"caCert,omitempty"`
+ CacheKey string `json:"cacheKey"`
// BypassCache is only accepted by the /sync task of ADC >= 0.27.0. Both ADC task
// schemas reject unknown fields, so omitempty is what keeps every other request --
// /validate, and every sync that is not recovering from a rejection -- byte for byte
@@ -103,6 +107,7 @@ func (r ADCServerRequest) MarshalLog() any {
"labelSelector": r.Task.Opts.LabelSelector,
"includeResourceType": r.Task.Opts.IncludeResourceType,
"tlsSkipVerify": r.Task.Opts.TlsSkipVerify,
+ "hasCaCert": r.Task.Opts.CaCert != "",
"cacheKey": r.Task.Opts.CacheKey,
"config": r.Task.Config.MarshalLog(),
}
@@ -244,7 +249,7 @@ func (e *HTTPADCExecutor) runHTTPSyncForSingleServer(ctx context.Context, server
}
// Build HTTP request
- req, err := e.buildHTTPRequest(ctx, serverAddr, config, labels, types, resources, http.MethodPut, pathSync)
+ req, err := e.buildHTTPRequest(ctx, serverAddr, config, labels, types, resources, pathSync)
if err != nil {
return fmt.Errorf("failed to build HTTP request: %w", err)
}
@@ -278,7 +283,7 @@ func (e *HTTPADCExecutor) runHTTPValidateForSingleServer(ctx context.Context, se
return fmt.Errorf("failed to load resources from file %s: %w", filePath, err)
}
- req, err := e.buildHTTPRequest(ctx, serverAddr, config, labels, types, resources, http.MethodPut, pathValidate)
+ req, err := e.buildHTTPRequest(ctx, serverAddr, config, labels, types, resources, pathValidate)
if err != nil {
return fmt.Errorf("failed to build validate request: %w", err)
}
@@ -349,7 +354,7 @@ func (e *HTTPADCExecutor) loadResourcesFromFile(filePath string) (*adctypes.Reso
}
// buildHTTPRequest builds the HTTP request for ADC Server
-func (e *HTTPADCExecutor) buildHTTPRequest(ctx context.Context, serverAddr string, config adctypes.Config, labels map[string]string, types []string, resources *adctypes.Resources, method string, path string) (*http.Request, error) {
+func (e *HTTPADCExecutor) buildHTTPRequest(ctx context.Context, serverAddr string, config adctypes.Config, labels map[string]string, types []string, resources *adctypes.Resources, path string) (*http.Request, error) {
// Prepare request body
tlsVerify := config.TlsVerify
bypassCache := path == pathSync && config.BypassCache
@@ -362,6 +367,7 @@ func (e *HTTPADCExecutor) buildHTTPRequest(ctx context.Context, serverAddr strin
LabelSelector: labels,
IncludeResourceType: types,
TlsSkipVerify: ptr.To(!tlsVerify),
+ CaCert: config.CaCert,
CacheKey: config.Name,
BypassCache: bypassCache,
},
@@ -385,10 +391,11 @@ func (e *HTTPADCExecutor) buildHTTPRequest(ctx context.Context, serverAddr strin
"labelSelector", labels,
"includeResourceType", types,
"tlsSkipVerify", !tlsVerify,
+ "hasCaCert", config.CaCert != "",
)
// Create HTTP request
- req, err := http.NewRequestWithContext(ctx, method, e.serverURL+path, bytes.NewBuffer(jsonData))
+ req, err := http.NewRequestWithContext(ctx, http.MethodPut, e.serverURL+path, bytes.NewBuffer(jsonData))
if err != nil {
return nil, fmt.Errorf("failed to create HTTP request: %w", err)
}
diff --git a/internal/adc/client/executor_test.go b/internal/adc/client/executor_test.go
index 9e7ee71c..a7f998e6 100644
--- a/internal/adc/client/executor_test.go
+++ b/internal/adc/client/executor_test.go
@@ -22,7 +22,6 @@ import (
"encoding/json"
"errors"
"io"
- "net/http"
"testing"
"github.com/go-logr/logr"
@@ -41,7 +40,7 @@ func TestHTTPADCExecutorBuildHTTPRequestBypassCache(t *testing.T) {
build := func(config adctypes.Config, path string) (ADCServerOpts, string) {
req, err := e.buildHTTPRequest(context.Background(), "http://apisix:9180", config, nil, nil,
- &adctypes.Resources{}, http.MethodPut, path)
+ &adctypes.Resources{}, path)
require.NoError(t, err)
body, err := io.ReadAll(req.Body)
require.NoError(t, err)
@@ -70,6 +69,37 @@ func TestHTTPADCExecutorBuildHTTPRequestBypassCache(t *testing.T) {
assert.NotContains(t, raw, "bypassCache")
}
+func TestHTTPADCExecutorBuildHTTPRequestCaCert(t *testing.T) {
+ e := &HTTPADCExecutor{
+ serverURL: "http://127.0.0.1:3000",
+ log: logr.Discard(),
+ }
+
+ build := func(config adctypes.Config) (ADCServerOpts, string) {
+ req, err := e.buildHTTPRequest(context.Background(), "https://apisix:9180", config, nil, nil,
+ &adctypes.Resources{}, pathSync)
+ require.NoError(t, err)
+ body, err := io.ReadAll(req.Body)
+ require.NoError(t, err)
+ var parsed ADCServerRequest
+ require.NoError(t, json.Unmarshal(body, &parsed))
+ return parsed.Task.Opts, string(body)
+ }
+
+ // Without a CA bundle the request stays what an ADC server that predates caCert
+ // already accepts.
+ opts, raw := build(adctypes.Config{Name: "GatewayProxy/ns/name", TlsVerify: true})
+ assert.Empty(t, opts.CaCert)
+ assert.NotContains(t, raw, "caCert")
+
+ const caCert = "-----BEGIN CERTIFICATE-----\nMIIB\n-----END CERTIFICATE-----"
+ opts, raw = build(adctypes.Config{Name: "GatewayProxy/ns/name", TlsVerify: true, CaCert: caCert})
+ assert.Equal(t, caCert, opts.CaCert)
+ assert.Contains(t, raw, "caCert")
+ // verification stays on, otherwise the bundle would be pointless
+ assert.Equal(t, false, *opts.TlsSkipVerify)
+}
+
// confVersionError is what a push carrying a conf_version older than the data plane's
// comes back as, once the ADC server has relayed the rejection to us.
func confVersionError() error {
diff --git a/internal/adc/translator/gatewayproxy.go b/internal/adc/translator/gatewayproxy.go
index 3a9a00f4..534a82fb 100644
--- a/internal/adc/translator/gatewayproxy.go
+++ b/internal/adc/translator/gatewayproxy.go
@@ -18,6 +18,8 @@
package translator
import (
+ "crypto/x509"
+ "encoding/pem"
"fmt"
"net"
"strconv"
@@ -56,6 +58,17 @@ func (t *Translator) TranslateGatewayProxyToConfig(tctx *provider.TranslateConte
cfg.TlsVerify = *cp.TlsVerify
}
+ if cp.CaCert != nil && cp.CaCert.Value != "" {
+ // reject unusable CA material here rather than at connect time
+ if err := validateCaCert(cp.CaCert.Value); err != nil {
+ return nil, err
+ }
+ if !cfg.TlsVerify {
+ t.Log.Info("caCert is ignored because tlsVerify is disabled", "gatewayproxy", utils.NamespacedNameKind(gatewayProxy))
+ }
+ cfg.CaCert = cp.CaCert.Value
+ }
+
if cp.Auth.Type == v1alpha1.AuthTypeAdminKey && cp.Auth.AdminKey != nil {
if cp.Auth.AdminKey.ValueFrom != nil && cp.Auth.AdminKey.ValueFrom.SecretKeyRef != nil {
secretRef := cp.Auth.AdminKey.ValueFrom.SecretKeyRef
@@ -142,3 +155,28 @@ func (t *Translator) TranslateGatewayProxyToConfig(tctx *provider.TranslateConte
return &cfg, nil
}
+
+// validateCaCert parses every certificate in the bundle. x509.CertPool skips
+// blocks it cannot decode, so a bundle whose second certificate is broken would
+// otherwise reach the ADC server and fail there instead.
+func validateCaCert(caCert string) error {
+ var count int
+ for rest := []byte(caCert); len(rest) > 0; {
+ var block *pem.Block
+ block, rest = pem.Decode(rest)
+ if block == nil {
+ break
+ }
+ if block.Type != "CERTIFICATE" {
+ return fmt.Errorf("invalid caCert: expected a CERTIFICATE block, got %s", block.Type)
+ }
+ if _, err := x509.ParseCertificate(block.Bytes); err != nil {
+ return fmt.Errorf("invalid caCert: %w", err)
+ }
+ count++
+ }
+ if count == 0 {
+ return errors.New("invalid caCert: no PEM-encoded certificate found")
+ }
+ return nil
+}
diff --git a/internal/adc/translator/gatewayproxy_test.go b/internal/adc/translator/gatewayproxy_test.go
new file mode 100644
index 00000000..629df2f6
--- /dev/null
+++ b/internal/adc/translator/gatewayproxy_test.go
@@ -0,0 +1,128 @@
+// Licensed to the Apache Software Foundation (ASF) under one
+// or more contributor license agreements. See the NOTICE file
+// distributed with this work for additional information
+// regarding copyright ownership. The ASF licenses this file
+// to you under the Apache License, Version 2.0 (the
+// "License"); you may not use this file except in compliance
+// with the License. You may obtain a copy of the License at
+//
+// http://www.apache.org/licenses/LICENSE-2.0
+//
+// Unless required by applicable law or agreed to in writing,
+// software distributed under the License is distributed on an
+// "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY
+// KIND, either express or implied. See the License for the
+// specific language governing permissions and limitations
+// under the License.
+
+package translator
+
+import (
+ "context"
+ "testing"
+
+ "github.com/go-logr/logr"
+ "github.com/stretchr/testify/assert"
+ "github.com/stretchr/testify/require"
+ metav1 "k8s.io/apimachinery/pkg/apis/meta/v1"
+ "k8s.io/utils/ptr"
+
+ "github.com/apache/apisix-ingress-controller/api/v1alpha1"
+ "github.com/apache/apisix-ingress-controller/internal/provider"
+)
+
+func newGatewayProxy(tlsVerify *bool, caCert string) *v1alpha1.GatewayProxy {
+ // an empty string stands for the field being unset
+ var caCertRef *v1alpha1.ControlPlaneCaCert
+ if caCert != "" {
+ caCertRef = &v1alpha1.ControlPlaneCaCert{Value: caCert}
+ }
+ return &v1alpha1.GatewayProxy{
+ ObjectMeta: metav1.ObjectMeta{
+ Namespace: "default",
+ Name: "gp",
+ },
+ Spec: v1alpha1.GatewayProxySpec{
+ Provider: &v1alpha1.GatewayProxyProvider{
+ Type: v1alpha1.ProviderTypeControlPlane,
+ ControlPlane: &v1alpha1.ControlPlaneProvider{
+ Endpoints: []string{"https://cp.example.com:9180"},
+ TlsVerify: tlsVerify,
+ CaCert: caCertRef,
+ Auth: v1alpha1.ControlPlaneAuth{
+ Type: v1alpha1.AuthTypeAdminKey,
+ AdminKey: &v1alpha1.AdminKeyAuth{
+ Value: "admin-key",
+ },
+ },
+ },
+ },
+ },
+ }
+}
+
+func TestTranslateGatewayProxyToConfigCaCert(t *testing.T) {
+ t.Run("carries the CA certificate into the config", func(t *testing.T) {
+ tr := &Translator{Log: logr.Discard()}
+ tctx := provider.NewDefaultTranslateContext(context.Background())
+
+ cfg, err := tr.TranslateGatewayProxyToConfig(tctx, newGatewayProxy(ptr.To(true), testCACert), false)
+ require.NoError(t, err)
+ require.NotNil(t, cfg)
+ assert.True(t, cfg.TlsVerify)
+ assert.Equal(t, testCACert, cfg.CaCert)
+ })
+
+ t.Run("leaves the CA certificate empty when unset", func(t *testing.T) {
+ tr := &Translator{Log: logr.Discard()}
+ tctx := provider.NewDefaultTranslateContext(context.Background())
+
+ cfg, err := tr.TranslateGatewayProxyToConfig(tctx, newGatewayProxy(ptr.To(true), ""), false)
+ require.NoError(t, err)
+ require.NotNil(t, cfg)
+ assert.Empty(t, cfg.CaCert)
+ })
+
+ // every certificate is parsed: x509.CertPool silently skips the blocks it
+ // cannot decode, which would let a broken one through to the ADC server.
+ for name, caCert := range map[string]string{
+ "not PEM at all": "not-a-certificate",
+ "a header with no certificate": "-----BEGIN CERTIFICATE-----",
+ "an unparseable body": "-----BEGIN CERTIFICATE-----\nAAAA\n-----END CERTIFICATE-----",
+ "a key rather than a certificate": "-----BEGIN RSA PRIVATE KEY-----\nAAAA\n-----END RSA PRIVATE KEY-----",
+ "one good and one broken certificate": testCACert +
+ "\n-----BEGIN CERTIFICATE-----\nAAAA\n-----END CERTIFICATE-----",
+ } {
+ t.Run("rejects a CA certificate that is "+name, func(t *testing.T) {
+ tr := &Translator{Log: logr.Discard()}
+ tctx := provider.NewDefaultTranslateContext(context.Background())
+
+ cfg, err := tr.TranslateGatewayProxyToConfig(tctx, newGatewayProxy(ptr.To(true), caCert), false)
+ require.Error(t, err)
+ assert.Contains(t, err.Error(), "invalid caCert")
+ assert.Nil(t, cfg)
+ })
+ }
+
+ t.Run("accepts a bundle of several certificates", func(t *testing.T) {
+ tr := &Translator{Log: logr.Discard()}
+ tctx := provider.NewDefaultTranslateContext(context.Background())
+
+ bundle := testCACert + "\n" + testCACert
+ cfg, err := tr.TranslateGatewayProxyToConfig(tctx, newGatewayProxy(ptr.To(true), bundle), false)
+ require.NoError(t, err)
+ require.NotNil(t, cfg)
+ assert.Equal(t, bundle, cfg.CaCert)
+ })
+
+ t.Run("still carries the CA certificate when verification is off", func(t *testing.T) {
+ tr := &Translator{Log: logr.Discard()}
+ tctx := provider.NewDefaultTranslateContext(context.Background())
+
+ cfg, err := tr.TranslateGatewayProxyToConfig(tctx, newGatewayProxy(ptr.To(false), testCACert), false)
+ require.NoError(t, err)
+ require.NotNil(t, cfg)
+ assert.False(t, cfg.TlsVerify)
+ assert.Equal(t, testCACert, cfg.CaCert)
+ })
+}
diff --git a/test/e2e/crds/v1alpha1/gatewayproxy_tls.go b/test/e2e/crds/v1alpha1/gatewayproxy_tls.go
new file mode 100644
index 00000000..5de91d35
--- /dev/null
+++ b/test/e2e/crds/v1alpha1/gatewayproxy_tls.go
@@ -0,0 +1,341 @@
+// Licensed to the Apache Software Foundation (ASF) under one
+// or more contributor license agreements. See the NOTICE file
+// distributed with this work for additional information
+// regarding copyright ownership. The ASF licenses this file
+// to you under the Apache License, Version 2.0 (the
+// "License"); you may not use this file except in compliance
+// with the License. You may obtain a copy of the License at
+//
+// http://www.apache.org/licenses/LICENSE-2.0
+//
+// Unless required by applicable law or agreed to in writing,
+// software distributed under the License is distributed on an
+// "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY
+// KIND, either express or implied. See the License for the
+// specific language governing permissions and limitations
+// under the License.
+
+package v1alpha1
+
+import (
+ "crypto/ecdsa"
+ "crypto/elliptic"
+ "crypto/rand"
+ "crypto/x509"
+ "crypto/x509/pkix"
+ "encoding/pem"
+ "fmt"
+ "math/big"
+ "net/http"
+ "strings"
+ "time"
+
+ . "github.com/onsi/ginkgo/v2"
+ . "github.com/onsi/gomega"
+ metav1 "k8s.io/apimachinery/pkg/apis/meta/v1"
+
+ "github.com/apache/apisix-ingress-controller/test/e2e/framework"
+ "github.com/apache/apisix-ingress-controller/test/e2e/scaffold"
+)
+
+var _ = Describe("Test GatewayProxy control plane TLS", Label("apisix.apache.org", "v1alpha1", "gatewayproxy"), func() {
+ var s = scaffold.NewDefaultScaffold()
+
+ // The admin API only listens on plain HTTP, so an openresty in front of it
+ // stands in for a control plane published over TLS. Its certificate is
+ // signed by a CA generated per test, which is exactly the case caCert
+ // exists for: nothing in the system trust store can verify it.
+ const adminTLSProxySpec = `
+apiVersion: v1
+kind: ConfigMap
+metadata:
+ name: admin-tls
+data:
+ nginx.conf: |
+ worker_processes 1;
+ pid /run/nginx.pid;
+ events {
+ worker_connections 1024;
+ }
+ http {
+ # the standalone provider PUTs the whole configuration through here
+ client_max_body_size 32m;
+ server {
+ listen 9543 ssl;
+ ssl_certificate /etc/nginx/ssl/tls.crt;
+ ssl_certificate_key /etc/nginx/ssl/tls.key;
+ location / {
+ proxy_pass %s;
+ proxy_http_version 1.1;
+ }
+ }
+ server {
+ listen 9544;
+ location /healthz {
+ return 200 'ok';
+ }
+ }
+ }
+---
+apiVersion: apps/v1
+kind: Deployment
+metadata:
+ name: admin-tls
+spec:
+ replicas: 1
+ selector:
+ matchLabels:
+ app: admin-tls
+ template:
+ metadata:
+ labels:
+ app: admin-tls
+ spec:
+ volumes:
+ - name: config
+ configMap:
+ name: admin-tls
+ - name: ssl
+ secret:
+ secretName: admin-tls
+ containers:
+ - name: admin-tls
+ image: "openresty/openresty:1.27.1.2-4-bullseye-fat"
+ imagePullPolicy: IfNotPresent
+ ports:
+ - containerPort: 9543
+ name: https
+ protocol: TCP
+ readinessProbe:
+ httpGet:
+ path: /healthz
+ port: 9544
+ initialDelaySeconds: 2
+ periodSeconds: 2
+ volumeMounts:
+ - mountPath: /usr/local/openresty/nginx/conf/nginx.conf
+ name: config
+ subPath: nginx.conf
+ - mountPath: /etc/nginx/ssl
+ name: ssl
+---
+apiVersion: v1
+kind: Service
+metadata:
+ name: admin-tls
+spec:
+ selector:
+ app: admin-tls
+ ports:
+ - name: https
+ port: 9543
+ protocol: TCP
+ targetPort: 9543
+`
+
+ const gatewayProxySpec = `
+apiVersion: apisix.apache.org/v1alpha1
+kind: GatewayProxy
+metadata:
+ name: apisix-proxy-config-tls
+spec:
+ provider:
+ type: ControlPlane
+ controlPlane:
+ endpoints:
+ - %s
+ tlsVerify: true
+%s
+ auth:
+ type: AdminKey
+ adminKey:
+ value: "%s"
+`
+
+ const gatewayClassSpec = `
+apiVersion: gateway.networking.k8s.io/v1
+kind: GatewayClass
+metadata:
+ name: %s
+spec:
+ controllerName: %s
+`
+
+ const gatewaySpec = `
+apiVersion: gateway.networking.k8s.io/v1
+kind: Gateway
+metadata:
+ name: %s
+spec:
+ gatewayClassName: %s
+ listeners:
+ - name: http1
+ protocol: HTTP
+ port: 80
+ infrastructure:
+ parametersRef:
+ group: apisix.apache.org
+ kind: GatewayProxy
+ name: apisix-proxy-config-tls
+`
+
+ const httpRouteSpec = `
+apiVersion: gateway.networking.k8s.io/v1
+kind: HTTPRoute
+metadata:
+ name: httpbin-tls-cp
+spec:
+ parentRefs:
+ - name: %s
+ hostnames:
+ - "%s"
+ rules:
+ - matches:
+ - path:
+ type: Exact
+ value: /get
+ backendRefs:
+ - name: httpbin-service-e2e-test
+ port: 80
+`
+
+ // indent a PEM block so it survives being inlined into the YAML above
+ indent := func(pem string, width int) string {
+ lines := strings.Split(strings.TrimRight(pem, "\n"), "\n")
+ for i, line := range lines {
+ lines[i] = strings.Repeat(" ", width) + line
+ }
+ return strings.Join(lines, "\n")
+ }
+
+ var caCert string
+
+ BeforeEach(func() {
+ By("generate a CA and a server certificate for the TLS control plane")
+ // the ADC sidecar dials the proxy by its in-cluster name, so that is
+ // what the certificate has to be valid for
+ var serverCert, serverKey string
+ caCert, serverCert, serverKey = generateSignedCert([]string{
+ "admin-tls",
+ fmt.Sprintf("admin-tls.%s", s.Namespace()),
+ fmt.Sprintf("admin-tls.%s.svc", s.Namespace()),
+ })
+
+ err := s.NewKubeTlsSecret("admin-tls", serverCert, serverKey)
+ Expect(err).NotTo(HaveOccurred(), "creating the server certificate secret")
+
+ By("deploy a TLS terminator in front of the admin API")
+ err = s.CreateResourceFromString(fmt.Sprintf(adminTLSProxySpec, s.Deployer.GetAdminEndpoint()))
+ Expect(err).NotTo(HaveOccurred(), "creating the TLS control plane")
+ Expect(framework.WaitPodsAvailable(s.GinkgoT, s.KubeOpts(), metav1.ListOptions{
+ LabelSelector: "app=admin-tls",
+ })).NotTo(HaveOccurred(), "waiting for the TLS control plane")
+ })
+
+ // routes the gateway through a GatewayProxy that reaches the control plane
+ // over TLS, with caCert set to caCertField
+ attachRoute := func(hostname, caCertField string) {
+ By("create GatewayProxy")
+ endpoint := fmt.Sprintf("https://admin-tls.%s:9543", s.Namespace())
+ err := s.CreateResourceFromString(fmt.Sprintf(gatewayProxySpec, endpoint, caCertField, s.AdminKey()))
+ Expect(err).NotTo(HaveOccurred(), "creating GatewayProxy")
+
+ By("create GatewayClass")
+ gatewayClassName := fmt.Sprintf("%s-tls", s.Namespace())
+ err = s.CreateResourceFromString(fmt.Sprintf(gatewayClassSpec, gatewayClassName, s.GetControllerName()))
+ Expect(err).NotTo(HaveOccurred(), "creating GatewayClass")
+
+ By("create Gateway")
+ gatewayName := fmt.Sprintf("%s-tls", s.Namespace())
+ err = s.CreateResourceFromString(fmt.Sprintf(gatewaySpec, gatewayName, gatewayClassName))
+ Expect(err).NotTo(HaveOccurred(), "creating Gateway")
+
+ By("create HTTPRoute")
+ err = s.CreateResourceFromString(fmt.Sprintf(httpRouteSpec, gatewayName, hostname))
+ Expect(err).NotTo(HaveOccurred(), "creating HTTPRoute")
+ }
+
+ It("syncs to a private-CA control plane when caCert is trusted", func() {
+ attachRoute("httpbin-tls-cp.org", " caCert:\n value: |\n"+indent(caCert, 10))
+
+ By("the route is programmed, so the sync got through TLS verification")
+ s.RequestAssert(&scaffold.RequestAssert{
+ Method: "GET",
+ Path: "/get",
+ Host: "httpbin-tls-cp.org",
+ Check: scaffold.WithExpectedStatus(http.StatusOK),
+ })
+ })
+
+ It("fails to sync when caCert is missing", func() {
+ attachRoute("httpbin-tls-cp-untrusted.org", "")
+
+ By("the control plane certificate cannot be verified")
+ // look back as well as forward: the sync can fail before the stream opens
+ s.WaitControllerManagerLog("unable to verify the first certificate", 60, time.Minute)
+
+ By("so the route is never programmed")
+ s.RequestAssert(&scaffold.RequestAssert{
+ Method: "GET",
+ Path: "/get",
+ Host: "httpbin-tls-cp-untrusted.org",
+ Check: scaffold.WithExpectedStatus(http.StatusNotFound),
+ })
+ })
+
+ It("rejects a caCert that is not a certificate", func() {
+ By("create GatewayProxy with an unparseable caCert")
+ endpoint := fmt.Sprintf("https://admin-tls.%s:9543", s.Namespace())
+ output, err := s.CreateResourceFromStringAndGetOutput(
+ fmt.Sprintf(gatewayProxySpec, endpoint, " caCert:\n value: not-a-certificate", s.AdminKey()),
+ )
+ Expect(err).To(HaveOccurred(), "the API server should reject it")
+ Expect(output + err.Error()).To(ContainSubstring("value must be a PEM-encoded certificate"))
+ })
+})
+
+// generateSignedCert returns a CA and a server certificate signed by it. The
+// scaffold's GenerateMACert gives both the same subject, which OpenSSL reads as
+// a self-issued certificate rather than one chaining to the CA, so a strict TLS
+// client rejects it before it ever looks at the bundle.
+func generateSignedCert(dnsNames []string) (caCertPEM, serverCertPEM, serverKeyPEM string) {
+ serial := func() *big.Int {
+ n, err := rand.Int(rand.Reader, new(big.Int).Lsh(big.NewInt(1), 128))
+ Expect(err).NotTo(HaveOccurred())
+ return n
+ }
+ caKey, err := ecdsa.GenerateKey(elliptic.P256(), rand.Reader)
+ Expect(err).NotTo(HaveOccurred())
+ caTmpl := &x509.Certificate{
+ SerialNumber: serial(),
+ Subject: pkix.Name{CommonName: "apisix-ingress-e2e-ca"},
+ NotBefore: time.Now().Add(-time.Hour),
+ NotAfter: time.Now().Add(24 * time.Hour),
+ KeyUsage: x509.KeyUsageCertSign | x509.KeyUsageDigitalSignature,
+ BasicConstraintsValid: true,
+ IsCA: true,
+ }
+ caDER, err := x509.CreateCertificate(rand.Reader, caTmpl, caTmpl, &caKey.PublicKey, caKey)
+ Expect(err).NotTo(HaveOccurred())
+
+ serverKeyECDSA, err := ecdsa.GenerateKey(elliptic.P256(), rand.Reader)
+ Expect(err).NotTo(HaveOccurred())
+ serverTmpl := &x509.Certificate{
+ SerialNumber: serial(),
+ Subject: pkix.Name{CommonName: dnsNames[0]},
+ NotBefore: time.Now().Add(-time.Hour),
+ NotAfter: time.Now().Add(24 * time.Hour),
+ KeyUsage: x509.KeyUsageDigitalSignature | x509.KeyUsageKeyEncipherment,
+ ExtKeyUsage: []x509.ExtKeyUsage{x509.ExtKeyUsageServerAuth},
+ DNSNames: dnsNames,
+ }
+ serverDER, err := x509.CreateCertificate(rand.Reader, serverTmpl, caTmpl, &serverKeyECDSA.PublicKey, caKey)
+ Expect(err).NotTo(HaveOccurred())
+
+ keyDER, err := x509.MarshalPKCS8PrivateKey(serverKeyECDSA)
+ Expect(err).NotTo(HaveOccurred())
+
+ return string(pem.EncodeToMemory(&pem.Block{Type: "CERTIFICATE", Bytes: caDER})),
+ string(pem.EncodeToMemory(&pem.Block{Type: "CERTIFICATE", Bytes: serverDER})),
+ string(pem.EncodeToMemory(&pem.Block{Type: "PRIVATE KEY", Bytes: keyDER}))
+}