Feat/k8s config etcd encryption

internal/app/machined/pkg/controllers/k8s/render_secrets_static_pod.go

@@ -23,6 +23,7 @@ import (
 	"github.com/siderolabs/talos/internal/app/machined/pkg/controllers/k8s/internal/k8stemplates"
 	"github.com/siderolabs/talos/internal/pkg/selinux"
 	"github.com/siderolabs/talos/pkg/machinery/constants"
+	"github.com/siderolabs/talos/pkg/machinery/resources/config"
 	"github.com/siderolabs/talos/pkg/machinery/resources/k8s"
 	"github.com/siderolabs/talos/pkg/machinery/resources/secrets"
 )
@@ -68,6 +69,12 @@ func (ctrl *RenderSecretsStaticPodController) Inputs() []controller.Input {
 			ID:        optional.Some(secrets.EtcdID),
 			Kind:      controller.InputWeak,
 		},
+		{
+			Namespace: config.NamespaceName,
+			Type:      config.MachineConfigType,
+			ID:        optional.Some(config.ActiveID),
+			Kind:      controller.InputWeak,
+		},
 	}
 }
 
@@ -140,6 +147,15 @@ func (ctrl *RenderSecretsStaticPodController) Run(ctx context.Context, r control
 			return fmt.Errorf("error getting secrets resource: %w", err)
 		}
 
+		cfg, err := safe.ReaderGetByID[*config.MachineConfig](ctx, r, config.ActiveID)
+		if err != nil {
+			if state.IsNotFoundError(err) {
+				continue
+			}
+
+			return fmt.Errorf("error getting machine config to check for custom etcd encryptionconfig: %w", err)
+		}
+
 		rootEtcdSecrets := rootEtcdRes.TypedSpec()
 		rootK8sSecrets := rootK8sRes.TypedSpec()
 		etcdSecrets := etcdRes.TypedSpec()
@@ -229,6 +245,14 @@ func (ctrl *RenderSecretsStaticPodController) Run(ctx context.Context, r control
 					{
 						filename: "encryptionconfig.yaml",
 						contentFunc: func() ([]byte, error) {
+							if cfg != nil {
+								customEtcdEncryption := cfg.Config().EtcdEncryption()
+
+								if customEtcdEncryption != nil {
+									return []byte(customEtcdEncryption.EtcdEncryptionConfig()), nil
+								}
+							}
+
 							return k8stemplates.Marshal(k8stemplates.APIServerEncryptionConfig(rootK8sSecrets))
 						},
 					},

internal/integration/api/etcd-encryption.go

@@ -0,0 +1,104 @@
+// This Source Code Form is subject to the terms of the Mozilla Public
+// License, v. 2.0. If a copy of the MPL was not distributed with this
+// file, You can obtain one at http://mozilla.org/MPL/2.0/.
+
+//go:build integration_api
+
+package api
+
+import (
+	"context"
+	"io"
+	"strings"
+	"time"
+
+	"github.com/siderolabs/talos/internal/integration/base"
+	"github.com/siderolabs/talos/pkg/machinery/client"
+	"github.com/siderolabs/talos/pkg/machinery/config/machine"
+	"github.com/siderolabs/talos/pkg/machinery/config/types/k8s"
+	"github.com/siderolabs/talos/pkg/machinery/constants"
+)
+
+// EtcdEncryptionSuite ...
+type EtcdEncryptionSuite struct {
+	base.APISuite
+
+	ctx       context.Context //nolint:containedctx
+	ctxCancel context.CancelFunc
+}
+
+// SuiteName ...
+func (suite *EtcdEncryptionSuite) SuiteName() string {
+	return "api.EtcdEncryptionSuite"
+}
+
+// SetupTest ...
+func (suite *EtcdEncryptionSuite) SetupTest() {
+	suite.ctx, suite.ctxCancel = context.WithTimeout(context.Background(), 1*time.Minute)
+}
+
+// TearDownTest ...
+func (suite *EtcdEncryptionSuite) TearDownTest() {
+	if suite.ctxCancel != nil {
+		suite.ctxCancel()
+	}
+}
+
+func (suite *EtcdEncryptionSuite) readEtcdEncryptionConfig(nodeCtx context.Context) string {
+	r, err := suite.Client.Read(nodeCtx, constants.KubernetesAPIServerSecretsDir+"/encryptionconfig.yaml")
+	suite.Require().NoError(err)
+
+	value, err := io.ReadAll(r)
+	suite.Require().NoError(err)
+
+	suite.Require().NoError(r.Close())
+
+	return string(value)
+}
+
+// TestEtcdEncryption verifies default and custom trusted CA roots.
+func (suite *EtcdEncryptionSuite) TestEtcdEncryption() {
+	// pick up a random node to test the EtcdEncryption on, and use it throughout the test
+	node := suite.RandomDiscoveredNodeInternalIP(machine.TypeControlPlane)
+
+	suite.T().Logf("testing EtcdEncryption on node %s", node)
+
+	// build a Talos API context which is tied to the node
+	nodeCtx := client.WithNode(suite.ctx, node)
+
+	cfgDocument := k8s.NewEtcdEncryptionConfigV1Alpha1()
+	cfgDocument.Config = `
+apiVersion: apiserver.config.k8s.io/v1
+kind: EncryptionConfiguration
+resources:
+  - resources:
+      - secrets
+    providers:
+      - aescbc:
+        keys:
+          - name: key1
+            secret: c2VjcmV0IGlzIHNlY3VyZQ==
+      - identity: {}
+`
+
+	// clean up custom config if it exists
+	suite.RemoveMachineConfigDocuments(nodeCtx, cfgDocument.MetaKind)
+
+	// enable custom etcd encryption
+	suite.PatchMachineConfig(nodeCtx, cfgDocument)
+
+	suite.Require().Eventually(func() bool {
+		return strings.Contains(suite.readEtcdEncryptionConfig(nodeCtx), "c2VjcmV0IGlzIHNlY3VyZQ==")
+	}, 5*time.Second, 100*time.Millisecond)
+
+	// deactivate the EtcdEncryption
+	suite.RemoveMachineConfigDocuments(nodeCtx, cfgDocument.MetaKind)
+
+	suite.Require().Eventually(func() bool {
+		return !strings.Contains(suite.readEtcdEncryptionConfig(nodeCtx), "c2VjcmV0IGlzIHNlY3VyZQ==")
+	}, 5*time.Second, 100*time.Millisecond)
+}
+
+func init() {
+	allSuites = append(allSuites, new(EtcdEncryptionSuite))
+}

pkg/machinery/config/config/config.go

@@ -24,4 +24,5 @@ type Config interface { //nolint:interfacebloat
 	ExistingVolumeConfigs() []ExistingVolumeConfig
 	SwapVolumeConfigs() []SwapVolumeConfig
 	ZswapConfig() ZswapConfig
+	EtcdEncryption() EtcdEncryptionConfig
 }

pkg/machinery/config/config/k8s.go

@@ -0,0 +1,10 @@
+// This Source Code Form is subject to the terms of the Mozilla Public
+// License, v. 2.0. If a copy of the MPL was not distributed with this
+// file, You can obtain one at http://mozilla.org/MPL/2.0/.
+
+package config
+
+// EtcdEncryptionConfig defines the interface to access etcd encryption configuration.
+type EtcdEncryptionConfig interface {
+	EtcdEncryptionConfig() string
+}

pkg/machinery/config/container/container.go

@@ -239,6 +239,16 @@ func (container *Container) PCIDriverRebindConfig() config.PCIDriverRebindConfig
 	return config.WrapPCIDriverRebindConfig(findMatchingDocs[config.PCIDriverRebindConfig](container.documents)...)
 }
 
+// EtcdEncryption implements config.Config interface.
+func (container *Container) EtcdEncryption() config.EtcdEncryptionConfig {
+	matching := findMatchingDocs[config.EtcdEncryptionConfig](container.documents)
+	if len(matching) == 0 {
+		return nil
+	}
+
+	return matching[0]
+}
+
 // EthernetConfigs implements config.Config interface.
 func (container *Container) EthernetConfigs() []config.EthernetConfig {
 	return findMatchingDocs[config.EthernetConfig](container.documents)

pkg/machinery/config/types/k8s/etcd_encryption.go

@@ -0,0 +1,89 @@
+// This Source Code Form is subject to the terms of the Mozilla Public
+// License, v. 2.0. If a copy of the MPL was not distributed with this
+// file, You can obtain one at http://mozilla.org/MPL/2.0/.
+
+package k8s
+
+//docgen:jsonschema
+
+import (
+	"github.com/siderolabs/talos/pkg/machinery/config/config"
+	"github.com/siderolabs/talos/pkg/machinery/config/internal/registry"
+	"github.com/siderolabs/talos/pkg/machinery/config/types/meta"
+)
+
+// EtcdEncryptionConfig is a default action config document kind.
+const EtcdEncryptionConfig = "EtcdEncryptionConfig"
+
+func init() {
+	registry.Register(EtcdEncryptionConfig, func(version string) config.Document {
+		switch version {
+		case "v1alpha1":
+			return &EtcdEncryptionConfigV1Alpha1{}
+		default:
+			return nil
+		}
+	})
+}
+
+// Check interfaces.
+var (
+	_ config.EtcdEncryptionConfig = &EtcdEncryptionConfigV1Alpha1{}
+)
+
+// EtcdEncryptionConfigV1Alpha1 allows to configure etcd encryption.
+//
+//	examples:
+//	  - value: exampleEtcdEncryptionConfigV1Alpha1()
+//	alias: EtcdEncryptionConfig
+//	schemaRoot: true
+//	schemaMeta: v1alpha1/EtcdEncryptionConfig
+type EtcdEncryptionConfigV1Alpha1 struct {
+	meta.Meta `yaml:",inline"`
+
+	//   description: |
+	//     Custom API server etcd encryption configuration document.
+	//     https://kubernetes.io/docs/reference/Config-api/apiserver-Config.v1/#apiserver-Config-k8s-io-v1-EncryptionConfiguration
+	//
+	Config string `yaml:"config"`
+}
+
+// NewEtcdEncryptionConfigV1Alpha1 creates a new EtcdEncryptionConfig config document.
+func NewEtcdEncryptionConfigV1Alpha1() *EtcdEncryptionConfigV1Alpha1 {
+	return &EtcdEncryptionConfigV1Alpha1{
+		Meta: meta.Meta{
+			MetaKind:       EtcdEncryptionConfig,
+			MetaAPIVersion: "v1alpha1",
+		},
+	}
+}
+
+func exampleEtcdEncryptionConfigV1Alpha1() *EtcdEncryptionConfigV1Alpha1 {
+	cfg := NewEtcdEncryptionConfigV1Alpha1()
+	cfg.Config = `---
+apiVersion: apiserver.config.k8s.io/v1
+kind: EncryptionConfiguration
+resources:
+  - resources:
+      - secrets
+    providers:
+      - aescbc:
+          keys:
+            - name: key1
+              secret: <BASE 64 ENCODED SECRET>
+      - identity: {} # this fallback allows reading unencrypted secrets;
+                     # for example, during initial migration
+`
+
+	return cfg
+}
+
+// Clone implements config.Document interface.
+func (s *EtcdEncryptionConfigV1Alpha1) Clone() config.Document {
+	return s.DeepCopy()
+}
+
+// EtcdEncryptionConfig implements config.EtcdEncryptionConfig interface.
+func (s *EtcdEncryptionConfigV1Alpha1) EtcdEncryptionConfig() string {
+	return s.Config
+}

pkg/machinery/config/types/k8s/k8s.go

@@ -0,0 +1,10 @@
+// This Source Code Form is subject to the terms of the Mozilla Public
+// License, v. 2.0. If a copy of the MPL was not distributed with this
+// file, You can obtain one at http://mozilla.org/MPL/2.0/.
+
+// Package k8s provides k8s-related machine configuration documents.
+package k8s
+
+//go:generate go tool github.com/siderolabs/talos/tools/docgen -output k8s_doc.go k8s.go etcd_encryption.go
+
+//go:generate go tool github.com/siderolabs/deep-copy -type EtcdEncryptionConfigV1Alpha1 -pointer-receiver -header-file ../../../../../hack/boilerplate.txt -o deep_copy.generated.go .

pkg/machinery/config/types/types.go

@@ -10,6 +10,7 @@ package types
 import (
 	_ "github.com/siderolabs/talos/pkg/machinery/config/types/block"              // import config types to register them
 	_ "github.com/siderolabs/talos/pkg/machinery/config/types/hardware"           // import config types to register them
+	_ "github.com/siderolabs/talos/pkg/machinery/config/types/k8s"                // import config types to register them
 	_ "github.com/siderolabs/talos/pkg/machinery/config/types/network"            // import config types to register them
 	_ "github.com/siderolabs/talos/pkg/machinery/config/types/runtime"            // import config types to register them
 	_ "github.com/siderolabs/talos/pkg/machinery/config/types/runtime/extensions" // import config types to register them