Skip to content

Commit 9e89937

Browse files
prometherionprometherion
committed
feat: support to datastore migration w/ the same driver
1 parent a260a92 commit 9e89937

File tree

9 files changed

+396
-38
lines changed

9 files changed

+396
-38
lines changed

‎api/v1alpha1/tenantcontrolplane_status.go

+2-1
Original file line numberDiff line numberDiff line change
@@ -187,12 +187,13 @@ type KubernetesStatus struct {
187187
Ingress *KubernetesIngressStatus `json:"ingress,omitempty"`
188188
}
189189

190-
// +kubebuilder:validation:Enum=Provisioning;Upgrading;Ready;NotReady
190+
// +kubebuilder:validation:Enum=Provisioning;Upgrading;Migrating;Ready;NotReady
191191
type KubernetesVersionStatus string
192192

193193
var (
194194
VersionProvisioning KubernetesVersionStatus = "Provisioning"
195195
VersionUpgrading KubernetesVersionStatus = "Upgrading"
196+
VersionMigrating KubernetesVersionStatus = "Migrating"
196197
VersionReady KubernetesVersionStatus = "Ready"
197198
VersionNotReady KubernetesVersionStatus = "NotReady"
198199
)

‎cmd/manager/cmd.go

+31-13
Original file line numberDiff line numberDiff line change
@@ -7,6 +7,7 @@ import (
77
"flag"
88
"fmt"
99
"io"
10+
"os"
1011
goRuntime "runtime"
1112

1213
"github.com/spf13/cobra"
@@ -32,17 +33,28 @@ func NewCmd(scheme *runtime.Scheme) *cobra.Command {
3233
tmpDirectory string
3334
kineImage string
3435
datastore string
36+
managerNamespace string
37+
serviceAccountName string
3538
)
3639

3740
cmd := &cobra.Command{
3841
Use: "manager",
3942
Short: "Start the Kamaji Kubernetes Operator",
40-
SilenceErrors: true,
43+
SilenceErrors: false,
4144
SilenceUsage: true,
42-
PreRun: func(cmd *cobra.Command, args []string) {
45+
PreRunE: func(cmd *cobra.Command, args []string) error {
4346
// Avoid to pollute Kamaji stdout with useless details by the underlying klog implementations
4447
klog.SetOutput(io.Discard)
4548
klog.LogToStderr(false)
49+
50+
for _, arg := range []string{"pod-namespace", "serviceaccount-name", "datastore", "kine-image", "tmp-directory"} {
51+
v, _ := cmd.Flags().GetString(arg)
52+
if len(v) == 0 {
53+
return fmt.Errorf("expecting a value for --%s arg", arg)
54+
}
55+
}
56+
57+
return nil
4658
},
4759
RunE: func(cmd *cobra.Command, args []string) error {
4860
setupLog := ctrl.Log.WithName("setup")
@@ -56,12 +68,13 @@ func NewCmd(scheme *runtime.Scheme) *cobra.Command {
5668
setupLog.Info(fmt.Sprintf("Go OS/Arch: %s/%s", goRuntime.GOOS, goRuntime.GOARCH))
5769

5870
mgr, err := ctrl.NewManager(ctrl.GetConfigOrDie(), ctrl.Options{
59-
Scheme: scheme,
60-
MetricsBindAddress: metricsBindAddress,
61-
Port: 9443,
62-
HealthProbeBindAddress: healthProbeBindAddress,
63-
LeaderElection: leaderElect,
64-
LeaderElectionID: "799b98bc.clastix.io",
71+
Scheme: scheme,
72+
MetricsBindAddress: metricsBindAddress,
73+
Port: 9443,
74+
HealthProbeBindAddress: healthProbeBindAddress,
75+
LeaderElection: leaderElect,
76+
LeaderElectionNamespace: managerNamespace,
77+
LeaderElectionID: "799b98bc.clastix.io",
6578
})
6679
if err != nil {
6780
setupLog.Error(err, "unable to start manager")
@@ -78,14 +91,17 @@ func NewCmd(scheme *runtime.Scheme) *cobra.Command {
7891
}
7992

8093
reconciler := &controllers.TenantControlPlaneReconciler{
81-
Client: mgr.GetClient(),
82-
Scheme: mgr.GetScheme(),
94+
Client: mgr.GetClient(),
95+
APIReader: mgr.GetAPIReader(),
96+
Scheme: mgr.GetScheme(),
8397
Config: controllers.TenantControlPlaneReconcilerConfig{
8498
DefaultDataStoreName: datastore,
8599
KineContainerImage: kineImage,
86100
TmpBaseDirectory: tmpDirectory,
87101
},
88-
TriggerChan: tcpChannel,
102+
TriggerChan: tcpChannel,
103+
KamajiNamespace: managerNamespace,
104+
KamajiServiceAccount: serviceAccountName,
89105
}
90106

91107
if err = reconciler.SetupWithManager(mgr); err != nil {
@@ -145,8 +161,10 @@ func NewCmd(scheme *runtime.Scheme) *cobra.Command {
145161
cmd.Flags().StringVar(&healthProbeBindAddress, "health-probe-bind-address", ":8081", "The address the probe endpoint binds to.")
146162
cmd.Flags().BoolVar(&leaderElect, "leader-elect", true, "Enable leader election for controller manager. Enabling this will ensure there is only one active controller manager.")
147163
cmd.Flags().StringVar(&tmpDirectory, "tmp-directory", "/tmp/kamaji", "Directory which will be used to work with temporary files.")
148-
cmd.Flags().StringVar(&kineImage, "kine-image", "rancher/kine:v0.9.2-amd64", "Container image along with tag to use for the Kine sidecar container (used only if etcd-storage-type is set to one of kine strategies)")
149-
cmd.Flags().StringVar(&datastore, "datastore", "etcd", "The default DataStore that should be used by Kamaji to setup the required storage")
164+
cmd.Flags().StringVar(&kineImage, "kine-image", "rancher/kine:v0.9.2-amd64", "Container image along with tag to use for the Kine sidecar container (used only if etcd-storage-type is set to one of kine strategies).")
165+
cmd.Flags().StringVar(&datastore, "datastore", "etcd", "The default DataStore that should be used by Kamaji to setup the required storage.")
166+
cmd.Flags().StringVar(&managerNamespace, "pod-namespace", os.Getenv("POD_NAMESPACE"), "The Kubernetes Namespace on which the Operator is running in, required for the TenantControlPlane migration jobs.")
167+
cmd.Flags().StringVar(&serviceAccountName, "serviceaccount-name", os.Getenv("SERVICE_ACCOUNT"), "The Kubernetes Namespace on which the Operator is running in, required for the TenantControlPlane migration jobs.")
150168

151169
cobra.OnInitialize(func() {
152170
viper.AutomaticEnv()

‎cmd/migrate/cmd.go

+119
Original file line numberDiff line numberDiff line change
@@ -0,0 +1,119 @@
1+
// Copyright 2022 Clastix Labs
2+
// SPDX-License-Identifier: Apache-2.0
3+
4+
package migrate
5+
6+
import (
7+
"context"
8+
"fmt"
9+
"strings"
10+
"time"
11+
12+
"github.com/spf13/cobra"
13+
"k8s.io/apimachinery/pkg/runtime"
14+
"k8s.io/apimachinery/pkg/types"
15+
ctrl "sigs.k8s.io/controller-runtime"
16+
ctrlclient "sigs.k8s.io/controller-runtime/pkg/client"
17+
18+
kamajiv1alpha1 "github.com/clastix/kamaji/api/v1alpha1"
19+
"github.com/clastix/kamaji/internal/datastore"
20+
)
21+
22+
func NewCmd(scheme *runtime.Scheme) *cobra.Command {
23+
// CLI flags
24+
var (
25+
tenantControlPlane string
26+
targetDataStore string
27+
timeout time.Duration
28+
)
29+
30+
cmd := &cobra.Command{
31+
Use: "migrate",
32+
Short: "Migrate the data of a TenantControlPlane to another compatible DataStore",
33+
SilenceUsage: true,
34+
RunE: func(cmd *cobra.Command, args []string) error {
35+
ctx, cancelFn := context.WithTimeout(context.Background(), timeout)
36+
defer cancelFn()
37+
38+
log := ctrl.Log
39+
40+
log.Info("generating the controller-runtime client")
41+
42+
client, err := ctrlclient.New(ctrl.GetConfigOrDie(), ctrlclient.Options{
43+
Scheme: scheme,
44+
})
45+
if err != nil {
46+
return err
47+
}
48+
49+
parts := strings.Split(tenantControlPlane, string(types.Separator))
50+
if len(parts) != 2 {
51+
return fmt.Errorf("non well-formed namespaced name for the tenant control plane, expected <NAMESPACE>/NAME, fot %s", tenantControlPlane)
52+
}
53+
54+
log.Info("retrieving the TenantControlPlane")
55+
56+
tcp := &kamajiv1alpha1.TenantControlPlane{}
57+
if err = client.Get(ctx, types.NamespacedName{Namespace: parts[0], Name: parts[1]}, tcp); err != nil {
58+
return err
59+
}
60+
61+
log.Info("retrieving the TenantControlPlane used DataStore")
62+
63+
originDs := &kamajiv1alpha1.DataStore{}
64+
if err = client.Get(ctx, types.NamespacedName{Name: tcp.Status.Storage.DataStoreName}, originDs); err != nil {
65+
return err
66+
}
67+
68+
log.Info("retrieving the target DataStore")
69+
70+
targetDs := &kamajiv1alpha1.DataStore{}
71+
if err = client.Get(ctx, types.NamespacedName{Name: targetDataStore}, targetDs); err != nil {
72+
return err
73+
}
74+
75+
if tcp.Status.Storage.Driver != string(targetDs.Spec.Driver) {
76+
return fmt.Errorf("migration between DataStore with different driver is not supported")
77+
}
78+
79+
if tcp.Status.Storage.DataStoreName == targetDs.GetName() {
80+
return fmt.Errorf("cannot migrate to the same DataStore")
81+
}
82+
83+
log.Info("generating the origin storage connection")
84+
85+
originConnection, err := datastore.NewStorageConnection(ctx, client, *originDs)
86+
if err != nil {
87+
return err
88+
}
89+
defer originConnection.Close()
90+
91+
log.Info("generating the target storage connection")
92+
93+
targetConnection, err := datastore.NewStorageConnection(ctx, client, *targetDs)
94+
if err != nil {
95+
return err
96+
}
97+
defer targetConnection.Close()
98+
// Start migrating from the old Datastore to the new one
99+
log.Info("migration from origin to target started")
100+
101+
if err = originConnection.Migrate(ctx, *tcp, targetConnection); err != nil {
102+
return fmt.Errorf("unable to migrate data from %s to %s: %w", originDs.GetName(), targetDs.GetName(), err)
103+
}
104+
105+
log.Info("migration completed")
106+
107+
return nil
108+
},
109+
}
110+
111+
cmd.Flags().StringVar(&tenantControlPlane, "tenant-control-plane", "", "Namespaced-name of the TenantControlPlane that must be migrated (e.g.: default/test)")
112+
cmd.Flags().StringVar(&targetDataStore, "target-datastore", "", "Name of the Datastore to which the TenantControlPlane will be migrated")
113+
cmd.Flags().DurationVar(&timeout, "timeout", 5*time.Minute, "Amount of time for the context timeout")
114+
115+
_ = cmd.MarkFlagRequired("tenant-control-plane")
116+
_ = cmd.MarkFlagRequired("target-datastore")
117+
118+
return cmd
119+
}

‎controllers/resources.go

+32-7
Original file line numberDiff line numberDiff line change
@@ -21,12 +21,14 @@ import (
2121
)
2222

2323
type GroupResourceBuilderConfiguration struct {
24-
client client.Client
25-
log logr.Logger
26-
tcpReconcilerConfig TenantControlPlaneReconcilerConfig
27-
tenantControlPlane kamajiv1alpha1.TenantControlPlane
28-
Connection datastore.Connection
29-
DataStore kamajiv1alpha1.DataStore
24+
client client.Client
25+
log logr.Logger
26+
tcpReconcilerConfig TenantControlPlaneReconcilerConfig
27+
tenantControlPlane kamajiv1alpha1.TenantControlPlane
28+
Connection datastore.Connection
29+
DataStore kamajiv1alpha1.DataStore
30+
KamajiNamespace string
31+
KamajiServiceAccount string
3032
}
3133

3234
type GroupDeletableResourceBuilderConfiguration struct {
@@ -61,13 +63,16 @@ func GetDeletableResources(tcp *kamajiv1alpha1.TenantControlPlane, config GroupD
6163
}
6264

6365
func getDefaultResources(config GroupResourceBuilderConfiguration) []resources.Resource {
64-
resources := append(getUpgradeResources(config.client), getKubernetesServiceResources(config.client)...)
66+
resources := getDataStoreMigratingResources(config.client, config.KamajiNamespace, config.KamajiServiceAccount)
67+
resources = append(resources, getUpgradeResources(config.client)...)
68+
resources = append(resources, getKubernetesServiceResources(config.client)...)
6569
resources = append(resources, getKubeadmConfigResources(config.client, getTmpDirectory(config.tcpReconcilerConfig.TmpBaseDirectory, config.tenantControlPlane), config.DataStore)...)
6670
resources = append(resources, getKubernetesCertificatesResources(config.client, config.tcpReconcilerConfig, config.tenantControlPlane)...)
6771
resources = append(resources, getKubeconfigResources(config.client, config.tcpReconcilerConfig, config.tenantControlPlane)...)
6872
resources = append(resources, getKubernetesStorageResources(config.client, config.Connection, config.DataStore)...)
6973
resources = append(resources, getInternalKonnectivityResources(config.client)...)
7074
resources = append(resources, getKubernetesDeploymentResources(config.client, config.tcpReconcilerConfig, config.DataStore)...)
75+
resources = append(resources, getDataStoreMigratingCleanup(config.client, config.KamajiNamespace)...)
7176
resources = append(resources, getKubernetesIngressResources(config.client)...)
7277
resources = append(resources, getKubeadmPhaseResources(config.client)...)
7378
resources = append(resources, getKubeadmAddonResources(config.client)...)
@@ -76,6 +81,26 @@ func getDefaultResources(config GroupResourceBuilderConfiguration) []resources.R
7681
return resources
7782
}
7883

84+
func getDataStoreMigratingCleanup(c client.Client, kamajiNamespace string) []resources.Resource {
85+
return []resources.Resource{
86+
&resources.DatastoreMigrate{
87+
Client: c,
88+
KamajiNamespace: kamajiNamespace,
89+
ShouldCleanUp: true,
90+
},
91+
}
92+
}
93+
94+
func getDataStoreMigratingResources(c client.Client, kamajiNamespace, kamajiServiceAccount string) []resources.Resource {
95+
return []resources.Resource{
96+
&resources.DatastoreMigrate{
97+
Client: c,
98+
KamajiNamespace: kamajiNamespace,
99+
KamajiServiceAccount: kamajiServiceAccount,
100+
},
101+
}
102+
}
103+
79104
func getUpgradeResources(c client.Client) []resources.Resource {
80105
return []resources.Resource{
81106
&resources.KubernetesUpgrade{

0 commit comments

Comments
 (0)