diff --git a/cli/cmd/backup.go b/cli/cmd/backup.go index 20f248f2..ed82dc6f 100644 --- a/cli/cmd/backup.go +++ b/cli/cmd/backup.go @@ -29,7 +29,7 @@ import ( controllers "github.com/softwarefactory-project/sf-operator/controllers" "github.com/spf13/cobra" "gopkg.in/yaml.v3" - "k8s.io/client-go/kubernetes" + apiv1 "k8s.io/api/core/v1" ctrl "sigs.k8s.io/controller-runtime" ) @@ -64,29 +64,15 @@ var SecretsToBackup = []string{ "zuul-auth-secret", } -func prepareBackup(kmd *cobra.Command, backupDir string) (string, *kubernetes.Clientset, string) { - - cliCtx, err := cliutils.GetCLIContext(kmd) - if err != nil { - ctrl.Log.Error(err, "Error initializing CLI:") - os.Exit(1) - } - - cliutils.CreateDirectory(backupDir, 0755) - - kubeContext := cliCtx.KubeContext - _, kubeClientSet := cliutils.GetClientset(kubeContext) - return cliCtx.Namespace, kubeClientSet, kubeContext -} - -func createSecretBackup(ns string, backupDir string, kubeClientSet *kubernetes.Clientset) { +func createSecretBackup(backupDir string, env cliutils.ENV) { ctrl.Log.Info("Creating secrets backup...") secretsDir := backupDir + "/" + SecretsBackupPath cliutils.CreateDirectory(secretsDir, 0755) - for _, sec := range SecretsToBackup { - secret := cliutils.GetSecretByName(sec, ns, kubeClientSet) + for _, secName := range SecretsToBackup { + secret := apiv1.Secret{} + cliutils.GetMOrDie(&env, secName, &secret) // convert secret content to string (was bytes) strMap := cliutils.ConvertMapOfBytesToMapOfStrings(secret.Data) @@ -115,12 +101,12 @@ func createSecretBackup(ns string, backupDir string, kubeClientSet *kubernetes.C ctrl.Log.Info("Finished doing secret backup!") } -func createZuulKeypairBackup(ns string, backupDir string, kubeClientSet *kubernetes.Clientset, - kubeContext string) { +func createZuulKeypairBackup(backupDir string, kubeContext string, env cliutils.ENV) { ctrl.Log.Info("Doing Zuul keys backup...") - pod := cliutils.GetPodByName(zuulBackupPod, ns, kubeClientSet) + pod := apiv1.Pod{} + cliutils.GetMOrDie(&env, zuulBackupPod, &pod) // https://zuul-ci.org/docs/zuul/latest/client.html zuulBackupPath := backupDir + "/" + ZuulBackupPath @@ -141,22 +127,21 @@ func createZuulKeypairBackup(ns string, backupDir string, kubeClientSet *kuberne } // Execute command for backup - cliutils.RunRemoteCmd(kubeContext, ns, pod.Name, controllers.ZuulSchedulerIdent, backupZuulCMD) + cliutils.RunRemoteCmd(kubeContext, env.Ns, pod.Name, controllers.ZuulSchedulerIdent, backupZuulCMD) // Take output of the backup - commandBuffer := cliutils.RunRemoteCmd(kubeContext, ns, pod.Name, controllers.ZuulSchedulerIdent, backupZuulPrintCMD) + commandBuffer := cliutils.RunRemoteCmd(kubeContext, env.Ns, pod.Name, controllers.ZuulSchedulerIdent, backupZuulPrintCMD) // write stdout to file cliutils.WriteContentToFile(zuulBackupPath, commandBuffer.Bytes(), 0640) // Remove key file from the pod - cliutils.RunRemoteCmd(kubeContext, ns, pod.Name, controllers.ZuulSchedulerIdent, backupZuulRemoveCMD) + cliutils.RunRemoteCmd(kubeContext, env.Ns, pod.Name, controllers.ZuulSchedulerIdent, backupZuulRemoveCMD) ctrl.Log.Info("Finished doing Zuul private keys backup!") } -func createMySQLBackup(ns string, backupDir string, kubeClientSet *kubernetes.Clientset, - kubeContext string) { +func createMySQLBackup(backupDir string, kubeContext string, env cliutils.ENV) { ctrl.Log.Info("Doing DB backup...") // create MariaDB dir @@ -165,7 +150,8 @@ func createMySQLBackup(ns string, backupDir string, kubeClientSet *kubernetes.Cl cliutils.CreateDirectory(mariaDBBackupDir, 0755) - pod := cliutils.GetPodByName(dbBackupPod, ns, kubeClientSet) + pod := apiv1.Pod{} + cliutils.GetMOrDie(&env, dbBackupPod, &pod) // NOTE: We use option: --single-transaction to avoid error: // "The user specified as a definer ('mariadb.sys'@'localhost') does not exist" when using LOCK TABLES @@ -177,7 +163,7 @@ func createMySQLBackup(ns string, backupDir string, kubeClientSet *kubernetes.Cl } // just create Zuul DB backup - commandBuffer := cliutils.RunRemoteCmd(kubeContext, ns, pod.Name, controllers.MariaDBIdent, backupZuulCMD) + commandBuffer := cliutils.RunRemoteCmd(kubeContext, env.Ns, pod.Name, controllers.MariaDBIdent, backupZuulCMD) // write stdout to file cliutils.WriteContentToFile(mariadbBackupPath, commandBuffer.Bytes(), 0640) @@ -192,24 +178,25 @@ func backupCmd(kmd *cobra.Command, args []string) { os.Exit(1) } - // prepare to make backup - ns, kubeClientSet, kubeContext := prepareBackup(kmd, backupDir) + cliutils.CreateDirectory(backupDir, 0755) + + kubeContext, env := cliutils.GetCLIENV(kmd) - if ns == "" { + if env.Ns == "" { ctrl.Log.Error(errors.New("no namespace set"), "You need to specify the namespace!") os.Exit(1) } - ctrl.Log.Info("Starting backup process for services in namespace: " + ns) + ctrl.Log.Info("Starting backup process for services in namespace: " + env.Ns) // create secret backup - createSecretBackup(ns, backupDir, kubeClientSet) + createSecretBackup(backupDir, env) // create zuul backup - createZuulKeypairBackup(ns, backupDir, kubeClientSet, kubeContext) + createZuulKeypairBackup(backupDir, kubeContext, env) // create DB backup - createMySQLBackup(ns, backupDir, kubeClientSet, kubeContext) + createMySQLBackup(backupDir, kubeContext, env) } diff --git a/cli/cmd/restore.go b/cli/cmd/restore.go index 81293b09..a07aac12 100644 --- a/cli/cmd/restore.go +++ b/cli/cmd/restore.go @@ -21,7 +21,6 @@ package cmd */ import ( - "context" "errors" "fmt" "os" @@ -33,39 +32,19 @@ import ( "github.com/spf13/cobra" appsv1 "k8s.io/api/apps/v1" - corev1 "k8s.io/api/core/v1" + apiv1 "k8s.io/api/core/v1" metav1 "k8s.io/apimachinery/pkg/apis/meta/v1" - "k8s.io/client-go/kubernetes" ctrl "sigs.k8s.io/controller-runtime" ) -func prepareRestore(kmd *cobra.Command) (string, *kubernetes.Clientset, string) { - - cliCtx, err := cliutils.GetCLIContext(kmd) - if err != nil { - ctrl.Log.Error(err, "Error initializing CLI:") - os.Exit(1) - } - - kubeContext := cliCtx.KubeContext - _, kubeClientSet := cliutils.GetClientset(kubeContext) - return cliCtx.Namespace, kubeClientSet, kubeContext -} - -func restoreSecret(ns string, backupDir string, kubeContext string) { +func restoreSecret(backupDir string, env cliutils.ENV) { ctrl.Log.Info("Restoring secrets...") - env := cliutils.ENV{ - Cli: cliutils.CreateKubernetesClientOrDie(kubeContext), - Ctx: context.TODO(), - Ns: ns, - } - for _, sec := range SecretsToBackup { pathToSecret := backupDir + "/" + SecretsBackupPath + "/" + sec + ".yaml" secretContent := cliutils.ReadYAMLToMapOrDie(pathToSecret) - var secret corev1.Secret + secret := apiv1.Secret{} if cliutils.GetMOrDie(&env, sec, &secret) { secretMap := secretContent["data"].(map[string]interface{}) for key, value := range secretMap { @@ -88,16 +67,17 @@ func restoreSecret(ns string, backupDir string, kubeContext string) { } -func restoreDB(ns string, backupDir string, kubeClientSet *kubernetes.Clientset, kubeContext string) { +func restoreDB(backupDir string, kubeContext string, env cliutils.ENV) { ctrl.Log.Info("Restoring DB...") - pod := cliutils.GetPodByName(dbBackupPod, ns, kubeClientSet) + pod := apiv1.Pod{} + cliutils.GetMOrDie(&env, dbBackupPod, &pod) kubectlPath := cliutils.GetKubectlPath() dropDBCMD := []string{ "mysql", "-e DROP DATABASE zuul;", } - cliutils.RunRemoteCmd(kubeContext, ns, pod.Name, controllers.MariaDBIdent, dropDBCMD) + cliutils.RunRemoteCmd(kubeContext, env.Ns, pod.Name, controllers.MariaDBIdent, dropDBCMD) mariadbBackupPath := backupDir + "/" + DBBackupPath @@ -106,32 +86,22 @@ func restoreDB(ns string, backupDir string, kubeClientSet *kubernetes.Clientset, // but in that case, we need to do it via system kubernetes client. executeCommand := fmt.Sprintf( "cat %s | %s -n %s exec -it %s -c %s -- sh -c \"mysql -h0\"", - mariadbBackupPath, kubectlPath, ns, pod.Name, controllers.MariaDBIdent, + mariadbBackupPath, kubectlPath, env.Ns, pod.Name, controllers.MariaDBIdent, ) - cliutils.ExecuteKubectlClient(ns, pod.Name, controllers.MariaDBIdent, executeCommand) + cliutils.ExecuteKubectlClient(env.Ns, pod.Name, controllers.MariaDBIdent, executeCommand) ctrl.Log.Info("Finished restoring DB from backup!") } -func restoreZuul(ns string, backupDir string, kubeClientSet *kubernetes.Clientset, kubeContext string) { +func restoreZuul(backupDir string, kubeContext string, env cliutils.ENV) { ctrl.Log.Info("Restoring Zuul...") - pod := cliutils.GetPodByName(zuulBackupPod, ns, kubeClientSet) + pod := apiv1.Pod{} + cliutils.GetMOrDie(&env, zuulBackupPod, &pod) // ensure that pod does not have any restore file - restoreZuulRemoveCMD := []string{ - "rm", - "-rf", - "/tmp/zuul-import", - } - cliutils.RunRemoteCmd(kubeContext, ns, pod.Name, controllers.ZuulSchedulerIdent, restoreZuulRemoveCMD) - - // create empty directory for future restore - restoreZuulCreateDirCMD := []string{ - "mkdir", - "-p", - "/tmp/zuul-import", - } - cliutils.RunRemoteCmd(kubeContext, ns, pod.Name, controllers.ZuulSchedulerIdent, restoreZuulCreateDirCMD) + cleanCMD := []string{ + "bash", "-c", "rm -rf /tmp/zuul-import && mkdir -p /tmp/zuul-import"} + cliutils.RunRemoteCmd(kubeContext, env.Ns, pod.Name, controllers.ZuulSchedulerIdent, cleanCMD) // copy the Zuul private keys backup to pod // tar cf - -C /tmp/backup/zuul zuul.keys | /usr/bin/kubectl exec -i -n sf zuul-scheduler-0 -c zuul-scheduler -- tar xf - -C /tmp @@ -140,38 +110,26 @@ func restoreZuul(ns string, backupDir string, kubeClientSet *kubernetes.Clientse baseFile := filepath.Base(ZuulBackupPath) executeCommand := fmt.Sprintf( "tar cf - -C %s %s | %s exec -i -n %s %s -c %s -- tar xf - -C /tmp/zuul-import", - basePath, baseFile, kubectlPath, ns, pod.Name, controllers.ZuulSchedulerIdent, + basePath, baseFile, kubectlPath, env.Ns, pod.Name, controllers.ZuulSchedulerIdent, ) ctrl.Log.Info("Executing " + executeCommand) - cliutils.ExecuteKubectlClient(ns, pod.Name, controllers.ZuulSchedulerIdent, executeCommand) + cliutils.ExecuteKubectlClient(env.Ns, pod.Name, controllers.ZuulSchedulerIdent, executeCommand) // https://zuul-ci.org/docs/zuul/latest/client.html - restoreZuulCMD := []string{ - "zuul-admin", - "import-keys", - "--force", - "/tmp/zuul-import/" + baseFile, - } + restoreCMD := []string{ + "bash", "-c", "zuul-admin import-keys --force /tmp/zuul-import/" + baseFile + " && " + + "rm -rf /tmp/zuul-import"} // Execute command for restore - cliutils.RunRemoteCmd(kubeContext, ns, pod.Name, controllers.ZuulSchedulerIdent, restoreZuulCMD) - - // remove after all - cliutils.RunRemoteCmd(kubeContext, ns, pod.Name, controllers.ZuulSchedulerIdent, restoreZuulRemoveCMD) + cliutils.RunRemoteCmd(kubeContext, env.Ns, pod.Name, controllers.ZuulSchedulerIdent, restoreCMD) ctrl.Log.Info("Finished doing Zuul private keys restore!") } -func clearComponents(ns string, kubeContext string) { - ctrl.Log.Info("Removing components requiring a complete restart ...") - - env := cliutils.ENV{ - Cli: cliutils.CreateKubernetesClientOrDie(kubeContext), - Ctx: context.TODO(), - Ns: ns, - } +func clearComponents(env cliutils.ENV) { + ctrl.Log.Info("Removing components requirering a complete restart ...") for _, stsName := range []string{"zuul-scheduler", "zuul-executor", "zuul-merger", "nodepool-builder", "zookeeper"} { cliutils.DeleteOrDie(&env, &appsv1.StatefulSet{ @@ -209,18 +167,17 @@ func restoreCmd(kmd *cobra.Command, args []string) { } - // prepare to make restore - ns, kubeClientSet, kubeContext := prepareRestore(kmd) + kubeContext, env := cliutils.GetCLIENV(kmd) - if ns == "" { + if env.Ns == "" { ctrl.Log.Info("You did not specify the namespace!") os.Exit(1) } - restoreZuul(ns, backupDir, kubeClientSet, kubeContext) - restoreSecret(ns, backupDir, kubeContext) - restoreDB(ns, backupDir, kubeClientSet, kubeContext) - clearComponents(ns, kubeContext) + restoreZuul(backupDir, kubeContext, env) + restoreSecret(backupDir, env) + restoreDB(backupDir, kubeContext, env) + clearComponents(env) } diff --git a/cli/cmd/utils/utils.go b/cli/cmd/utils/utils.go index b71df52e..a7b05420 100644 --- a/cli/cmd/utils/utils.go +++ b/cli/cmd/utils/utils.go @@ -22,13 +22,14 @@ import ( "context" "errors" "fmt" - "gopkg.in/yaml.v3" "io/fs" "os" "os/exec" "reflect" "strings" + "gopkg.in/yaml.v3" + apiv1 "k8s.io/api/core/v1" apierrors "k8s.io/apimachinery/pkg/api/errors" @@ -51,7 +52,6 @@ import ( sfv1 "github.com/softwarefactory-project/sf-operator/api/v1" controllers "github.com/softwarefactory-project/sf-operator/controllers" - metav1 "k8s.io/apimachinery/pkg/apis/meta/v1" "k8s.io/client-go/kubernetes" ) @@ -196,6 +196,25 @@ func CreateKubernetesClientOrDie(contextName string) client.Client { return cli } +func GetCLIENV(kmd *cobra.Command) (string, ENV) { + + cliCtx, err := GetCLIContext(kmd) + if err != nil { + ctrl.Log.Error(err, "Error initializing CLI:") + os.Exit(1) + } + + kubeContext := cliCtx.KubeContext + + env := ENV{ + Cli: CreateKubernetesClientOrDie(kubeContext), + Ctx: context.TODO(), + Ns: cliCtx.Namespace, + } + + return kubeContext, env +} + func GetM(env *ENV, name string, obj client.Object) (bool, error) { err := env.Cli.Get(env.Ctx, client.ObjectKey{ @@ -420,24 +439,6 @@ func RunRemoteCmd(kubeContext string, namespace string, podName string, containe return buffer } -func GetPodByName(podName string, ns string, kubeClientSet *kubernetes.Clientset) *apiv1.Pod { - pod, err := kubeClientSet.CoreV1().Pods(ns).Get(context.TODO(), podName, metav1.GetOptions{}) - if err != nil { - ctrl.Log.Error(err, "Can not get pod "+podName) - os.Exit(1) - } - return pod -} - -func GetSecretByName(secretName string, ns string, kubeClientSet *kubernetes.Clientset) *apiv1.Secret { - secret, err := kubeClientSet.CoreV1().Secrets(ns).Get(context.TODO(), secretName, metav1.GetOptions{}) - if err != nil { - ctrl.Log.Error(err, "Can not get pod "+secretName) - os.Exit(1) - } - return secret -} - func ReadYAMLToMapOrDie(filePath string) map[string]interface{} { readFile, _ := GetFileContent(filePath) secretContent := make(map[string]interface{})