Skip to content

Commit

Permalink
cli backup-restore: small refactoring to remove code dup
Browse files Browse the repository at this point in the history 
Change-Id: I109453bb4a63f5188be0abf0edb83c43c45a338f
  • Loading branch information
@morucci
morucci committed Apr 30, 2024
1 parent 9ef5607 commit 05f3b82
Show file tree
Hide file tree
Showing 3 changed files with 73 additions and 128 deletions.
59 changes: 23 additions & 36 deletions cli/cmd/backup.go
View file
Original file line number Diff line number Diff line change
Expand Up @@ -29,7 +29,7 @@ import (
controllers "github.com/softwarefactory-project/sf-operator/controllers"
"github.com/spf13/cobra"
"gopkg.in/yaml.v3"
"k8s.io/client-go/kubernetes"
apiv1 "k8s.io/api/core/v1"
ctrl "sigs.k8s.io/controller-runtime"
)

Expand Down Expand Up @@ -64,29 +64,15 @@ var SecretsToBackup = []string{
"zuul-auth-secret",
}

func prepareBackup(kmd *cobra.Command, backupDir string) (string, *kubernetes.Clientset, string) {

cliCtx, err := cliutils.GetCLIContext(kmd)
if err != nil {
ctrl.Log.Error(err, "Error initializing CLI:")
os.Exit(1)
}

cliutils.CreateDirectory(backupDir, 0755)

kubeContext := cliCtx.KubeContext
_, kubeClientSet := cliutils.GetClientset(kubeContext)
return cliCtx.Namespace, kubeClientSet, kubeContext
}

func createSecretBackup(ns string, backupDir string, kubeClientSet *kubernetes.Clientset) {
func createSecretBackup(backupDir string, env cliutils.ENV) {
ctrl.Log.Info("Creating secrets backup...")

secretsDir := backupDir + "/" + SecretsBackupPath
cliutils.CreateDirectory(secretsDir, 0755)

for _, sec := range SecretsToBackup {
secret := cliutils.GetSecretByName(sec, ns, kubeClientSet)
for _, secName := range SecretsToBackup {
secret := apiv1.Secret{}
cliutils.GetMOrDie(&env, secName, &secret)

// convert secret content to string (was bytes)
strMap := cliutils.ConvertMapOfBytesToMapOfStrings(secret.Data)
Expand Down Expand Up @@ -115,12 +101,12 @@ func createSecretBackup(ns string, backupDir string, kubeClientSet *kubernetes.C
ctrl.Log.Info("Finished doing secret backup!")
}

func createZuulKeypairBackup(ns string, backupDir string, kubeClientSet *kubernetes.Clientset,
kubeContext string) {
func createZuulKeypairBackup(backupDir string, kubeContext string, env cliutils.ENV) {

ctrl.Log.Info("Doing Zuul keys backup...")

pod := cliutils.GetPodByName(zuulBackupPod, ns, kubeClientSet)
pod := apiv1.Pod{}
cliutils.GetMOrDie(&env, zuulBackupPod, &pod)

// https://zuul-ci.org/docs/zuul/latest/client.html
zuulBackupPath := backupDir + "/" + ZuulBackupPath
Expand All @@ -141,22 +127,21 @@ func createZuulKeypairBackup(ns string, backupDir string, kubeClientSet *kuberne
}

// Execute command for backup
cliutils.RunRemoteCmd(kubeContext, ns, pod.Name, controllers.ZuulSchedulerIdent, backupZuulCMD)
cliutils.RunRemoteCmd(kubeContext, env.Ns, pod.Name, controllers.ZuulSchedulerIdent, backupZuulCMD)

// Take output of the backup
commandBuffer := cliutils.RunRemoteCmd(kubeContext, ns, pod.Name, controllers.ZuulSchedulerIdent, backupZuulPrintCMD)
commandBuffer := cliutils.RunRemoteCmd(kubeContext, env.Ns, pod.Name, controllers.ZuulSchedulerIdent, backupZuulPrintCMD)

// write stdout to file
cliutils.WriteContentToFile(zuulBackupPath, commandBuffer.Bytes(), 0640)

// Remove key file from the pod
cliutils.RunRemoteCmd(kubeContext, ns, pod.Name, controllers.ZuulSchedulerIdent, backupZuulRemoveCMD)
cliutils.RunRemoteCmd(kubeContext, env.Ns, pod.Name, controllers.ZuulSchedulerIdent, backupZuulRemoveCMD)

ctrl.Log.Info("Finished doing Zuul private keys backup!")
}

func createMySQLBackup(ns string, backupDir string, kubeClientSet *kubernetes.Clientset,
kubeContext string) {
func createMySQLBackup(backupDir string, kubeContext string, env cliutils.ENV) {
ctrl.Log.Info("Doing DB backup...")

// create MariaDB dir
Expand All @@ -165,7 +150,8 @@ func createMySQLBackup(ns string, backupDir string, kubeClientSet *kubernetes.Cl

cliutils.CreateDirectory(mariaDBBackupDir, 0755)

pod := cliutils.GetPodByName(dbBackupPod, ns, kubeClientSet)
pod := apiv1.Pod{}
cliutils.GetMOrDie(&env, dbBackupPod, &pod)

// NOTE: We use option: --single-transaction to avoid error:
// "The user specified as a definer ('mariadb.sys'@'localhost') does not exist" when using LOCK TABLES
Expand All @@ -177,7 +163,7 @@ func createMySQLBackup(ns string, backupDir string, kubeClientSet *kubernetes.Cl
}

// just create Zuul DB backup
commandBuffer := cliutils.RunRemoteCmd(kubeContext, ns, pod.Name, controllers.MariaDBIdent, backupZuulCMD)
commandBuffer := cliutils.RunRemoteCmd(kubeContext, env.Ns, pod.Name, controllers.MariaDBIdent, backupZuulCMD)

// write stdout to file
cliutils.WriteContentToFile(mariadbBackupPath, commandBuffer.Bytes(), 0640)
Expand All @@ -192,24 +178,25 @@ func backupCmd(kmd *cobra.Command, args []string) {
os.Exit(1)
}

// prepare to make backup
ns, kubeClientSet, kubeContext := prepareBackup(kmd, backupDir)
cliutils.CreateDirectory(backupDir, 0755)

kubeContext, env := cliutils.GetCLIENV(kmd)

if ns == "" {
if env.Ns == "" {
ctrl.Log.Error(errors.New("no namespace set"), "You need to specify the namespace!")
os.Exit(1)
}

ctrl.Log.Info("Starting backup process for services in namespace: " + ns)
ctrl.Log.Info("Starting backup process for services in namespace: " + env.Ns)

// create secret backup
createSecretBackup(ns, backupDir, kubeClientSet)
createSecretBackup(backupDir, env)

// create zuul backup
createZuulKeypairBackup(ns, backupDir, kubeClientSet, kubeContext)
createZuulKeypairBackup(backupDir, kubeContext, env)

// create DB backup
createMySQLBackup(ns, backupDir, kubeClientSet, kubeContext)
createMySQLBackup(backupDir, kubeContext, env)

}

Expand Down
101 changes: 29 additions & 72 deletions cli/cmd/restore.go
View file
Original file line number Diff line number Diff line change
Expand Up @@ -21,7 +21,6 @@ package cmd
*/

import (
"context"
"errors"
"fmt"
"os"
Expand All @@ -33,39 +32,19 @@ import (
"github.com/spf13/cobra"

appsv1 "k8s.io/api/apps/v1"
corev1 "k8s.io/api/core/v1"
apiv1 "k8s.io/api/core/v1"
metav1 "k8s.io/apimachinery/pkg/apis/meta/v1"
"k8s.io/client-go/kubernetes"
ctrl "sigs.k8s.io/controller-runtime"
)

func prepareRestore(kmd *cobra.Command) (string, *kubernetes.Clientset, string) {

cliCtx, err := cliutils.GetCLIContext(kmd)
if err != nil {
ctrl.Log.Error(err, "Error initializing CLI:")
os.Exit(1)
}

kubeContext := cliCtx.KubeContext
_, kubeClientSet := cliutils.GetClientset(kubeContext)
return cliCtx.Namespace, kubeClientSet, kubeContext
}

func restoreSecret(ns string, backupDir string, kubeContext string) {
func restoreSecret(backupDir string, env cliutils.ENV) {
ctrl.Log.Info("Restoring secrets...")

env := cliutils.ENV{
Cli: cliutils.CreateKubernetesClientOrDie(kubeContext),
Ctx: context.TODO(),
Ns: ns,
}

for _, sec := range SecretsToBackup {
pathToSecret := backupDir + "/" + SecretsBackupPath + "/" + sec + ".yaml"
secretContent := cliutils.ReadYAMLToMapOrDie(pathToSecret)

var secret corev1.Secret
secret := apiv1.Secret{}
if cliutils.GetMOrDie(&env, sec, &secret) {
secretMap := secretContent["data"].(map[string]interface{})
for key, value := range secretMap {
Expand All @@ -88,16 +67,17 @@ func restoreSecret(ns string, backupDir string, kubeContext string) {

}

func restoreDB(ns string, backupDir string, kubeClientSet *kubernetes.Clientset, kubeContext string) {
func restoreDB(backupDir string, kubeContext string, env cliutils.ENV) {
ctrl.Log.Info("Restoring DB...")
pod := cliutils.GetPodByName(dbBackupPod, ns, kubeClientSet)
pod := apiv1.Pod{}
cliutils.GetMOrDie(&env, dbBackupPod, &pod)

kubectlPath := cliutils.GetKubectlPath()
dropDBCMD := []string{
"mysql",
"-e DROP DATABASE zuul;",
}
cliutils.RunRemoteCmd(kubeContext, ns, pod.Name, controllers.MariaDBIdent, dropDBCMD)
cliutils.RunRemoteCmd(kubeContext, env.Ns, pod.Name, controllers.MariaDBIdent, dropDBCMD)

mariadbBackupPath := backupDir + "/" + DBBackupPath

Expand All @@ -106,32 +86,22 @@ func restoreDB(ns string, backupDir string, kubeClientSet *kubernetes.Clientset,
// but in that case, we need to do it via system kubernetes client.
executeCommand := fmt.Sprintf(
"cat %s | %s -n %s exec -it %s -c %s -- sh -c \"mysql -h0\"",
mariadbBackupPath, kubectlPath, ns, pod.Name, controllers.MariaDBIdent,
mariadbBackupPath, kubectlPath, env.Ns, pod.Name, controllers.MariaDBIdent,
)

cliutils.ExecuteKubectlClient(ns, pod.Name, controllers.MariaDBIdent, executeCommand)
cliutils.ExecuteKubectlClient(env.Ns, pod.Name, controllers.MariaDBIdent, executeCommand)

ctrl.Log.Info("Finished restoring DB from backup!")
}
func restoreZuul(ns string, backupDir string, kubeClientSet *kubernetes.Clientset, kubeContext string) {
func restoreZuul(backupDir string, kubeContext string, env cliutils.ENV) {
ctrl.Log.Info("Restoring Zuul...")
pod := cliutils.GetPodByName(zuulBackupPod, ns, kubeClientSet)
pod := apiv1.Pod{}
cliutils.GetMOrDie(&env, zuulBackupPod, &pod)

// ensure that pod does not have any restore file
restoreZuulRemoveCMD := []string{
"rm",
"-rf",
"/tmp/zuul-import",
}
cliutils.RunRemoteCmd(kubeContext, ns, pod.Name, controllers.ZuulSchedulerIdent, restoreZuulRemoveCMD)

// create empty directory for future restore
restoreZuulCreateDirCMD := []string{
"mkdir",
"-p",
"/tmp/zuul-import",
}
cliutils.RunRemoteCmd(kubeContext, ns, pod.Name, controllers.ZuulSchedulerIdent, restoreZuulCreateDirCMD)
cleanCMD := []string{
"bash", "-c", "rm -rf /tmp/zuul-import && mkdir -p /tmp/zuul-import"}
cliutils.RunRemoteCmd(kubeContext, env.Ns, pod.Name, controllers.ZuulSchedulerIdent, cleanCMD)

// copy the Zuul private keys backup to pod
// tar cf - -C /tmp/backup/zuul zuul.keys | /usr/bin/kubectl exec -i -n sf zuul-scheduler-0 -c zuul-scheduler -- tar xf - -C /tmp
Expand All @@ -140,38 +110,26 @@ func restoreZuul(ns string, backupDir string, kubeClientSet *kubernetes.Clientse
baseFile := filepath.Base(ZuulBackupPath)
executeCommand := fmt.Sprintf(
"tar cf - -C %s %s | %s exec -i -n %s %s -c %s -- tar xf - -C /tmp/zuul-import",
basePath, baseFile, kubectlPath, ns, pod.Name, controllers.ZuulSchedulerIdent,
basePath, baseFile, kubectlPath, env.Ns, pod.Name, controllers.ZuulSchedulerIdent,
)
ctrl.Log.Info("Executing " + executeCommand)

cliutils.ExecuteKubectlClient(ns, pod.Name, controllers.ZuulSchedulerIdent, executeCommand)
cliutils.ExecuteKubectlClient(env.Ns, pod.Name, controllers.ZuulSchedulerIdent, executeCommand)

// https://zuul-ci.org/docs/zuul/latest/client.html
restoreZuulCMD := []string{
"zuul-admin",
"import-keys",
"--force",
"/tmp/zuul-import/" + baseFile,
}
restoreCMD := []string{
"bash", "-c", "zuul-admin import-keys --force /tmp/zuul-import/" + baseFile + " && " +
"rm -rf /tmp/zuul-import"}

// Execute command for restore
cliutils.RunRemoteCmd(kubeContext, ns, pod.Name, controllers.ZuulSchedulerIdent, restoreZuulCMD)

// remove after all
cliutils.RunRemoteCmd(kubeContext, ns, pod.Name, controllers.ZuulSchedulerIdent, restoreZuulRemoveCMD)
cliutils.RunRemoteCmd(kubeContext, env.Ns, pod.Name, controllers.ZuulSchedulerIdent, restoreCMD)

ctrl.Log.Info("Finished doing Zuul private keys restore!")

}

func clearComponents(ns string, kubeContext string) {
ctrl.Log.Info("Removing components requiring a complete restart ...")

env := cliutils.ENV{
Cli: cliutils.CreateKubernetesClientOrDie(kubeContext),
Ctx: context.TODO(),
Ns: ns,
}
func clearComponents(env cliutils.ENV) {
ctrl.Log.Info("Removing components requirering a complete restart ...")

for _, stsName := range []string{"zuul-scheduler", "zuul-executor", "zuul-merger", "nodepool-builder", "zookeeper"} {
cliutils.DeleteOrDie(&env, &appsv1.StatefulSet{
Expand Down Expand Up @@ -209,18 +167,17 @@ func restoreCmd(kmd *cobra.Command, args []string) {

}

// prepare to make restore
ns, kubeClientSet, kubeContext := prepareRestore(kmd)
kubeContext, env := cliutils.GetCLIENV(kmd)

if ns == "" {
if env.Ns == "" {
ctrl.Log.Info("You did not specify the namespace!")
os.Exit(1)
}

restoreZuul(ns, backupDir, kubeClientSet, kubeContext)
restoreSecret(ns, backupDir, kubeContext)
restoreDB(ns, backupDir, kubeClientSet, kubeContext)
clearComponents(ns, kubeContext)
restoreZuul(backupDir, kubeContext, env)
restoreSecret(backupDir, env)
restoreDB(backupDir, kubeContext, env)
clearComponents(env)

}

Expand Down
41 changes: 21 additions & 20 deletions cli/cmd/utils/utils.go
View file
Original file line number Diff line number Diff line change
Expand Up @@ -22,13 +22,14 @@ import (
"context"
"errors"
"fmt"
"gopkg.in/yaml.v3"
"io/fs"
"os"
"os/exec"
"reflect"
"strings"

"gopkg.in/yaml.v3"

apiv1 "k8s.io/api/core/v1"
apierrors "k8s.io/apimachinery/pkg/api/errors"

Expand All @@ -51,7 +52,6 @@ import (
sfv1 "github.com/softwarefactory-project/sf-operator/api/v1"
controllers "github.com/softwarefactory-project/sf-operator/controllers"

metav1 "k8s.io/apimachinery/pkg/apis/meta/v1"
"k8s.io/client-go/kubernetes"
)

Expand Down Expand Up @@ -196,6 +196,25 @@ func CreateKubernetesClientOrDie(contextName string) client.Client {
return cli
}

func GetCLIENV(kmd *cobra.Command) (string, ENV) {

cliCtx, err := GetCLIContext(kmd)
if err != nil {
ctrl.Log.Error(err, "Error initializing CLI:")
os.Exit(1)
}

kubeContext := cliCtx.KubeContext

env := ENV{
Cli: CreateKubernetesClientOrDie(kubeContext),
Ctx: context.TODO(),
Ns: cliCtx.Namespace,
}

return kubeContext, env
}

func GetM(env *ENV, name string, obj client.Object) (bool, error) {
err := env.Cli.Get(env.Ctx,
client.ObjectKey{
Expand Down Expand Up @@ -420,24 +439,6 @@ func RunRemoteCmd(kubeContext string, namespace string, podName string, containe
return buffer
}

func GetPodByName(podName string, ns string, kubeClientSet *kubernetes.Clientset) *apiv1.Pod {
pod, err := kubeClientSet.CoreV1().Pods(ns).Get(context.TODO(), podName, metav1.GetOptions{})
if err != nil {
ctrl.Log.Error(err, "Can not get pod "+podName)
os.Exit(1)
}
return pod
}

func GetSecretByName(secretName string, ns string, kubeClientSet *kubernetes.Clientset) *apiv1.Secret {
secret, err := kubeClientSet.CoreV1().Secrets(ns).Get(context.TODO(), secretName, metav1.GetOptions{})
if err != nil {
ctrl.Log.Error(err, "Can not get pod "+secretName)
os.Exit(1)
}
return secret
}

func ReadYAMLToMapOrDie(filePath string) map[string]interface{} {
readFile, _ := GetFileContent(filePath)
secretContent := make(map[string]interface{})
Expand Down

0 comments on commit 05f3b82

Please sign in to comment.