Backup/Restore RBAC related objects from Zookeeper via direct connection to zookeeper/keeper, fix #604

Author: Slach

ChangeLog.md

@@ -3,6 +3,7 @@ IMPROVEMENTS
 - first implementation for properly backup S3/GCS/Azure disks, support server-side copy to back up bucket during `clickhouse-backup` create and during `clickhouse-backup restore`, requires add `object_disk_path` to `s3`,`gcs`,`azblob` section, fix [447](https://github.com/Altinity/clickhouse-backup/issues/447)
 - Implementation blacklist for table engines during backup / download / upload / restore [537](https://github.com/Altinity/clickhouse-backup/issues/537)
 - restore RBAC / configs, refactoring restart clickhouse-server via `sql:SYSTEM SHUTDOWN` or `exec:systemctl restart clickhouse-server`, add `--rbac-only` and `--configs-only` options to `create`, `upload`, `download`, `restore` command. fix [706]https://github.com/Altinity/clickhouse-backup/issues/706
+- Backup/Restore RBAC related objects from Zookeeper via direct connection to zookeeper/keeper, fix [604](https://github.com/Altinity/clickhouse-backup/issues/604)
 
 BUG FIXES
 - fix possible create backup failures during UNFREEZE not exists tables, affected 2.2.7+ version, fix [704](https://github.com/Altinity/clickhouse-backup/issues/704)

Dockerfile

@@ -27,31 +27,29 @@ RUN go env
 WORKDIR /src/
 # cache modules when go.mod go.sum changed
 COPY go.mod go.sum ./
-RUN --mount=type=cache,id=clickhouse-backup-gobuild,target=/root/.cache/go go mod download -x
+RUN --mount=type=cache,id=clickhouse-backup-gobuild,target=/root/ go mod download -x
 
 FROM builder-base AS builder-race
 ARG TARGETPLATFORM
 COPY ./ /src/
 RUN mkdir -p ./clickhouse-backup/
-# RUN --mount=type=cache,id=clickhouse-backup-gobuild,target=/root/.cache/go GOOS=$( echo ${TARGETPLATFORM} | cut -d "/" -f 1) GOARCH=$( echo ${TARGETPLATFORM} | cut -d "/" -f 2) CC=musl-gcc CGO_ENABLED=1 go build -a -cover -buildvcs=false -ldflags "-X 'main.version=race' -linkmode=external -extldflags '-static'" -gcflags "all=-N -l" -race -o ./clickhouse-backup/clickhouse-backup-race ./cmd/clickhouse-backup
-RUN --mount=type=cache,id=clickhouse-backup-gobuild,target=/root/.cache/go GOOS=$( echo ${TARGETPLATFORM} | cut -d "/" -f 1) GOARCH=$( echo ${TARGETPLATFORM} | cut -d "/" -f 2) CC=musl-gcc CGO_ENABLED=1 go build -a -cover -buildvcs=false -ldflags "-X 'main.version=race' -linkmode=external -extldflags '-static'" -race -o ./clickhouse-backup/clickhouse-backup-race ./cmd/clickhouse-backup
+RUN --mount=type=cache,id=clickhouse-backup-gobuild,target=/root/ GOOS=$( echo ${TARGETPLATFORM} | cut -d "/" -f 1) GOARCH=$( echo ${TARGETPLATFORM} | cut -d "/" -f 2) CC=musl-gcc CGO_ENABLED=1 go build -a -cover -buildvcs=false -ldflags "-X 'main.version=race' -linkmode=external -extldflags '-static'" -race -o ./clickhouse-backup/clickhouse-backup-race ./cmd/clickhouse-backup
 RUN cp -l ./clickhouse-backup/clickhouse-backup-race /bin/clickhouse-backup && echo "$(ldd ./clickhouse-backup/clickhouse-backup-race 2>&1 || true)" | grep -c "not a dynamic executable"
-# RUN --mount=type=cache,id=clickhouse-backup-gobuild,target=/root/.cache/go GOOS=$( echo ${TARGETPLATFORM} | cut -d "/" -f 1) GOARCH=$( echo ${TARGETPLATFORM} | cut -d "/" -f 2) GOEXPERIMENT=boringcrypto CC=musl-gcc CGO_ENABLED=1 go build -cover -buildvcs=false -ldflags "-X 'main.version=race-fips' -linkmode=external -extldflags '-static'" -gcflags "all=-N -l" -race -o ./clickhouse-backup/clickhouse-backup-race-fips ./cmd/clickhouse-backup
-RUN --mount=type=cache,id=clickhouse-backup-gobuild,target=/root/.cache/go GOOS=$( echo ${TARGETPLATFORM} | cut -d "/" -f 1) GOARCH=$( echo ${TARGETPLATFORM} | cut -d "/" -f 2) GOEXPERIMENT=boringcrypto CC=musl-gcc CGO_ENABLED=1 go build -cover -buildvcs=false -ldflags "-X 'main.version=race-fips' -linkmode=external -extldflags '-static'" -race -o ./clickhouse-backup/clickhouse-backup-race-fips ./cmd/clickhouse-backup
+RUN --mount=type=cache,id=clickhouse-backup-gobuild,target=/root/ GOOS=$( echo ${TARGETPLATFORM} | cut -d "/" -f 1) GOARCH=$( echo ${TARGETPLATFORM} | cut -d "/" -f 2) GOEXPERIMENT=boringcrypto CC=musl-gcc CGO_ENABLED=1 go build -cover -buildvcs=false -ldflags "-X 'main.version=race-fips' -linkmode=external -extldflags '-static'" -race -o ./clickhouse-backup/clickhouse-backup-race-fips ./cmd/clickhouse-backup
 RUN cp -l ./clickhouse-backup/clickhouse-backup-race-fips /bin/clickhouse-backup-fips && echo "$(ldd ./clickhouse-backup/clickhouse-backup-race-fips 2>&1 || true)" | grep -c "not a dynamic executable"
 COPY entrypoint.sh /entrypoint.sh
 
 
 FROM builder-base AS builder-docker
 COPY ./ /src/
 RUN mkdir -p ./build/
-RUN --mount=type=cache,id=clickhouse-backup-gobuild,target=/root/.cache/go make build
+RUN --mount=type=cache,id=clickhouse-backup-gobuild,target=/root/ make build
 
 
 FROM builder-base AS builder-fips
 COPY ./ /src/
 RUN mkdir -p ./build/
-RUN --mount=type=cache,id=clickhouse-backup-gobuild,target=/root/.cache/go make build-fips
+RUN --mount=type=cache,id=clickhouse-backup-gobuild,target=/root/ make build-fips
 
 
 FROM scratch AS make-build-race

go.mod

@@ -20,6 +20,7 @@ require (
 	github.com/djherbis/nio/v3 v3.0.1
 	github.com/eapache/go-resiliency v1.3.0
 	github.com/go-logfmt/logfmt v0.6.0
+	github.com/go-zookeeper/zk v1.0.3
 	github.com/google/shlex v0.0.0-20191202100458-e7afc7fbc510
 	github.com/google/uuid v1.3.0
 	github.com/gorilla/mux v1.8.0

go.sum

@@ -167,6 +167,8 @@ github.com/go-gl/glfw/v3.3/glfw v0.0.0-20191125211704-12ad95a8df72/go.mod h1:tQ2
 github.com/go-logfmt/logfmt v0.4.0/go.mod h1:3RMwSq7FuexP4Kalkev3ejPJsZTpXXBr9+V4qmtdjCk=
 github.com/go-logfmt/logfmt v0.6.0 h1:wGYYu3uicYdqXVgoYbvnkrPVXkuLM1p1ifugDMEdRi4=
 github.com/go-logfmt/logfmt v0.6.0/go.mod h1:WYhtIu8zTZfxdn5+rREduYbwxfcBr/Vr6KEVveWlfTs=
+github.com/go-zookeeper/zk v1.0.3 h1:7M2kwOsc//9VeeFiPtf+uSJlVpU66x9Ba5+8XK7/TDg=
+github.com/go-zookeeper/zk v1.0.3/go.mod h1:nOB03cncLtlp4t+UAkGSV+9beXP/akpekBwL+UX1Qcw=
 github.com/gogo/protobuf v1.3.2/go.mod h1:P1XiOD3dCwIKUDQYPy72D8LYyHL2YPYrpS2s69NZV8Q=
 github.com/golang-jwt/jwt/v4 v4.0.0/go.mod h1:/xlHOz8bRuivTWchD4jCa+NbatV+wEUSzwAxVc6locg=
 github.com/golang-jwt/jwt/v4 v4.5.0 h1:7cYmW1XlMY7h7ii7UhUyChSgS5wUJEnm9uZVTGqOWzg=

pkg/backup/create.go

@@ -6,6 +6,7 @@ import (
 	"errors"
 	"fmt"
 	"github.com/Altinity/clickhouse-backup/pkg/config"
+	"github.com/Altinity/clickhouse-backup/pkg/keeper"
 	"github.com/Altinity/clickhouse-backup/pkg/partition"
 	"github.com/Altinity/clickhouse-backup/pkg/status"
 	"github.com/Altinity/clickhouse-backup/pkg/storage"
@@ -229,14 +230,14 @@ func (b *Backuper) createBackupLocal(ctx context.Context, backupName string, par
 
 	if createRBAC || rbacOnly {
 		if backupRBACSize, err = b.createBackupRBAC(ctx, backupPath, disks); err != nil {
-			log.Errorf("error during do RBAC backup: %v", err)
+			log.Fatalf("error during do RBAC backup: %v", err)
 		} else {
 			log.WithField("size", utils.FormatBytes(backupRBACSize)).Info("done createBackupRBAC")
 		}
 	}
 	if createConfigs || configsOnly {
 		if backupConfigSize, err = b.createBackupConfigs(ctx, backupPath); err != nil {
-			log.Errorf("error during do CONFIG backup: %v", err)
+			log.Fatalf("error during do CONFIG backup: %v", err)
 		} else {
 			log.WithField("size", utils.FormatBytes(backupConfigSize)).Info("done createBackupConfigs")
 		}
@@ -451,8 +452,43 @@ func (b *Backuper) createBackupRBAC(ctx context.Context, backupPath string, disk
 				return false, nil
 			},
 		})
-		return rbacDataSize, copyErr
+		if copyErr != nil {
+			return 0, copyErr
+		}
+		replicatedRBACDataSize, err := b.createBackupRBACReplicated(ctx, rbacBackup)
+		if err != nil {
+			return 0, err
+		}
+		return rbacDataSize + replicatedRBACDataSize, nil
+	}
+}
+
+func (b *Backuper) createBackupRBACReplicated(ctx context.Context, rbacBackup string) (replicatedRBACDataSize uint64, err error) {
+	replicatedRBAC := make([]struct {
+		Name string `ch:"name"`
+	}, 0)
+	rbacDataSize := uint64(0)
+	if err = b.ch.SelectContext(ctx, &replicatedRBAC, "SELECT name FROM system.user_directories WHERE type='replicated'"); err == nil && len(replicatedRBAC) > 0 {
+		k := keeper.Keeper{Log: b.log.WithField("logger", "keeper")}
+		if err = k.Connect(ctx, b.ch, b.cfg); err != nil {
+			return 0, err
+		}
+		defer k.Close()
+		for _, userDirectory := range replicatedRBAC {
+			replicatedAccessPath, err := k.GetReplicatedAccessPath(userDirectory.Name)
+			if err != nil {
+				return 0, err
+			}
+			dumpFile := path.Join(rbacBackup, userDirectory.Name+".jsonl")
+			b.log.WithField("logger", "createBackupRBACReplicated").Infof("keeper.Dump %s -> %s", replicatedAccessPath, dumpFile)
+			dumpRBACSize, dumpErr := k.Dump(replicatedAccessPath, dumpFile)
+			if dumpErr != nil {
+				return 0, dumpErr
+			}
+			rbacDataSize += uint64(dumpRBACSize)
+		}
 	}
+	return rbacDataSize, nil
 }
 
 func (b *Backuper) AddTableToBackup(ctx context.Context, backupName, shadowBackupUUID string, diskList []clickhouse.Disk, table *clickhouse.Table, partitionsIdsMap common.EmptyMap) (map[string][]metadata.Part, map[string]int64, error) {

pkg/backup/restore.go

@@ -5,6 +5,7 @@ import (
 	"encoding/json"
 	"fmt"
 	"github.com/Altinity/clickhouse-backup/pkg/config"
+	"github.com/Altinity/clickhouse-backup/pkg/keeper"
 	"github.com/Altinity/clickhouse-backup/pkg/status"
 	"github.com/Altinity/clickhouse-backup/pkg/storage"
 	"github.com/Altinity/clickhouse-backup/pkg/storage/object_disk"
@@ -277,7 +278,7 @@ func (b *Backuper) restoreRBAC(ctx context.Context, backupName string, disks []c
 	if err != nil {
 		return err
 	}
-	if err = b.restoreBackupRelatedDir(backupName, "access", accessPath, disks); err == nil {
+	if err = b.restoreBackupRelatedDir(backupName, "access", accessPath, disks, []string{"*.jsonl"}); err == nil {
 		markFile := path.Join(accessPath, "need_rebuild_lists.mark")
 		log.Infof("create %s for properly rebuild RBAC after restart clickhouse-server", markFile)
 		file, err := os.Create(markFile)
@@ -298,22 +299,81 @@ func (b *Backuper) restoreRBAC(ctx context.Context, backupName string, disks []c
 			}
 		}
 	}
-	if !os.IsNotExist(err) {
+	if err != nil && !os.IsNotExist(err) {
+		return err
+	}
+	if err = b.restoreRBACReplicated(ctx, backupName, "access", disks); err != nil {
 		return err
 	}
 	return nil
 }
 
+func (b *Backuper) restoreRBACReplicated(ctx context.Context, backupName string, backupPrefixDir string, disks []clickhouse.Disk) error {
+	log := b.log.WithField("logger", "restoreRBACReplicated")
+	defaultDataPath, err := b.ch.GetDefaultPath(disks)
+	if err != nil {
+		return ErrUnknownClickhouseDataPath
+	}
+	srcBackupDir := path.Join(defaultDataPath, "backup", backupName, backupPrefixDir)
+	info, err := os.Stat(srcBackupDir)
+	if err != nil {
+		return err
+	}
+
+	if !info.IsDir() {
+		return fmt.Errorf("%s is not a dir", srcBackupDir)
+	}
+	replicatedRBAC := make([]struct {
+		Name string `ch:"name"`
+	}, 0)
+	if err = b.ch.SelectContext(ctx, &replicatedRBAC, "SELECT name FROM system.user_directories WHERE type='replicated'"); err == nil && len(replicatedRBAC) > 0 {
+		jsonLFiles, err := filepathx.Glob(path.Join(srcBackupDir, "*.jsonl"))
+		if err != nil {
+			return err
+		}
+		if len(jsonLFiles) == 0 {
+			return nil
+		}
+		k := keeper.Keeper{Log: b.log.WithField("logger", "keeper")}
+		if err = k.Connect(ctx, b.ch, b.cfg); err != nil {
+			return err
+		}
+		defer k.Close()
+		restoreReplicatedRBACMap := make(map[string]string, len(jsonLFiles))
+		for _, jsonLFile := range jsonLFiles {
+			for _, userDirectory := range replicatedRBAC {
+				if strings.HasSuffix(jsonLFile, userDirectory.Name+".jsonl") {
+					restoreReplicatedRBACMap[jsonLFile] = userDirectory.Name
+				}
+			}
+			if _, exists := restoreReplicatedRBACMap[jsonLFile]; !exists {
+				restoreReplicatedRBACMap[jsonLFile] = replicatedRBAC[0].Name
+			}
+		}
+		for jsonLFile, userDirectoryName := range restoreReplicatedRBACMap {
+			replicatedAccessPath, err := k.GetReplicatedAccessPath(userDirectoryName)
+			if err != nil {
+				return err
+			}
+			log.Infof("keeper.Restore(%s) -> %s", jsonLFile, replicatedAccessPath)
+			if err := k.Restore(jsonLFile, replicatedAccessPath); err != nil {
+				return err
+			}
+		}
+	}
+	return nil
+}
+
 // restoreConfigs - copy backup_name/configs folder to /etc/clickhouse-server/
 func (b *Backuper) restoreConfigs(backupName string, disks []clickhouse.Disk) error {
-	if err := b.restoreBackupRelatedDir(backupName, "configs", b.ch.Config.ConfigDir, disks); err != nil && os.IsNotExist(err) {
+	if err := b.restoreBackupRelatedDir(backupName, "configs", b.ch.Config.ConfigDir, disks, nil); err != nil && os.IsNotExist(err) {
 		return nil
 	} else {
 		return err
 	}
 }
 
-func (b *Backuper) restoreBackupRelatedDir(backupName, backupPrefixDir, destinationDir string, disks []clickhouse.Disk) error {
+func (b *Backuper) restoreBackupRelatedDir(backupName, backupPrefixDir, destinationDir string, disks []clickhouse.Disk, skipPatterns []string) error {
 	log := b.log.WithField("logger", "restoreBackupRelatedDir")
 	defaultDataPath, err := b.ch.GetDefaultPath(disks)
 	if err != nil {
@@ -329,9 +389,19 @@ func (b *Backuper) restoreBackupRelatedDir(backupName, backupPrefixDir, destinat
 		return fmt.Errorf("%s is not a dir", srcBackupDir)
 	}
 	log.Debugf("copy %s -> %s", srcBackupDir, destinationDir)
-	copyOptions := recursiveCopy.Options{OnDirExists: func(src, dest string) recursiveCopy.DirExistsAction {
-		return recursiveCopy.Merge
-	}}
+	copyOptions := recursiveCopy.Options{
+		OnDirExists: func(src, dest string) recursiveCopy.DirExistsAction {
+			return recursiveCopy.Merge
+		},
+		Skip: func(srcinfo os.FileInfo, src, dest string) (bool, error) {
+			for _, pattern := range skipPatterns {
+				if matched, matchErr := filepath.Match(pattern, filepath.Base(src)); matchErr != nil || matched {
+					return true, matchErr
+				}
+			}
+			return false, nil
+		},
+	}
 	if err := recursiveCopy.Copy(srcBackupDir, destinationDir, copyOptions); err != nil {
 		return err
 	}

pkg/clickhouse/clickhouse.go

@@ -975,7 +975,7 @@ func (ch *ClickHouse) SelectSingleRow(ctx context.Context, dest interface{}, que
 
 func (ch *ClickHouse) SelectSingleRowNoCtx(dest interface{}, query string, args ...interface{}) error {
 	err := ch.conn.QueryRow(context.Background(), ch.LogQuery(query, args...), args...).Scan(dest)
-	if err != nil && err == sql.ErrNoRows {
+	if err != nil && errors.Is(err, sql.ErrNoRows) {
 		return nil
 	}
 	return err
@@ -1004,21 +1004,32 @@ func (ch *ClickHouse) IsAtomic(database string) (bool, error) {
 	return isDatabaseAtomic == "Atomic", nil
 }
 
-// GetAccessManagementPath @todo think about how to properly extract access_management_path from /etc/clickhouse-server/
+// GetAccessManagementPath extract path from following sources system.user_directories, access_control_path from /var/lib/clickhouse/preprocessed_configs/config.xml, system.disks
 func (ch *ClickHouse) GetAccessManagementPath(ctx context.Context, disks []Disk) (string, error) {
 	accessPath := "/var/lib/clickhouse/access"
 	rows := make([]struct {
 		AccessPath string `ch:"access_path"`
 	}, 0)
 	if err := ch.SelectContext(ctx, &rows, "SELECT JSONExtractString(params,'path') AS access_path FROM system.user_directories WHERE type='local directory'"); err != nil || len(rows) == 0 {
+		configFile, doc, err := ch.ParseXML(ctx, "config.xml")
+		if err != nil {
+			ch.Log.Warnf("can't parse config.xml from %s, error: %v", configFile, err)
+		}
+		if err == nil {
+			accessControlPathNode := doc.SelectElement("access_control_path")
+			if accessControlPathNode != nil {
+				return accessControlPathNode.InnerText(), nil
+			}
+		}
+
 		if disks == nil {
 			disks, err = ch.GetDisks(ctx, false)
 			if err != nil {
 				return "", err
 			}
 		}
 		for _, disk := range disks {
-			if _, err := os.Stat(path.Join(disk.Path, "access")); !os.IsNotExist(err) {
+			if fInfo, err := os.Stat(path.Join(disk.Path, "access")); err == nil && fInfo.IsDir() {
 				accessPath = path.Join(disk.Path, "access")
 				break
 			}
@@ -1201,6 +1212,23 @@ func (ch *ClickHouse) GetPreprocessedConfigPath(ctx context.Context) (string, er
 	return path.Join("/", path.Join(paths[:len(paths)-1]...), "preprocessed_configs"), nil
 }
 
+func (ch *ClickHouse) ParseXML(ctx context.Context, configName string) (configFile string, doc *xmlquery.Node, err error) {
+	preprocessedConfigPath, err := ch.GetPreprocessedConfigPath(ctx)
+	if err != nil {
+		return "", nil, err
+	}
+	configFile = path.Join(preprocessedConfigPath, configName)
+	f, err := os.Open(configFile)
+	if err != nil {
+		return "", nil, err
+	}
+	doc, err = xmlquery.Parse(f)
+	if err != nil {
+		return "", nil, err
+	}
+	return configFile, doc, nil
+}
+
 var preprocessedXMLSettings map[string]map[string]string
 
 // GetPreprocessedXMLSettings - @todo think about from_end and from_zookeeper corner cases
@@ -1211,7 +1239,7 @@ func (ch *ClickHouse) GetPreprocessedXMLSettings(ctx context.Context, settingsXP
 	}
 	resultSettings := make(map[string]string, len(settingsXPath))
 	if _, exists := preprocessedXMLSettings[fileName]; !exists {
-		preprocessedXMLSettings[fileName] = make(map[string]string, 0)
+		preprocessedXMLSettings[fileName] = make(map[string]string)
 	}
 	var doc *xmlquery.Node
 	for settingName, xpathExpr := range settingsXPath {