Skip to content

Commit

Permalink
fix test failures for TestChangeReplicationPathIfReplicaExists
Browse files Browse the repository at this point in the history
  • Loading branch information
@Slach
Slach committed Nov 2, 2024
1 parent f367685 commit 4a1b23d
Show file tree
Hide file tree
Showing 2 changed files with 32 additions and 14 deletions.
10 changes: 8 additions & 2 deletions pkg/backup/restore.go
View file
Original file line number Diff line number Diff line change
Expand Up @@ -1098,7 +1098,7 @@ func (b *Backuper) restoreSchemaRegular(ctx context.Context, tablesForRestore Li
b.replaceCreateToAttachForView(&schema)
// https://github.com/Altinity/clickhouse-backup/issues/849
log.Info().Msgf("SUKA BEFORE!!! schema.Query=%s", schema.Query)
b.checkReplicaAlreadyExistsAndChangeReplicationPath(ctx, &schema)
b.checkReplicaAlreadyExistsAndChangeReplicationPath(ctx, &schema, version)
log.Info().Msgf("SUKA AFTER!!! schema.Query=%s", schema.Query)

// https://github.com/Altinity/clickhouse-backup/issues/466
Expand Down Expand Up @@ -1134,7 +1134,7 @@ func (b *Backuper) restoreSchemaRegular(ctx context.Context, tablesForRestore Li
var replicatedParamsRE = regexp.MustCompile(`(Replicated[a-zA-Z]*MergeTree)\('([^']+)'(\s*,\s*)'([^']+)'\)|(Replicated[a-zA-Z]*MergeTree)\(\)`)
var replicatedUuidRE = regexp.MustCompile(` UUID '([^']+)'`)

func (b *Backuper) checkReplicaAlreadyExistsAndChangeReplicationPath(ctx context.Context, schema *metadata.TableMetadata) {
func (b *Backuper) checkReplicaAlreadyExistsAndChangeReplicationPath(ctx context.Context, schema *metadata.TableMetadata, version int) {
if matches := replicatedParamsRE.FindAllStringSubmatch(schema.Query, -1); len(matches) > 0 {
var err error
if len(matches[0]) < 1 {
Expand Down Expand Up @@ -1169,6 +1169,9 @@ func (b *Backuper) checkReplicaAlreadyExistsAndChangeReplicationPath(ctx context
if matches = replicatedUuidRE.FindAllStringSubmatch(schema.Query, 1); len(matches) > 0 {
resolvedReplicaPath = strings.Replace(resolvedReplicaPath, "{uuid}", matches[0][1], -1)
}
if version < 19017000 {
resolvedReplicaPath = strings.NewReplacer("{database}", schema.Database, "{table}", schema.Table).Replace(resolvedReplicaPath)
}

isReplicaPresent := uint64(0)
fullReplicaPath := path.Join(resolvedReplicaPath, "replicas", resolvedReplicaName)
Expand All @@ -1186,6 +1189,9 @@ func (b *Backuper) checkReplicaAlreadyExistsAndChangeReplicationPath(ctx context
} else {
schema.Query = strings.Replace(schema.Query, engine+"('"+replicaPath+"'"+delimiter+"'"+replicaName+"')", engine+"('"+newReplicaPath+"', '"+newReplicaName+"')", 1)
}
if version < 19017000 {
schema.Query = strings.NewReplacer("{database}", schema.Database, "{table}", schema.Table).Replace(schema.Query)
}
}
}

Expand Down
36 changes: 24 additions & 12 deletions test/integration/integration_test.go
View file
Original file line number Diff line number Diff line change
Expand Up @@ -583,19 +583,26 @@ func TestChangeReplicationPathIfReplicaExists(t *testing.T) {
return createSQL
}
createUUID := uuid.New()
createWithUUIDSQL := ""
createSQL := createReplicatedTable("test_replica_wrong_path", "", "'/clickhouse/tables/wrong_path','{replica}'")
createWithUUIDSQL := createReplicatedTable("test_replica_wrong_path_uuid", fmt.Sprintf(" UUID '%s' ", createUUID.String()), "")
if compareVersion(os.Getenv("CLICKHOUSE_VERSION"), "20.7") >= 0 {
createWithUUIDSQL = createReplicatedTable("test_replica_wrong_path_uuid", fmt.Sprintf(" UUID '%s' ", createUUID.String()), "")
}

r.NoError(env.DockerExec("clickhouse-backup", "clickhouse-backup", "-c", "/etc/clickhouse-backup/config-s3.yml", "create", "--tables", "default.test_replica_wrong_path*", "test_wrong_path"))

r.NoError(env.ch.DropTable(clickhouse.Table{Database: "default", Name: "test_replica_wrong_path"}, createSQL, "", false, version, ""))
r.NoError(env.ch.DropTable(clickhouse.Table{Database: "default", Name: "test_replica_wrong_path_uuid"}, createWithUUIDSQL, "", false, version, ""))

if compareVersion(os.Getenv("CLICKHOUSE_VERSION"), "20.7") >= 0 {
r.NoError(env.ch.DropTable(clickhouse.Table{Database: "default", Name: "test_replica_wrong_path_uuid"}, createWithUUIDSQL, "", false, version, ""))
}
// hack for drop tables without drop data from keeper
_ = createReplicatedTable("test_replica_wrong_path2", "", "'/clickhouse/tables/wrong_path','{replica}'")
_ = createReplicatedTable("test_replica_wrong_path_uuid2", fmt.Sprintf(" UUID '%s' ", createUUID.String()), "")
r.NoError(env.DockerExec("clickhouse", "rm", "-fv", "/var/lib/clickhouse/metadata/default/test_replica_wrong_path2.sql"))
r.NoError(env.DockerExec("clickhouse", "rm", "-fv", "/var/lib/clickhouse/metadata/default/test_replica_wrong_path_uuid2.sql"))
r.NoError(env.DockerExec("clickhouse", "rm", "-rfv", fmt.Sprintf("/var/lib/clickhouse/store/%s/%s", createUUID.String()[:3], createUUID.String())))
if compareVersion(os.Getenv("CLICKHOUSE_VERSION"), "20.7") >= 0 {
_ = createReplicatedTable("test_replica_wrong_path_uuid2", fmt.Sprintf(" UUID '%s' ", createUUID.String()), "")
r.NoError(env.DockerExec("clickhouse", "rm", "-fv", "/var/lib/clickhouse/metadata/default/test_replica_wrong_path_uuid2.sql"))
r.NoError(env.DockerExec("clickhouse", "rm", "-rfv", fmt.Sprintf("/var/lib/clickhouse/store/%s/%s", createUUID.String()[:3], createUUID.String())))
}
env.ch.Close()
r.NoError(utils.ExecCmd(context.Background(), 180*time.Second, "docker", append(env.GetDefaultComposeCommand(), "restart", "clickhouse")...))
env.connectWithWait(r, 0*time.Second, 1*time.Second, 1*time.Minute)
Expand All @@ -605,24 +612,29 @@ func TestChangeReplicationPathIfReplicaExists(t *testing.T) {
log.Debug().Msg(restoreOut)
r.NoError(err)
r.Contains(restoreOut, "replica /clickhouse/tables/wrong_path/replicas/clickhouse already exists in system.zookeeper will replace to /clickhouse/tables/{cluster}/{shard}/{database}/{table}/replicas/{replica}")
r.Contains(restoreOut, fmt.Sprintf("replica /clickhouse/tables/%s/0/replicas/clickhouse already exists in system.zookeeper will replace to /clickhouse/tables/{cluster}/{shard}/{database}/{table}/replicas/{replica}", createUUID.String()))

if compareVersion(os.Getenv("CLICKHOUSE_VERSION"), "20.7") >= 0 {
r.Contains(restoreOut, fmt.Sprintf("replica /clickhouse/tables/%s/0/replicas/clickhouse already exists in system.zookeeper will replace to /clickhouse/tables/{cluster}/{shard}/{database}/{table}/replicas/{replica}", createUUID.String()))
}
checkRestoredTable := func(table string, expectedRows uint64, expectedEngine string) {
rows := uint64(0)
r.NoError(env.ch.SelectSingleRowNoCtx(&rows, fmt.Sprintf("SELECT count() FROM default.%s", table)))
r.Equal(expectedRows, rows)

engineFull := ""
r.NoError(env.ch.SelectSingleRowNoCtx(&engineFull, "SELECT engine_full FROM system.tables WHERE database=? AND table=?", "default", table))
r.NoError(env.ch.SelectSingleRowNoCtx(&engineFull, "SELECT engine_full FROM system.tables WHERE database=? AND name=?", "default", table))
r.Contains(engineFull, expectedEngine)

}
checkRestoredTable("test_replica_wrong_path", 10, "/clickhouse/tables/{cluster}/{shard}/default/test_replica_wrong_path")
checkRestoredTable("test_replica_wrong_path_uuid", 10, "/clickhouse/tables/{cluster}/{shard}/default/test_replica_wrong_path_uuid")

r.NoError(env.ch.DropTable(clickhouse.Table{Database: "default", Name: "test_replica_wrong_path"}, createSQL, "", false, version, ""))
r.NoError(env.ch.DropTable(clickhouse.Table{Database: "default", Name: "test_replica_wrong_path_uuid"}, createWithUUIDSQL, "", false, version, ""))
if compareVersion(os.Getenv("CLICKHOUSE_VERSION"), "20.7") >= 0 {
checkRestoredTable("test_replica_wrong_path_uuid", 10, "/clickhouse/tables/{cluster}/{shard}/default/test_replica_wrong_path_uuid")
}

r.NoError(env.ch.DropTable(clickhouse.Table{Database: "default", Name: "test_replica_wrong_path"}, createSQL, "", false, version, ""))
if compareVersion(os.Getenv("CLICKHOUSE_VERSION"), "20.7") >= 0 {
r.NoError(env.ch.DropTable(clickhouse.Table{Database: "default", Name: "test_replica_wrong_path_uuid"}, createWithUUIDSQL, "", false, version, ""))
}
r.NoError(env.DockerExec("clickhouse-backup", "clickhouse-backup", "-c", "/etc/clickhouse-backup/config-s3.yml", "delete", "local", "test_wrong_path"))
}

Expand Down

0 comments on commit 4a1b23d

Please sign in to comment.