fix applying macros to s3 endpoint in object disk during restore embedded backups, fix #750

Author: Slach

ChangeLog.md

@@ -5,7 +5,8 @@ IMPROVEMENTS
 BUG FIXES
 - fix restore for object disk frozen_metadata.txt fix [752](https://github.com/Altinity/clickhouse-backup/issues/752)
 - fix more corner cases for `check_parts_columns: true`, fix [747](https://github.com/Altinity/clickhouse-backup/issues/747)
-
+- fix applying macros to s3 endpoint in object disk during restore embedded backups, fix [750](https://github.com/Altinity/clickhouse-backup/issues/750)
+ 
 # v2.4.1
 IMPROVEMENTS
 - switch to go-1.21

pkg/storage/object_disk/object_disk.go

@@ -320,16 +320,18 @@ func getObjectDisksCredentials(ctx context.Context, ch *clickhouse.ClickHouse) (
 				}
 				if endPointNode := d.SelectElement("endpoint"); endPointNode != nil {
 					creds.EndPoint = strings.Trim(endPointNode.InnerText(), "\r\n \t")
+					if creds.EndPoint, err = ch.ApplyMacros(ctx, creds.EndPoint); err != nil {
+						return nil, fmt.Errorf("%s -> /%s/storage_configuration/disks/%s apply macros to <endpoint> error: %v", configFile, root.Data, diskName, err)
+					}
 				} else {
 					return nil, fmt.Errorf("%s -> /%s/storage_configuration/disks/%s doesn't contains <endpoint>", configFile, root.Data, diskName)
 				}
 				if regionNode := d.SelectElement("region"); regionNode != nil {
 					creds.S3Region = strings.Trim(regionNode.InnerText(), "\r\n \t")
 				}
+				creds.S3StorageClass = "STANDARD"
 				if storageClassNode := d.SelectElement("s3_storage_class"); storageClassNode != nil {
 					creds.S3StorageClass = strings.Trim(storageClassNode.InnerText(), "\r\n \t")
-				} else {
-					creds.S3StorageClass = "STANDARD"
 				}
 				accessKeyNode := d.SelectElement("access_key_id")
 				secretKeyNode := d.SelectElement("secret_access_key")

test/integration/config-s3-embedded.yml

@@ -29,6 +29,7 @@ s3:
   acl: private
   force_path_style: true
   path: backup/{cluster}/{shard}
+  object_disk_path: object_disk/{cluster}/{shard}
   disable_ssl: true
   compression_format: none
 api:

test/integration/dynamic_settings.sh

@@ -89,7 +89,7 @@ cat <<EOT > /etc/clickhouse-server/config.d/storage_configuration_s3.xml
     <disks>
       <disk_s3>
         <type>s3</type>
-        <endpoint>http://minio:9000/clickhouse/disk_s3/</endpoint>
+        <endpoint>http://minio:9000/clickhouse/disk_s3/{cluster}/{shard}/</endpoint>
         <!-- https://github.com/Altinity/clickhouse-backup/issues/691
         <access_key_id>access-key</access_key_id>
         <secret_access_key>it-is-my-super-secret-key</secret_access_key>
@@ -124,7 +124,7 @@ cat <<EOT > /etc/clickhouse-server/config.d/storage_configuration_gcs.xml
     <disks>
       <disk_gcs_over_s3>
         <type>s3</type>
-        <endpoint>https://storage.googleapis.com/${QA_GCS_OVER_S3_BUCKET}/clickhouse_backup_disk_gcs_over_s3/${HOSTNAME}/</endpoint>
+        <endpoint>https://storage.googleapis.com/${QA_GCS_OVER_S3_BUCKET}/clickhouse_backup_disk_gcs_over_s3/${HOSTNAME}/{cluster}/{shard}/</endpoint>
         <access_key_id>${QA_GCS_OVER_S3_ACCESS_KEY}</access_key_id>
         <secret_access_key>${QA_GCS_OVER_S3_SECRET_KEY}</secret_access_key>
         <!-- to avoid slow startup -->
@@ -206,7 +206,7 @@ cat <<EOT > /etc/clickhouse-server/config.d/backup_storage_configuration_s3.xml
     <disks>
       <backups_s3>
         <type>s3</type>
-        <endpoint>http://minio:9000/clickhouse/backups_s3/</endpoint>
+        <endpoint>http://minio:9000/clickhouse/backups_s3/{cluster}/{shard}/</endpoint>
         <!-- https://github.com/Altinity/clickhouse-backup/issues/691
         <access_key_id>access-key</access_key_id>
         <secret_access_key>it-is-my-super-secret-key</secret_access_key>
@@ -243,7 +243,7 @@ cat <<EOT > /etc/clickhouse-server/config.d/backup_storage_configuration_s3_plai
     <disks>
       <backups_s3_plain>
         <type>s3_plain</type>
-        <endpoint>http://minio:9000/clickhouse/backups_s3_plain/</endpoint>
+        <endpoint>http://minio:9000/clickhouse/backups_s3_plain/{cluster}/{shard}/</endpoint>
         <!-- https://github.com/Altinity/clickhouse-backup/issues/691
         <access_key_id>access-key</access_key_id>
         <secret_access_key>it-is-my-super-secret-key</secret_access_key>