Implementation blacklist for table engines during backup / download /…

… upload / restore fix #537

Signed-off-by: Slach <[email protected]>

ReadMe.md

@@ -389,11 +389,14 @@ clickhouse:
   # The format for this env variable is "disk_name1:disk_path1,disk_name2:disk_path2". For YAML please continue using map syntax 
   disk_mapping: {}
   # CLICKHOUSE_SKIP_TABLES, the list of tables (pattern are allowed) which are ignored during backup and restore process
-  # The format for this env variable is "pattern1,pattern2,pattern3". For YAML please continue using map syntax 
+  # The format for this env variable is "pattern1,pattern2,pattern3". For YAML please continue using list syntax 
   skip_tables:                     
     - system.*
     - INFORMATION_SCHEMA.*
     - information_schema.*
+  # CLICKHOUSE_SKIP_TABLE_ENGINES, the list of tables engines which are ignored during backup, upload, download, restore process
+  # The format for this env variable is "Engine1,Engine2,engine3". For YAML please continue using list syntax 
+  skip_table_engines: [] 
   timeout: 5m                  # CLICKHOUSE_TIMEOUT
   freeze_by_part: false        # CLICKHOUSE_FREEZE_BY_PART, allow freezing by part instead of freezing the whole table
   freeze_by_part_where: ""     # CLICKHOUSE_FREEZE_BY_PART_WHERE, allow parts filtering during freezing when freeze_by_part: true
@@ -421,6 +424,7 @@ azblob:
   use_managed_identity: false  # AZBLOB_USE_MANAGED_IDENTITY
   container: ""                # AZBLOB_CONTAINER
   path: ""                     # AZBLOB_PATH, `system.macros` values could be applied as {macro_name}
+  object_disk_path: ""         # AZBLOB_OBJECT_DISK_PATH, path for backup of part from `azure_blob_storage` object disk, if disk present, then shall not be zero and shall not be prefixed by `path`   
   compression_level: 1         # AZBLOB_COMPRESSION_LEVEL
   compression_format: tar      # AZBLOB_COMPRESSION_FORMAT, allowed values tar, lz4, bzip2, gzip, sz, xz, brortli, zstd, `none` for upload data part folders as is
   sse_key: ""                  # AZBLOB_SSE_KEY
@@ -437,6 +441,7 @@ s3:
   assume_role_arn: ""              # S3_ASSUME_ROLE_ARN
   force_path_style: false          # S3_FORCE_PATH_STYLE
   path: ""                         # S3_PATH, `system.macros` values could be applied as {macro_name} 
+  object_disk_path: ""             # S3_OBJECT_DISK_PATH, path for backup of part from `s3` object disk, if disk present, then shall not be zero and shall not be prefixed by `path`   
   disable_ssl: false               # S3_DISABLE_SSL
   compression_level: 1             # S3_COMPRESSION_LEVEL
   compression_format: tar          # S3_COMPRESSION_FORMAT, allowed values tar, lz4, bzip2, gzip, sz, xz, brortli, zstd, `none` for upload data part folders as is
@@ -471,6 +476,7 @@ gcs:
   credentials_json_encoded: "" # GCS_CREDENTIALS_JSON_ENCODED
   bucket: ""                   # GCS_BUCKET
   path: ""                     # GCS_PATH, `system.macros` values could be applied as {macro_name}
+  object_disk_path: ""         # GCS_OBJECT_DISK_PATH, path for backup of part from `s3` object disk (clickhouse support only gcs over s3 protocol), if disk present, then shall not be zero and shall not be prefixed by `path`    
   compression_level: 1         # GCS_COMPRESSION_LEVEL
   compression_format: tar      # GCS_COMPRESSION_FORMAT, allowed values tar, lz4, bzip2, gzip, sz, xz, brortli, zstd, `none` for upload data part folders as is
   storage_class: STANDARD      # GCS_STORAGE_CLASS

pkg/backup/download.go

@@ -203,6 +203,9 @@ func (b *Backuper) Download(backupName string, tablePattern string, partitions [
 			if err != nil {
 				return err
 			}
+			if b.shouldSkipByTableEngine(*downloadedMetadata) {
+				return nil
+			}
 			tableMetadataAfterDownload[idx] = *downloadedMetadata
 			atomic.AddUint64(&metadataSize, size)
 			return nil

test/testflows/clickhouse_backup/tests/snapshots/cli.py.cli.snapshot

@@ -1,4 +1,4 @@
-default_config = r"""'[\'general:\', \'    remote_storage: none\', \'    disable_progress_bar: true\', \'    backups_to_keep_local: 0\', \'    backups_to_keep_remote: 0\', \'    log_level: info\', \'    allow_empty_backups: false\', \'    use_resumable_state: true\', \'    restore_schema_on_cluster: ""\', \'    upload_by_part: true\', \'    download_by_part: true\', \'    restore_database_mapping: {}\', \'    retries_on_failure: 3\', \'    retries_pause: 30s\', \'    watch_interval: 1h\', \'    full_interval: 24h\', \'    watch_backup_name_template: shard{shard}-{type}-{time:20060102150405}\', \'    retriesduration: 100ms\', \'    watchduration: 1h0m0s\', \'    fullduration: 24h0m0s\', \'clickhouse:\', \'    username: default\', \'    password: ""\', \'    host: localhost\', \'    port: 9000\', \'    disk_mapping: {}\', \'    skip_tables:\', \'        - system.*\', \'        - INFORMATION_SCHEMA.*\', \'        - information_schema.*\', \'        - _temporary_and_external_tables.*\', \'    timeout: 5m\', \'    freeze_by_part: false\', \'    freeze_by_part_where: ""\', \'    use_embedded_backup_restore: false\', \'    embedded_backup_disk: ""\', \'    backup_mutations: true\', \'    restore_as_attach: false\', \'    check_parts_columns: true\', \'    secure: false\', \'    skip_verify: false\', \'    sync_replicated_tables: false\', \'    log_sql_queries: true\', \'    config_dir: /etc/clickhouse-server/\', \'    restart_command: systemctl restart clickhouse-server\', \'    ignore_not_exists_error_during_freeze: true\', \'    check_replicas_before_attach: true\', \'    tls_key: ""\', \'    tls_cert: ""\', \'    tls_ca: ""\', \'    debug: false\', \'s3:\', \'    access_key: ""\', \'    secret_key: ""\', \'    bucket: ""\', \'    endpoint: ""\', \'    region: us-east-1\', \'    acl: private\', \'    assume_role_arn: ""\', \'    force_path_style: false\', \'    path: ""\', \'    object_disk_path: ""\', \'    disable_ssl: false\', \'    compression_level: 1\', \'    compression_format: tar\', \'    sse: ""\', \'    sse_kms_key_id: ""\', \'    sse_customer_algorithm: ""\', \'    sse_customer_key: ""\', \'    sse_customer_key_md5: ""\', \'    sse_kms_encryption_context: ""\', \'    disable_cert_verification: false\', \'    use_custom_storage_class: false\', \'    storage_class: STANDARD\', \'    custom_storage_class_map: {}\', \'    part_size: 0\', \'    allow_multipart_download: false\', \'    object_labels: {}\', \'    debug: false\', \'gcs:\', \'    credentials_file: ""\', \'    credentials_json: ""\', \'    credentials_json_encoded: ""\', \'    bucket: ""\', \'    path: ""\', \'    object_disk_path: ""\', \'    compression_level: 1\', \'    compression_format: tar\', \'    debug: false\', \'    endpoint: ""\', \'    storage_class: STANDARD\', \'    object_labels: {}\', \'    custom_storage_class_map: {}\', \'cos:\', \'    url: ""\', \'    timeout: 2m\', \'    secret_id: ""\', \'    secret_key: ""\', \'    path: ""\', \'    compression_format: tar\', \'    compression_level: 1\', \'    debug: false\', \'api:\', \'    listen: localhost:7171\', \'    enable_metrics: true\', \'    enable_pprof: false\', \'    username: ""\', \'    password: ""\', \'    secure: false\', \'    certificate_file: ""\', \'    private_key_file: ""\', \'    ca_cert_file: ""\', \'    ca_key_file: ""\', \'    create_integration_tables: false\', \'    integration_tables_host: ""\', \'    allow_parallel: false\', \'    complete_resumable_after_restart: true\', \'ftp:\', \'    address: ""\', \'    timeout: 2m\', \'    username: ""\', \'    password: ""\', \'    tls: false\', \'    path: ""\', \'    object_disk_path: ""\', \'    compression_format: tar\', \'    compression_level: 1\', \'    debug: false\', \'sftp:\', \'    address: ""\', \'    port: 22\', \'    username: ""\', \'    password: ""\', \'    key: ""\', \'    path: ""\', \'    object_disk_path: ""\', \'    compression_format: tar\', \'    compression_level: 1\', \'    debug: false\', \'azblob:\', \'    endpoint_schema: https\', \'    endpoint_suffix: core.windows.net\', \'    account_name: ""\', \'    account_key: ""\', \'    sas: ""\', \'    use_managed_identity: false\', \'    container: ""\', \'    path: ""\', \'    object_disk_path: ""\', \'    compression_level: 1\', \'    compression_format: tar\', \'    sse_key: ""\', \'    buffer_size: 0\', \'    buffer_count: 3\', \'    timeout: 15m\', \'custom:\', \'    upload_command: ""\', \'    download_command: ""\', \'    list_command: ""\', \'    delete_command: ""\', \'    command_timeout: 4h\', \'    commandtimeoutduration: 4h0m0s\']'"""
+default_config = r"""'[\'general:\', \'    remote_storage: none\', \'    disable_progress_bar: true\', \'    backups_to_keep_local: 0\', \'    backups_to_keep_remote: 0\', \'    log_level: info\', \'    allow_empty_backups: false\', \'    use_resumable_state: true\', \'    restore_schema_on_cluster: ""\', \'    upload_by_part: true\', \'    download_by_part: true\', \'    restore_database_mapping: {}\', \'    retries_on_failure: 3\', \'    retries_pause: 30s\', \'    watch_interval: 1h\', \'    full_interval: 24h\', \'    watch_backup_name_template: shard{shard}-{type}-{time:20060102150405}\', \'    retriesduration: 100ms\', \'    watchduration: 1h0m0s\', \'    fullduration: 24h0m0s\', \'clickhouse:\', \'    username: default\', \'    password: ""\', \'    host: localhost\', \'    port: 9000\', \'    disk_mapping: {}\', \'    skip_tables:\', \'        - system.*\', \'        - INFORMATION_SCHEMA.*\', \'        - information_schema.*\', \'        - _temporary_and_external_tables.*\', \'    skip_table_engines: []\', \'    timeout: 5m\', \'    freeze_by_part: false\', \'    freeze_by_part_where: ""\', \'    use_embedded_backup_restore: false\', \'    embedded_backup_disk: ""\', \'    backup_mutations: true\', \'    restore_as_attach: false\', \'    check_parts_columns: true\', \'    secure: false\', \'    skip_verify: false\', \'    sync_replicated_tables: false\', \'    log_sql_queries: true\', \'    config_dir: /etc/clickhouse-server/\', \'    restart_command: systemctl restart clickhouse-server\', \'    ignore_not_exists_error_during_freeze: true\', \'    check_replicas_before_attach: true\', \'    tls_key: ""\', \'    tls_cert: ""\', \'    tls_ca: ""\', \'    debug: false\', \'s3:\', \'    access_key: ""\', \'    secret_key: ""\', \'    bucket: ""\', \'    endpoint: ""\', \'    region: us-east-1\', \'    acl: private\', \'    assume_role_arn: ""\', \'    force_path_style: false\', \'    path: ""\', \'    object_disk_path: ""\', \'    disable_ssl: false\', \'    compression_level: 1\', \'    compression_format: tar\', \'    sse: ""\', \'    sse_kms_key_id: ""\', \'    sse_customer_algorithm: ""\', \'    sse_customer_key: ""\', \'    sse_customer_key_md5: ""\', \'    sse_kms_encryption_context: ""\', \'    disable_cert_verification: false\', \'    use_custom_storage_class: false\', \'    storage_class: STANDARD\', \'    custom_storage_class_map: {}\', \'    part_size: 0\', \'    allow_multipart_download: false\', \'    object_labels: {}\', \'    debug: false\', \'gcs:\', \'    credentials_file: ""\', \'    credentials_json: ""\', \'    credentials_json_encoded: ""\', \'    bucket: ""\', \'    path: ""\', \'    object_disk_path: ""\', \'    compression_level: 1\', \'    compression_format: tar\', \'    debug: false\', \'    endpoint: ""\', \'    storage_class: STANDARD\', \'    object_labels: {}\', \'    custom_storage_class_map: {}\', \'cos:\', \'    url: ""\', \'    timeout: 2m\', \'    secret_id: ""\', \'    secret_key: ""\', \'    path: ""\', \'    compression_format: tar\', \'    compression_level: 1\', \'    debug: false\', \'api:\', \'    listen: localhost:7171\', \'    enable_metrics: true\', \'    enable_pprof: false\', \'    username: ""\', \'    password: ""\', \'    secure: false\', \'    certificate_file: ""\', \'    private_key_file: ""\', \'    ca_cert_file: ""\', \'    ca_key_file: ""\', \'    create_integration_tables: false\', \'    integration_tables_host: ""\', \'    allow_parallel: false\', \'    complete_resumable_after_restart: true\', \'ftp:\', \'    address: ""\', \'    timeout: 2m\', \'    username: ""\', \'    password: ""\', \'    tls: false\', \'    path: ""\', \'    object_disk_path: ""\', \'    compression_format: tar\', \'    compression_level: 1\', \'    debug: false\', \'sftp:\', \'    address: ""\', \'    port: 22\', \'    username: ""\', \'    password: ""\', \'    key: ""\', \'    path: ""\', \'    object_disk_path: ""\', \'    compression_format: tar\', \'    compression_level: 1\', \'    debug: false\', \'azblob:\', \'    endpoint_schema: https\', \'    endpoint_suffix: core.windows.net\', \'    account_name: ""\', \'    account_key: ""\', \'    sas: ""\', \'    use_managed_identity: false\', \'    container: ""\', \'    path: ""\', \'    object_disk_path: ""\', \'    compression_level: 1\', \'    compression_format: tar\', \'    sse_key: ""\', \'    buffer_size: 0\', \'    buffer_count: 3\', \'    timeout: 15m\', \'custom:\', \'    upload_command: ""\', \'    download_command: ""\', \'    list_command: ""\', \'    delete_command: ""\', \'    command_timeout: 4h\', \'    commandtimeoutduration: 4h0m0s\']'"""
 
 help_flag = r"""'NAME:\n   clickhouse-backup - Tool for easy backup of ClickHouse with cloud supportUSAGE:\n   clickhouse-backup <command> [-t, --tables=<db>.<table>] <backup_name>DESCRIPTION:\n   Run as \'root\' or \'clickhouse\' userCOMMANDS:\n   tables               List of tables, exclude skip_tables\n   create               Create new backup\n   create_remote        Create and upload new backup\n   upload               Upload backup to remote storage\n   list                 List of backups\n   download             Download backup from remote storage\n   restore              Create schema and restore data from backup\n   restore_remote       Download and restore\n   delete               Delete specific backup\n   default-config       Print default config\n   print-config         Print current config merged with environment variables\n   clean                Remove data in \'shadow\' folder from all \'path\' folders available from \'system.disks\'\n   clean_remote_broken  Remove all broken remote backups\n   watch                Run infinite loop which create full + incremental backup sequence to allow efficient backup sequences\n   server               Run API server\n   help, h              Shows a list of commands or help for one commandGLOBAL OPTIONS:\n   --config value, -c value  Config \'FILE\' name. (default: "/etc/clickhouse-backup/config.yml") [$CLICKHOUSE_BACKUP_CONFIG]\n   --help, -h                show help\n   --version, -v             print the version'"""