add support connection pool for google cloud storage, GCS_CLIENT_POOL_SIZE, fix #724

Slach · Slach · commit 8a0d3179da0a · 2023-08-28T18:54:05.000+04:00
diff --git a/ChangeLog.md b/ChangeLog.md
@@ -10,6 +10,7 @@ IMPROVEMENTS
 - restore functions via `CREATE OR REPLACE` for more atomic behavior
 - prepare to Make ./tests/integration/ test parallel fix [721](https://github.com/Altinity/clickhouse-backup/issues/721)
 - touch `/var/lib/clickhouse/flags/force_drop_table` before every DROP TABLE execution, fix [683](https://github.com/Altinity/clickhouse-backup/issues/683)
+- add support connection pool for google cloud storage, `GCS_CLIENT_POOL_SIZE`, fix [724](https://github.com/Altinity/clickhouse-backup/issues/724)
 
 BUG FIXES
 - fix possible create backup failures during UNFREEZE not exists tables, affected 2.2.7+ version, fix [704](https://github.com/Altinity/clickhouse-backup/issues/704)
diff --git a/ReadMe.md b/ReadMe.md
@@ -486,7 +486,7 @@ gcs:
   compression_level: 1         # GCS_COMPRESSION_LEVEL
   compression_format: tar      # GCS_COMPRESSION_FORMAT, allowed values tar, lz4, bzip2, gzip, sz, xz, brortli, zstd, `none` for upload data part folders as is
   storage_class: STANDARD      # GCS_STORAGE_CLASS
-
+  client_pool_size: 500        # GCS_CLIENT_POOL_SIZE, should be at least 2 times bigger than `UPLOAD_CONCURRENCY` or `DOWNLOAD_CONCURRENCY` in each upload and download case
   # GCS_OBJECT_LABELS, allow setup metadata for each object during upload, use {macro_name} from system.macros and {backupName} for current backup name
   # The format for this env variable is "key1:value1,key2:value2". For YAML please continue using map syntax
   object_labels: {}
diff --git a/pkg/config/config.go b/pkg/config/config.go
@@ -77,7 +77,7 @@ type GCSConfig struct {
 	StorageClass           string            `yaml:"storage_class" envconfig:"GCS_STORAGE_CLASS"`
 	ObjectLabels           map[string]string `yaml:"object_labels" envconfig:"GCS_OBJECT_LABELS"`
 	CustomStorageClassMap  map[string]string `yaml:"custom_storage_class_map" envconfig:"GCS_CUSTOM_STORAGE_CLASS_MAP"`
-	// NOTE: ClientPoolSize should be atleast 2 times bigger than
+	// NOTE: ClientPoolSize should be at least 2 times bigger than
 	// 			UploadConcurrency or DownloadConcurrency in each upload and download case
 	ClientPoolSize int `yaml:"client_pool_size" envconfig:"GCS_CLIENT_POOL_SIZE"`
 }
diff --git a/test/testflows/clickhouse_backup/tests/snapshots/cli.py.cli.snapshot b/test/testflows/clickhouse_backup/tests/snapshots/cli.py.cli.snapshot
@@ -1,4 +1,4 @@
-default_config = r"""'[\'general:\', \'    remote_storage: none\', \'    disable_progress_bar: true\', \'    backups_to_keep_local: 0\', \'    backups_to_keep_remote: 0\', \'    log_level: info\', \'    allow_empty_backups: false\', \'    use_resumable_state: true\', \'    restore_schema_on_cluster: ""\', \'    upload_by_part: true\', \'    download_by_part: true\', \'    restore_database_mapping: {}\', \'    retries_on_failure: 3\', \'    retries_pause: 30s\', \'    watch_interval: 1h\', \'    full_interval: 24h\', \'    watch_backup_name_template: shard{shard}-{type}-{time:20060102150405}\', \'    sharded_operation_mode: ""\', \'    retriesduration: 100ms\', \'    watchduration: 1h0m0s\', \'    fullduration: 24h0m0s\', \'clickhouse:\', \'    username: default\', \'    password: ""\', \'    host: localhost\', \'    port: 9000\', \'    disk_mapping: {}\', \'    skip_tables:\', \'        - system.*\', \'        - INFORMATION_SCHEMA.*\', \'        - information_schema.*\', \'        - _temporary_and_external_tables.*\', \'    skip_table_engines: []\', \'    timeout: 5m\', \'    freeze_by_part: false\', \'    freeze_by_part_where: ""\', \'    use_embedded_backup_restore: false\', \'    embedded_backup_disk: ""\', \'    backup_mutations: true\', \'    restore_as_attach: false\', \'    check_parts_columns: true\', \'    secure: false\', \'    skip_verify: false\', \'    sync_replicated_tables: false\', \'    log_sql_queries: true\', \'    config_dir: /etc/clickhouse-server/\', \'    restart_command: exec:systemctl restart clickhouse-server\', \'    ignore_not_exists_error_during_freeze: true\', \'    check_replicas_before_attach: true\', \'    tls_key: ""\', \'    tls_cert: ""\', \'    tls_ca: ""\', \'    debug: false\', \'s3:\', \'    access_key: ""\', \'    secret_key: ""\', \'    bucket: ""\', \'    endpoint: ""\', \'    region: us-east-1\', \'    acl: private\', \'    assume_role_arn: ""\', \'    force_path_style: false\', \'    path: ""\', \'    object_disk_path: ""\', \'    disable_ssl: false\', \'    compression_level: 1\', \'    compression_format: tar\', \'    sse: ""\', \'    sse_kms_key_id: ""\', \'    sse_customer_algorithm: ""\', \'    sse_customer_key: ""\', \'    sse_customer_key_md5: ""\', \'    sse_kms_encryption_context: ""\', \'    disable_cert_verification: false\', \'    use_custom_storage_class: false\', \'    storage_class: STANDARD\', \'    custom_storage_class_map: {}\', \'    part_size: 0\', \'    allow_multipart_download: false\', \'    object_labels: {}\', \'    debug: false\', \'gcs:\', \'    credentials_file: ""\', \'    credentials_json: ""\', \'    credentials_json_encoded: ""\', \'    bucket: ""\', \'    path: ""\', \'    object_disk_path: ""\', \'    compression_level: 1\', \'    compression_format: tar\', \'    debug: false\', \'    endpoint: ""\', \'    storage_class: STANDARD\', \'    object_labels: {}\', \'    custom_storage_class_map: {}\', \'cos:\', \'    url: ""\', \'    timeout: 2m\', \'    secret_id: ""\', \'    secret_key: ""\', \'    path: ""\', \'    compression_format: tar\', \'    compression_level: 1\', \'    debug: false\', \'api:\', \'    listen: localhost:7171\', \'    enable_metrics: true\', \'    enable_pprof: false\', \'    username: ""\', \'    password: ""\', \'    secure: false\', \'    certificate_file: ""\', \'    private_key_file: ""\', \'    ca_cert_file: ""\', \'    ca_key_file: ""\', \'    create_integration_tables: false\', \'    integration_tables_host: ""\', \'    allow_parallel: false\', \'    complete_resumable_after_restart: true\', \'ftp:\', \'    address: ""\', \'    timeout: 2m\', \'    username: ""\', \'    password: ""\', \'    tls: false\', \'    path: ""\', \'    object_disk_path: ""\', \'    compression_format: tar\', \'    compression_level: 1\', \'    debug: false\', \'sftp:\', \'    address: ""\', \'    port: 22\', \'    username: ""\', \'    password: ""\', \'    key: ""\', \'    path: ""\', \'    object_disk_path: ""\', \'    compression_format: tar\', \'    compression_level: 1\', \'    debug: false\', \'azblob:\', \'    endpoint_schema: https\', \'    endpoint_suffix: core.windows.net\', \'    account_name: ""\', \'    account_key: ""\', \'    sas: ""\', \'    use_managed_identity: false\', \'    container: ""\', \'    path: ""\', \'    object_disk_path: ""\', \'    compression_level: 1\', \'    compression_format: tar\', \'    sse_key: ""\', \'    buffer_size: 0\', \'    buffer_count: 3\', \'    timeout: 15m\', \'custom:\', \'    upload_command: ""\', \'    download_command: ""\', \'    list_command: ""\', \'    delete_command: ""\', \'    command_timeout: 4h\', \'    commandtimeoutduration: 4h0m0s\']'"""
+default_config = r"""'[\'general:\', \'    remote_storage: none\', \'    disable_progress_bar: true\', \'    backups_to_keep_local: 0\', \'    backups_to_keep_remote: 0\', \'    log_level: info\', \'    allow_empty_backups: false\', \'    use_resumable_state: true\', \'    restore_schema_on_cluster: ""\', \'    upload_by_part: true\', \'    download_by_part: true\', \'    restore_database_mapping: {}\', \'    retries_on_failure: 3\', \'    retries_pause: 30s\', \'    watch_interval: 1h\', \'    full_interval: 24h\', \'    watch_backup_name_template: shard{shard}-{type}-{time:20060102150405}\', \'    sharded_operation_mode: ""\', \'    retriesduration: 100ms\', \'    watchduration: 1h0m0s\', \'    fullduration: 24h0m0s\', \'clickhouse:\', \'    username: default\', \'    password: ""\', \'    host: localhost\', \'    port: 9000\', \'    disk_mapping: {}\', \'    skip_tables:\', \'        - system.*\', \'        - INFORMATION_SCHEMA.*\', \'        - information_schema.*\', \'        - _temporary_and_external_tables.*\', \'    skip_table_engines: []\', \'    timeout: 5m\', \'    freeze_by_part: false\', \'    freeze_by_part_where: ""\', \'    use_embedded_backup_restore: false\', \'    embedded_backup_disk: ""\', \'    backup_mutations: true\', \'    restore_as_attach: false\', \'    check_parts_columns: true\', \'    secure: false\', \'    skip_verify: false\', \'    sync_replicated_tables: false\', \'    log_sql_queries: true\', \'    config_dir: /etc/clickhouse-server/\', \'    restart_command: exec:systemctl restart clickhouse-server\', \'    ignore_not_exists_error_during_freeze: true\', \'    check_replicas_before_attach: true\', \'    tls_key: ""\', \'    tls_cert: ""\', \'    tls_ca: ""\', \'    debug: false\', \'s3:\', \'    access_key: ""\', \'    secret_key: ""\', \'    bucket: ""\', \'    endpoint: ""\', \'    region: us-east-1\', \'    acl: private\', \'    assume_role_arn: ""\', \'    force_path_style: false\', \'    path: ""\', \'    object_disk_path: ""\', \'    disable_ssl: false\', \'    compression_level: 1\', \'    compression_format: tar\', \'    sse: ""\', \'    sse_kms_key_id: ""\', \'    sse_customer_algorithm: ""\', \'    sse_customer_key: ""\', \'    sse_customer_key_md5: ""\', \'    sse_kms_encryption_context: ""\', \'    disable_cert_verification: false\', \'    use_custom_storage_class: false\', \'    storage_class: STANDARD\', \'    custom_storage_class_map: {}\', \'    part_size: 0\', \'    allow_multipart_download: false\', \'    object_labels: {}\', \'    debug: false\', \'gcs:\', \'    credentials_file: ""\', \'    credentials_json: ""\', \'    credentials_json_encoded: ""\', \'    bucket: ""\', \'    path: ""\', \'    object_disk_path: ""\', \'    compression_level: 1\', \'    compression_format: tar\', \'    debug: false\', \'    endpoint: ""\', \'    storage_class: STANDARD\', \'    object_labels: {}\', \'    custom_storage_class_map: {}\', \'    client_pool_size: 500\', \'cos:\', \'    url: ""\', \'    timeout: 2m\', \'    secret_id: ""\', \'    secret_key: ""\', \'    path: ""\', \'    compression_format: tar\', \'    compression_level: 1\', \'    debug: false\', \'api:\', \'    listen: localhost:7171\', \'    enable_metrics: true\', \'    enable_pprof: false\', \'    username: ""\', \'    password: ""\', \'    secure: false\', \'    certificate_file: ""\', \'    private_key_file: ""\', \'    ca_cert_file: ""\', \'    ca_key_file: ""\', \'    create_integration_tables: false\', \'    integration_tables_host: ""\', \'    allow_parallel: false\', \'    complete_resumable_after_restart: true\', \'ftp:\', \'    address: ""\', \'    timeout: 2m\', \'    username: ""\', \'    password: ""\', \'    tls: false\', \'    path: ""\', \'    object_disk_path: ""\', \'    compression_format: tar\', \'    compression_level: 1\', \'    debug: false\', \'sftp:\', \'    address: ""\', \'    port: 22\', \'    username: ""\', \'    password: ""\', \'    key: ""\', \'    path: ""\', \'    object_disk_path: ""\', \'    compression_format: tar\', \'    compression_level: 1\', \'    debug: false\', \'azblob:\', \'    endpoint_schema: https\', \'    endpoint_suffix: core.windows.net\', \'    account_name: ""\', \'    account_key: ""\', \'    sas: ""\', \'    use_managed_identity: false\', \'    container: ""\', \'    path: ""\', \'    object_disk_path: ""\', \'    compression_level: 1\', \'    compression_format: tar\', \'    sse_key: ""\', \'    buffer_size: 0\', \'    buffer_count: 3\', \'    timeout: 15m\', \'custom:\', \'    upload_command: ""\', \'    download_command: ""\', \'    list_command: ""\', \'    delete_command: ""\', \'    command_timeout: 4h\', \'    commandtimeoutduration: 4h0m0s\']'"""
 
 help_flag = r"""'NAME:\n   clickhouse-backup - Tool for easy backup of ClickHouse with cloud supportUSAGE:\n   clickhouse-backup <command> [-t, --tables=<db>.<table>] <backup_name>DESCRIPTION:\n   Run as \'root\' or \'clickhouse\' userCOMMANDS:\n   tables               List of tables, exclude skip_tables\n   create               Create new backup\n   create_remote        Create and upload new backup\n   upload               Upload backup to remote storage\n   list                 List of backups\n   download             Download backup from remote storage\n   restore              Create schema and restore data from backup\n   restore_remote       Download and restore\n   delete               Delete specific backup\n   default-config       Print default config\n   print-config         Print current config merged with environment variables\n   clean                Remove data in \'shadow\' folder from all \'path\' folders available from \'system.disks\'\n   clean_remote_broken  Remove all broken remote backups\n   watch                Run infinite loop which create full + incremental backup sequence to allow efficient backup sequences\n   server               Run API server\n   help, h              Shows a list of commands or help for one commandGLOBAL OPTIONS:\n   --config value, -c value  Config \'FILE\' name. (default: "/etc/clickhouse-backup/config.yml") [$CLICKHOUSE_BACKUP_CONFIG]\n   --help, -h                show help\n   --version, -v             print the version'"""
 

Original file line number	Diff line number	Diff line change
`@@ -77,7 +77,7 @@ type GCSConfig struct {`
`77`	`77`	StorageClass string `yaml:"storage_class" envconfig:"GCS_STORAGE_CLASS"`
`78`	`78`	ObjectLabels map[string]string `yaml:"object_labels" envconfig:"GCS_OBJECT_LABELS"`
`79`	`79`	CustomStorageClassMap map[string]string `yaml:"custom_storage_class_map" envconfig:"GCS_CUSTOM_STORAGE_CLASS_MAP"`
`80`		`- // NOTE: ClientPoolSize should be atleast 2 times bigger than`
	`80`	`+ // NOTE: ClientPoolSize should be at least 2 times bigger than`
`81`	`81`	`// UploadConcurrency or DownloadConcurrency in each upload and download case`
`82`	`82`	ClientPoolSize int `yaml:"client_pool_size" envconfig:"GCS_CLIENT_POOL_SIZE"`
`83`	`83`	`}`
Original file line number	Diff line number	Diff line change
`@@ -1,4 +1,4 @@`
`1`		-default_config = r"""'[\'general:\', \' remote_storage: none\', \' disable_progress_bar: true\', \' backups_to_keep_local: 0\', \' backups_to_keep_remote: 0\', \' log_level: info\', \' allow_empty_backups: false\', \' use_resumable_state: true\', \' restore_schema_on_cluster: ""\', \' upload_by_part: true\', \' download_by_part: true\', \' restore_database_mapping: {}\', \' retries_on_failure: 3\', \' retries_pause: 30s\', \' watch_interval: 1h\', \' full_interval: 24h\', \' watch_backup_name_template: shard{shard}-{type}-{time:20060102150405}\', \' sharded_operation_mode: ""\', \' retriesduration: 100ms\', \' watchduration: 1h0m0s\', \' fullduration: 24h0m0s\', \'clickhouse:\', \' username: default\', \' password: ""\', \' host: localhost\', \' port: 9000\', \' disk_mapping: {}\', \' skip_tables:\', \' - system.\', \' - INFORMATION_SCHEMA.\', \' - information_schema.\', \' - _temporary_and_external_tables.\', \' skip_table_engines: []\', \' timeout: 5m\', \' freeze_by_part: false\', \' freeze_by_part_where: ""\', \' use_embedded_backup_restore: false\', \' embedded_backup_disk: ""\', \' backup_mutations: true\', \' restore_as_attach: false\', \' check_parts_columns: true\', \' secure: false\', \' skip_verify: false\', \' sync_replicated_tables: false\', \' log_sql_queries: true\', \' config_dir: /etc/clickhouse-server/\', \' restart_command: exec:systemctl restart clickhouse-server\', \' ignore_not_exists_error_during_freeze: true\', \' check_replicas_before_attach: true\', \' tls_key: ""\', \' tls_cert: ""\', \' tls_ca: ""\', \' debug: false\', \'s3:\', \' access_key: ""\', \' secret_key: ""\', \' bucket: ""\', \' endpoint: ""\', \' region: us-east-1\', \' acl: private\', \' assume_role_arn: ""\', \' force_path_style: false\', \' path: ""\', \' object_disk_path: ""\', \' disable_ssl: false\', \' compression_level: 1\', \' compression_format: tar\', \' sse: ""\', \' sse_kms_key_id: ""\', \' sse_customer_algorithm: ""\', \' sse_customer_key: ""\', \' sse_customer_key_md5: ""\', \' sse_kms_encryption_context: ""\', \' disable_cert_verification: false\', \' use_custom_storage_class: false\', \' storage_class: STANDARD\', \' custom_storage_class_map: {}\', \' part_size: 0\', \' allow_multipart_download: false\', \' object_labels: {}\', \' debug: false\', \'gcs:\', \' credentials_file: ""\', \' credentials_json: ""\', \' credentials_json_encoded: ""\', \' bucket: ""\', \' path: ""\', \' object_disk_path: ""\', \' compression_level: 1\', \' compression_format: tar\', \' debug: false\', \' endpoint: ""\', \' storage_class: STANDARD\', \' object_labels: {}\', \' custom_storage_class_map: {}\', \'cos:\', \' url: ""\', \' timeout: 2m\', \' secret_id: ""\', \' secret_key: ""\', \' path: ""\', \' compression_format: tar\', \' compression_level: 1\', \' debug: false\', \'api:\', \' listen: localhost:7171\', \' enable_metrics: true\', \' enable_pprof: false\', \' username: ""\', \' password: ""\', \' secure: false\', \' certificate_file: ""\', \' private_key_file: ""\', \' ca_cert_file: ""\', \' ca_key_file: ""\', \' create_integration_tables: false\', \' integration_tables_host: ""\', \' allow_parallel: false\', \' complete_resumable_after_restart: true\', \'ftp:\', \' address: ""\', \' timeout: 2m\', \' username: ""\', \' password: ""\', \' tls: false\', \' path: ""\', \' object_disk_path: ""\', \' compression_format: tar\', \' compression_level: 1\', \' debug: false\', \'sftp:\', \' address: ""\', \' port: 22\', \' username: ""\', \' password: ""\', \' key: ""\', \' path: ""\', \' object_disk_path: ""\', \' compression_format: tar\', \' compression_level: 1\', \' debug: false\', \'azblob:\', \' endpoint_schema: https\', \' endpoint_suffix: core.windows.net\', \' account_name: ""\', \' account_key: ""\', \' sas: ""\', \' use_managed_identity: false\', \' container: ""\', \' path: ""\', \' object_disk_path: ""\', \' compression_level: 1\', \' compression_format: tar\', \' sse_key: ""\', \' buffer_size: 0\', \' buffer_count: 3\', \' timeout: 15m\', \'custom:\', \' upload_command: ""\', \' download_command: ""\', \' list_command: ""\', \' delete_command: ""\', \' command_timeout: 4h\', \' commandtimeoutduration: 4h0m0s\']'"""
	`1`	+default_config = r"""'[\'general:\', \' remote_storage: none\', \' disable_progress_bar: true\', \' backups_to_keep_local: 0\', \' backups_to_keep_remote: 0\', \' log_level: info\', \' allow_empty_backups: false\', \' use_resumable_state: true\', \' restore_schema_on_cluster: ""\', \' upload_by_part: true\', \' download_by_part: true\', \' restore_database_mapping: {}\', \' retries_on_failure: 3\', \' retries_pause: 30s\', \' watch_interval: 1h\', \' full_interval: 24h\', \' watch_backup_name_template: shard{shard}-{type}-{time:20060102150405}\', \' sharded_operation_mode: ""\', \' retriesduration: 100ms\', \' watchduration: 1h0m0s\', \' fullduration: 24h0m0s\', \'clickhouse:\', \' username: default\', \' password: ""\', \' host: localhost\', \' port: 9000\', \' disk_mapping: {}\', \' skip_tables:\', \' - system.\', \' - INFORMATION_SCHEMA.\', \' - information_schema.\', \' - _temporary_and_external_tables.\', \' skip_table_engines: []\', \' timeout: 5m\', \' freeze_by_part: false\', \' freeze_by_part_where: ""\', \' use_embedded_backup_restore: false\', \' embedded_backup_disk: ""\', \' backup_mutations: true\', \' restore_as_attach: false\', \' check_parts_columns: true\', \' secure: false\', \' skip_verify: false\', \' sync_replicated_tables: false\', \' log_sql_queries: true\', \' config_dir: /etc/clickhouse-server/\', \' restart_command: exec:systemctl restart clickhouse-server\', \' ignore_not_exists_error_during_freeze: true\', \' check_replicas_before_attach: true\', \' tls_key: ""\', \' tls_cert: ""\', \' tls_ca: ""\', \' debug: false\', \'s3:\', \' access_key: ""\', \' secret_key: ""\', \' bucket: ""\', \' endpoint: ""\', \' region: us-east-1\', \' acl: private\', \' assume_role_arn: ""\', \' force_path_style: false\', \' path: ""\', \' object_disk_path: ""\', \' disable_ssl: false\', \' compression_level: 1\', \' compression_format: tar\', \' sse: ""\', \' sse_kms_key_id: ""\', \' sse_customer_algorithm: ""\', \' sse_customer_key: ""\', \' sse_customer_key_md5: ""\', \' sse_kms_encryption_context: ""\', \' disable_cert_verification: false\', \' use_custom_storage_class: false\', \' storage_class: STANDARD\', \' custom_storage_class_map: {}\', \' part_size: 0\', \' allow_multipart_download: false\', \' object_labels: {}\', \' debug: false\', \'gcs:\', \' credentials_file: ""\', \' credentials_json: ""\', \' credentials_json_encoded: ""\', \' bucket: ""\', \' path: ""\', \' object_disk_path: ""\', \' compression_level: 1\', \' compression_format: tar\', \' debug: false\', \' endpoint: ""\', \' storage_class: STANDARD\', \' object_labels: {}\', \' custom_storage_class_map: {}\', \' client_pool_size: 500\', \'cos:\', \' url: ""\', \' timeout: 2m\', \' secret_id: ""\', \' secret_key: ""\', \' path: ""\', \' compression_format: tar\', \' compression_level: 1\', \' debug: false\', \'api:\', \' listen: localhost:7171\', \' enable_metrics: true\', \' enable_pprof: false\', \' username: ""\', \' password: ""\', \' secure: false\', \' certificate_file: ""\', \' private_key_file: ""\', \' ca_cert_file: ""\', \' ca_key_file: ""\', \' create_integration_tables: false\', \' integration_tables_host: ""\', \' allow_parallel: false\', \' complete_resumable_after_restart: true\', \'ftp:\', \' address: ""\', \' timeout: 2m\', \' username: ""\', \' password: ""\', \' tls: false\', \' path: ""\', \' object_disk_path: ""\', \' compression_format: tar\', \' compression_level: 1\', \' debug: false\', \'sftp:\', \' address: ""\', \' port: 22\', \' username: ""\', \' password: ""\', \' key: ""\', \' path: ""\', \' object_disk_path: ""\', \' compression_format: tar\', \' compression_level: 1\', \' debug: false\', \'azblob:\', \' endpoint_schema: https\', \' endpoint_suffix: core.windows.net\', \' account_name: ""\', \' account_key: ""\', \' sas: ""\', \' use_managed_identity: false\', \' container: ""\', \' path: ""\', \' object_disk_path: ""\', \' compression_level: 1\', \' compression_format: tar\', \' sse_key: ""\', \' buffer_size: 0\', \' buffer_count: 3\', \' timeout: 15m\', \'custom:\', \' upload_command: ""\', \' download_command: ""\', \' list_command: ""\', \' delete_command: ""\', \' command_timeout: 4h\', \' commandtimeoutduration: 4h0m0s\']'"""
`2`	`2`
`3`	`3`	help_flag = r"""'NAME:\n clickhouse-backup - Tool for easy backup of ClickHouse with cloud supportUSAGE:\n clickhouse-backup <command> [-t, --tables=<db>.<table>] <backup_name>DESCRIPTION:\n Run as \'root\' or \'clickhouse\' userCOMMANDS:\n tables List of tables, exclude skip_tables\n create Create new backup\n create_remote Create and upload new backup\n upload Upload backup to remote storage\n list List of backups\n download Download backup from remote storage\n restore Create schema and restore data from backup\n restore_remote Download and restore\n delete Delete specific backup\n default-config Print default config\n print-config Print current config merged with environment variables\n clean Remove data in \'shadow\' folder from all \'path\' folders available from \'system.disks\'\n clean_remote_broken Remove all broken remote backups\n watch Run infinite loop which create full + incremental backup sequence to allow efficient backup sequences\n server Run API server\n help, h Shows a list of commands or help for one commandGLOBAL OPTIONS:\n --config value, -c value Config \'FILE\' name. (default: "/etc/clickhouse-backup/config.yml") [$CLICKHOUSE_BACKUP_CONFIG]\n --help, -h show help\n --version, -v print the version'"""
`4`	`4`