diff --git a/Makefile b/Makefile index 1df5219140d..cee6312d75c 100644 --- a/Makefile +++ b/Makefile @@ -352,7 +352,7 @@ check: check-copyright generate_mock go-generate fmt check-static tidy terror_ch @git --no-pager diff --exit-code || (echo "Please add changed files!" && false) fast_check: check-copyright fmt check-static tidy terror_check errdoc \ - check-merge-conflicts check-ticdc-dashboard check-diff-line-width check-makefiles \ + check-merge-conflicts check-ticdc-dashboard check-diff-line-width swagger-spec check-makefiles \ check_cdc_integration_test check_dm_integration_test check_engine_integration_test @git --no-pager diff --exit-code || (echo "Please add changed files!" && false) @@ -365,6 +365,9 @@ else go tool cover -html "$(TEST_DIR)/all_cov.out" -o "$(TEST_DIR)/all_cov.html" endif +swagger-spec: tools/bin/swag + tools/bin/swag init --exclude dm,engine --parseVendor -generalInfo cdc/api/v1/api.go --output docs/swagger + unit_test_coverage: grep -vE ".*.pb.go|$(CDC_PKG)/testing_utils/.*|$(CDC_PKG)/cdc/sink/simple_mysql_tester.go|.*.__failpoint_binding__.go" "$(TEST_DIR)/cov.unit.out" > "$(TEST_DIR)/unit_cov.out" go tool cover -html "$(TEST_DIR)/unit_cov.out" -o "$(TEST_DIR)/unit_cov.html" diff --git a/docs/swagger/docs.go b/docs/swagger/docs.go index ca7a3347066..5538290ee95 100644 --- a/docs/swagger/docs.go +++ b/docs/swagger/docs.go @@ -1103,6 +1103,57 @@ var doc = `{ } } }, + "/api/v2/changefeeds/{changefeed_id}/synced": { + "get": { + "description": "get the synced status of a changefeed", + "consumes": [ + "application/json" + ], + "produces": [ + "application/json" + ], + "tags": [ + "changefeed", + "v2" + ], + "summary": "Get synced status", + "parameters": [ + { + "type": "string", + "description": "changefeed_id", + "name": "changefeed_id", + "in": "path", + "required": true + }, + { + "type": "string", + "description": "default", + "name": "namespace", + "in": "query" + } + ], + "responses": { + "200": { + "description": "OK", + "schema": { + "$ref": "#/definitions/v2.SyncedStatus" + } + }, + "400": { + "description": "Bad Request", + "schema": { + "$ref": "#/definitions/model.HTTPError" + } + }, + "500": { + "description": "Internal Server Error", + "schema": { + "$ref": "#/definitions/model.HTTPError" + } + } + } + } + }, "/api/v2/health": { "get": { "description": "Check the health status of a TiCDC cluster", @@ -1350,7 +1401,7 @@ var doc = `{ "type": "string" }, "delimiter": { - "description": "delimiter between fields", + "description": "delimiter between fields, it can be 1 character or at most 2 characters\nIt can not be CR or LF or contains CR or LF.\nIt should have exclusive characters with quote.", "type": "string" }, "include-commit-ts": { @@ -1361,6 +1412,14 @@ var doc = `{ "description": "representation of null values", "type": "string" }, + "output-handle-key": { + "description": "output handle key", + "type": "boolean" + }, + "output-old-value": { + "description": "output old value", + "type": "boolean" + }, "quote": { "description": "quoting character", "type": "string" @@ -1370,9 +1429,18 @@ var doc = `{ "config.CloudStorageConfig": { "type": "object", "properties": { + "file-cleanup-cron-spec": { + "type": "string" + }, + "file-expiration-days": { + "type": "integer" + }, "file-size": { "type": "integer" }, + "flush-concurrency": { + "type": "integer" + }, "flush-interval": { "type": "string" }, @@ -1399,6 +1467,9 @@ var doc = `{ "enable-tidb-extension": { "type": "boolean" }, + "encoding-format": { + "type": "string" + }, "max-batch-size": { "type": "integer" } @@ -1742,10 +1813,10 @@ var doc = `{ "description": "SendTimeout specifies the timeout for a message that has not been acknowledged by the server since sent.\nSend and SendAsync returns an error after timeout.\ndefault: 30s", "type": "integer" }, - "tls-certificate-path": { + "tls-certificate-file": { "type": "string" }, - "tls-private-key-path": { + "tls-key-file-path": { "type": "string" }, "tls-trust-certs-file-path": { @@ -1768,12 +1839,15 @@ var doc = `{ "$ref": "#/definitions/config.CloudStorageConfig" }, "column-selectors": { - "description": "ColumnSelectors is Deprecated.", "type": "array", "items": { "$ref": "#/definitions/config.ColumnSelector" } }, + "content-compatible": { + "description": "ContentCompatible is only available when the downstream is MQ.", + "type": "boolean" + }, "csv": { "description": "CSVConfig is only available when the downstream is Storage.", "$ref": "#/definitions/config.CSVConfig" @@ -1782,6 +1856,10 @@ var doc = `{ "description": "DateSeparator is only available when the downstream is Storage.", "type": "string" }, + "debezium-disable-schema": { + "description": "Debezium only. Whether schema should be excluded in the output.", + "type": "boolean" + }, "delete-only-output-handle-key-columns": { "description": "DeleteOnlyOutputHandleKeyColumns is only available when the downstream is MQ.", "type": "boolean" @@ -1834,6 +1912,18 @@ var doc = `{ "description": "SchemaRegistry is only available when the downstream is MQ using avro protocol.", "type": "string" }, + "send-bootstrap-in-msg-count": { + "description": "SendBootstrapInMsgCount means bootstrap messages are being sent every SendBootstrapInMsgCount row change messages.", + "type": "integer" + }, + "send-bootstrap-interval-in-sec": { + "description": "Simple Protocol only config, use to control the behavior of sending bootstrap message.\nNote: When one of the following conditions is set to negative value,\nbootstrap sending function will be disabled.\nSendBootstrapIntervalInSec is the interval in seconds to send bootstrap message.", + "type": "integer" + }, + "send-bootstrap-to-all-partition": { + "description": "SendBootstrapToAllPartition determines whether to send bootstrap message to all partitions.\nIf set to false, bootstrap message will only be sent to the first partition of each topic.\nDefault value is true.", + "type": "boolean" + }, "terminator": { "description": "Terminator is NOT available when the downstream is DB.", "type": "string" @@ -2143,6 +2233,12 @@ var doc = `{ "null": { "type": "string" }, + "output_handle_key": { + "type": "boolean" + }, + "output_old_value": { + "type": "boolean" + }, "quote": { "type": "string" } @@ -2314,9 +2410,18 @@ var doc = `{ "v2.CloudStorageConfig": { "type": "object", "properties": { + "file_cleanup_cron_spec": { + "type": "string" + }, + "file_expiration_days": { + "type": "integer" + }, "file_size": { "type": "integer" }, + "flush_concurrency": { + "type": "integer" + }, "flush_interval": { "type": "string" }, @@ -2343,6 +2448,9 @@ var doc = `{ "enable_tidb_extension": { "type": "boolean" }, + "encoding_format": { + "type": "string" + }, "max_batch_size": { "type": "integer" } @@ -2368,15 +2476,33 @@ var doc = `{ "v2.ConsistentConfig": { "type": "object", "properties": { + "compression": { + "type": "string" + }, + "encoding_worker_num": { + "type": "integer" + }, + "flush_concurrency": { + "type": "integer" + }, "flush_interval": { "type": "integer" }, + "flush_worker_num": { + "type": "integer" + }, "level": { "type": "string" }, "max_log_size": { "type": "integer" }, + "memory_usage": { + "$ref": "#/definitions/v2.ConsistentMemoryUsage" + }, + "meta_flush_interval": { + "type": "integer" + }, "storage": { "type": "string" }, @@ -2385,6 +2511,14 @@ var doc = `{ } } }, + "v2.ConsistentMemoryUsage": { + "type": "object", + "properties": { + "memory_quota_percentage": { + "type": "integer" + } + } + }, "v2.DispatchRule": { "type": "object", "properties": { @@ -2454,40 +2588,12 @@ var doc = `{ "v2.FilterConfig": { "type": "object", "properties": { - "do_dbs": { - "description": "DoDBs is an allowlist of schemas.", - "type": "array", - "items": { - "type": "string" - } - }, - "do_tables": { - "description": "DoTables is an allowlist of tables.", - "type": "array", - "items": { - "$ref": "#/definitions/v2.Table" - } - }, "event_filters": { "type": "array", "items": { "$ref": "#/definitions/v2.EventFilterRule" } }, - "ignore_dbs": { - "description": "IgnoreDBs is a blocklist of schemas.", - "type": "array", - "items": { - "type": "string" - } - }, - "ignore_tables": { - "description": "IgnoreTables is a blocklist of tables.", - "type": "array", - "items": { - "$ref": "#/definitions/v2.Table" - } - }, "ignore_txn_start_ts": { "type": "array", "items": { @@ -2537,6 +2643,9 @@ var doc = `{ } } }, + "v2.JSONDuration": { + "type": "object" + }, "v2.KafkaConfig": { "type": "object", "properties": { @@ -2847,6 +2956,9 @@ var doc = `{ "case_sensitive": { "type": "boolean" }, + "changefeed_error_stuck_duration": { + "$ref": "#/definitions/v2.JSONDuration" + }, "check_gc_safe_point": { "type": "boolean" }, @@ -2856,6 +2968,9 @@ var doc = `{ "enable_sync_point": { "type": "boolean" }, + "enable_table_monitor": { + "type": "boolean" + }, "filter": { "$ref": "#/definitions/v2.FilterConfig" }, @@ -2880,11 +2995,18 @@ var doc = `{ "sink": { "$ref": "#/definitions/v2.SinkConfig" }, + "sql_mode": { + "description": "Deprecated: we don't use this field since v8.0.0.", + "type": "string" + }, "sync_point_interval": { "type": "string" }, "sync_point_retention": { "type": "string" + }, + "synced_status": { + "$ref": "#/definitions/v2.SyncedStatusConfig" } } }, @@ -2975,12 +3097,18 @@ var doc = `{ "$ref": "#/definitions/v2.ColumnSelector" } }, + "content_compatible": { + "type": "boolean" + }, "csv": { "$ref": "#/definitions/v2.CSVConfig" }, "date_separator": { "type": "string" }, + "debezium_disable_schema": { + "type": "boolean" + }, "delete_only_output_handle_key_columns": { "type": "boolean" }, @@ -3023,6 +3151,15 @@ var doc = `{ "schema_registry": { "type": "string" }, + "send_bootstrap_in_msg_count": { + "type": "integer" + }, + "send_bootstrap_interval_in_sec": { + "type": "integer" + }, + "send_bootstrap_to_all_partition": { + "type": "boolean" + }, "terminator": { "type": "string" }, @@ -3031,16 +3168,39 @@ var doc = `{ } } }, - "v2.Table": { + "v2.SyncedStatus": { "type": "object", "properties": { - "database_name": { - "description": "Schema is the name of the schema (database) containing this table.", + "info": { + "type": "string" + }, + "last_synced_ts": { "type": "string" }, - "table_name": { - "description": "Name is the unqualified table name.", + "now_ts": { "type": "string" + }, + "puller_resolved_ts": { + "type": "string" + }, + "sink_checkpoint_ts": { + "type": "string" + }, + "synced": { + "type": "boolean" + } + } + }, + "v2.SyncedStatusConfig": { + "type": "object", + "properties": { + "checkpoint_interval": { + "description": "The maximum interval between latest checkpoint ts and now or\nbetween latest sink's checkpoint ts and puller's checkpoint ts required to reach synced state", + "type": "integer" + }, + "synced_check_interval": { + "description": "The minimum interval between the latest synced ts and now required to reach synced state", + "type": "integer" } } } diff --git a/docs/swagger/swagger.json b/docs/swagger/swagger.json index d748dd19560..05a3f7e1b14 100644 --- a/docs/swagger/swagger.json +++ b/docs/swagger/swagger.json @@ -1084,6 +1084,57 @@ } } }, + "/api/v2/changefeeds/{changefeed_id}/synced": { + "get": { + "description": "get the synced status of a changefeed", + "consumes": [ + "application/json" + ], + "produces": [ + "application/json" + ], + "tags": [ + "changefeed", + "v2" + ], + "summary": "Get synced status", + "parameters": [ + { + "type": "string", + "description": "changefeed_id", + "name": "changefeed_id", + "in": "path", + "required": true + }, + { + "type": "string", + "description": "default", + "name": "namespace", + "in": "query" + } + ], + "responses": { + "200": { + "description": "OK", + "schema": { + "$ref": "#/definitions/v2.SyncedStatus" + } + }, + "400": { + "description": "Bad Request", + "schema": { + "$ref": "#/definitions/model.HTTPError" + } + }, + "500": { + "description": "Internal Server Error", + "schema": { + "$ref": "#/definitions/model.HTTPError" + } + } + } + } + }, "/api/v2/health": { "get": { "description": "Check the health status of a TiCDC cluster", @@ -1331,7 +1382,7 @@ "type": "string" }, "delimiter": { - "description": "delimiter between fields", + "description": "delimiter between fields, it can be 1 character or at most 2 characters\nIt can not be CR or LF or contains CR or LF.\nIt should have exclusive characters with quote.", "type": "string" }, "include-commit-ts": { @@ -1342,6 +1393,14 @@ "description": "representation of null values", "type": "string" }, + "output-handle-key": { + "description": "output handle key", + "type": "boolean" + }, + "output-old-value": { + "description": "output old value", + "type": "boolean" + }, "quote": { "description": "quoting character", "type": "string" @@ -1351,9 +1410,18 @@ "config.CloudStorageConfig": { "type": "object", "properties": { + "file-cleanup-cron-spec": { + "type": "string" + }, + "file-expiration-days": { + "type": "integer" + }, "file-size": { "type": "integer" }, + "flush-concurrency": { + "type": "integer" + }, "flush-interval": { "type": "string" }, @@ -1380,6 +1448,9 @@ "enable-tidb-extension": { "type": "boolean" }, + "encoding-format": { + "type": "string" + }, "max-batch-size": { "type": "integer" } @@ -1723,10 +1794,10 @@ "description": "SendTimeout specifies the timeout for a message that has not been acknowledged by the server since sent.\nSend and SendAsync returns an error after timeout.\ndefault: 30s", "type": "integer" }, - "tls-certificate-path": { + "tls-certificate-file": { "type": "string" }, - "tls-private-key-path": { + "tls-key-file-path": { "type": "string" }, "tls-trust-certs-file-path": { @@ -1749,12 +1820,15 @@ "$ref": "#/definitions/config.CloudStorageConfig" }, "column-selectors": { - "description": "ColumnSelectors is Deprecated.", "type": "array", "items": { "$ref": "#/definitions/config.ColumnSelector" } }, + "content-compatible": { + "description": "ContentCompatible is only available when the downstream is MQ.", + "type": "boolean" + }, "csv": { "description": "CSVConfig is only available when the downstream is Storage.", "$ref": "#/definitions/config.CSVConfig" @@ -1763,6 +1837,10 @@ "description": "DateSeparator is only available when the downstream is Storage.", "type": "string" }, + "debezium-disable-schema": { + "description": "Debezium only. Whether schema should be excluded in the output.", + "type": "boolean" + }, "delete-only-output-handle-key-columns": { "description": "DeleteOnlyOutputHandleKeyColumns is only available when the downstream is MQ.", "type": "boolean" @@ -1815,6 +1893,18 @@ "description": "SchemaRegistry is only available when the downstream is MQ using avro protocol.", "type": "string" }, + "send-bootstrap-in-msg-count": { + "description": "SendBootstrapInMsgCount means bootstrap messages are being sent every SendBootstrapInMsgCount row change messages.", + "type": "integer" + }, + "send-bootstrap-interval-in-sec": { + "description": "Simple Protocol only config, use to control the behavior of sending bootstrap message.\nNote: When one of the following conditions is set to negative value,\nbootstrap sending function will be disabled.\nSendBootstrapIntervalInSec is the interval in seconds to send bootstrap message.", + "type": "integer" + }, + "send-bootstrap-to-all-partition": { + "description": "SendBootstrapToAllPartition determines whether to send bootstrap message to all partitions.\nIf set to false, bootstrap message will only be sent to the first partition of each topic.\nDefault value is true.", + "type": "boolean" + }, "terminator": { "description": "Terminator is NOT available when the downstream is DB.", "type": "string" @@ -2124,6 +2214,12 @@ "null": { "type": "string" }, + "output_handle_key": { + "type": "boolean" + }, + "output_old_value": { + "type": "boolean" + }, "quote": { "type": "string" } @@ -2295,9 +2391,18 @@ "v2.CloudStorageConfig": { "type": "object", "properties": { + "file_cleanup_cron_spec": { + "type": "string" + }, + "file_expiration_days": { + "type": "integer" + }, "file_size": { "type": "integer" }, + "flush_concurrency": { + "type": "integer" + }, "flush_interval": { "type": "string" }, @@ -2324,6 +2429,9 @@ "enable_tidb_extension": { "type": "boolean" }, + "encoding_format": { + "type": "string" + }, "max_batch_size": { "type": "integer" } @@ -2349,15 +2457,33 @@ "v2.ConsistentConfig": { "type": "object", "properties": { + "compression": { + "type": "string" + }, + "encoding_worker_num": { + "type": "integer" + }, + "flush_concurrency": { + "type": "integer" + }, "flush_interval": { "type": "integer" }, + "flush_worker_num": { + "type": "integer" + }, "level": { "type": "string" }, "max_log_size": { "type": "integer" }, + "memory_usage": { + "$ref": "#/definitions/v2.ConsistentMemoryUsage" + }, + "meta_flush_interval": { + "type": "integer" + }, "storage": { "type": "string" }, @@ -2366,6 +2492,14 @@ } } }, + "v2.ConsistentMemoryUsage": { + "type": "object", + "properties": { + "memory_quota_percentage": { + "type": "integer" + } + } + }, "v2.DispatchRule": { "type": "object", "properties": { @@ -2435,40 +2569,12 @@ "v2.FilterConfig": { "type": "object", "properties": { - "do_dbs": { - "description": "DoDBs is an allowlist of schemas.", - "type": "array", - "items": { - "type": "string" - } - }, - "do_tables": { - "description": "DoTables is an allowlist of tables.", - "type": "array", - "items": { - "$ref": "#/definitions/v2.Table" - } - }, "event_filters": { "type": "array", "items": { "$ref": "#/definitions/v2.EventFilterRule" } }, - "ignore_dbs": { - "description": "IgnoreDBs is a blocklist of schemas.", - "type": "array", - "items": { - "type": "string" - } - }, - "ignore_tables": { - "description": "IgnoreTables is a blocklist of tables.", - "type": "array", - "items": { - "$ref": "#/definitions/v2.Table" - } - }, "ignore_txn_start_ts": { "type": "array", "items": { @@ -2518,6 +2624,9 @@ } } }, + "v2.JSONDuration": { + "type": "object" + }, "v2.KafkaConfig": { "type": "object", "properties": { @@ -2828,6 +2937,9 @@ "case_sensitive": { "type": "boolean" }, + "changefeed_error_stuck_duration": { + "$ref": "#/definitions/v2.JSONDuration" + }, "check_gc_safe_point": { "type": "boolean" }, @@ -2837,6 +2949,9 @@ "enable_sync_point": { "type": "boolean" }, + "enable_table_monitor": { + "type": "boolean" + }, "filter": { "$ref": "#/definitions/v2.FilterConfig" }, @@ -2861,11 +2976,18 @@ "sink": { "$ref": "#/definitions/v2.SinkConfig" }, + "sql_mode": { + "description": "Deprecated: we don't use this field since v8.0.0.", + "type": "string" + }, "sync_point_interval": { "type": "string" }, "sync_point_retention": { "type": "string" + }, + "synced_status": { + "$ref": "#/definitions/v2.SyncedStatusConfig" } } }, @@ -2956,12 +3078,18 @@ "$ref": "#/definitions/v2.ColumnSelector" } }, + "content_compatible": { + "type": "boolean" + }, "csv": { "$ref": "#/definitions/v2.CSVConfig" }, "date_separator": { "type": "string" }, + "debezium_disable_schema": { + "type": "boolean" + }, "delete_only_output_handle_key_columns": { "type": "boolean" }, @@ -3004,6 +3132,15 @@ "schema_registry": { "type": "string" }, + "send_bootstrap_in_msg_count": { + "type": "integer" + }, + "send_bootstrap_interval_in_sec": { + "type": "integer" + }, + "send_bootstrap_to_all_partition": { + "type": "boolean" + }, "terminator": { "type": "string" }, @@ -3012,16 +3149,39 @@ } } }, - "v2.Table": { + "v2.SyncedStatus": { "type": "object", "properties": { - "database_name": { - "description": "Schema is the name of the schema (database) containing this table.", + "info": { + "type": "string" + }, + "last_synced_ts": { "type": "string" }, - "table_name": { - "description": "Name is the unqualified table name.", + "now_ts": { "type": "string" + }, + "puller_resolved_ts": { + "type": "string" + }, + "sink_checkpoint_ts": { + "type": "string" + }, + "synced": { + "type": "boolean" + } + } + }, + "v2.SyncedStatusConfig": { + "type": "object", + "properties": { + "checkpoint_interval": { + "description": "The maximum interval between latest checkpoint ts and now or\nbetween latest sink's checkpoint ts and puller's checkpoint ts required to reach synced state", + "type": "integer" + }, + "synced_check_interval": { + "description": "The minimum interval between the latest synced ts and now required to reach synced state", + "type": "integer" } } } diff --git a/docs/swagger/swagger.yaml b/docs/swagger/swagger.yaml index ce21f335acb..36693b13470 100644 --- a/docs/swagger/swagger.yaml +++ b/docs/swagger/swagger.yaml @@ -5,7 +5,10 @@ definitions: description: encoding method of binary type type: string delimiter: - description: delimiter between fields + description: |- + delimiter between fields, it can be 1 character or at most 2 characters + It can not be CR or LF or contains CR or LF. + It should have exclusive characters with quote. type: string include-commit-ts: description: whether to include commit ts @@ -13,14 +16,26 @@ definitions: "null": description: representation of null values type: string + output-handle-key: + description: output handle key + type: boolean + output-old-value: + description: output old value + type: boolean quote: description: quoting character type: string type: object config.CloudStorageConfig: properties: + file-cleanup-cron-spec: + type: string + file-expiration-days: + type: integer file-size: type: integer + flush-concurrency: + type: integer flush-interval: type: string output-column-id: @@ -38,6 +53,8 @@ definitions: type: boolean enable-tidb-extension: type: boolean + encoding-format: + type: string max-batch-size: type: integer type: object @@ -291,9 +308,9 @@ definitions: Send and SendAsync returns an error after timeout. default: 30s type: integer - tls-certificate-path: + tls-certificate-file: type: string - tls-private-key-path: + tls-key-file-path: type: string tls-trust-certs-file-path: type: string @@ -313,16 +330,21 @@ definitions: cloud-storage-config: $ref: '#/definitions/config.CloudStorageConfig' column-selectors: - description: ColumnSelectors is Deprecated. items: $ref: '#/definitions/config.ColumnSelector' type: array + content-compatible: + description: ContentCompatible is only available when the downstream is MQ. + type: boolean csv: $ref: '#/definitions/config.CSVConfig' description: CSVConfig is only available when the downstream is Storage. date-separator: description: DateSeparator is only available when the downstream is Storage. type: string + debezium-disable-schema: + description: Debezium only. Whether schema should be excluded in the output. + type: boolean delete-only-output-handle-key-columns: description: DeleteOnlyOutputHandleKeyColumns is only available when the downstream is MQ. @@ -367,6 +389,23 @@ definitions: description: SchemaRegistry is only available when the downstream is MQ using avro protocol. type: string + send-bootstrap-in-msg-count: + description: SendBootstrapInMsgCount means bootstrap messages are being sent + every SendBootstrapInMsgCount row change messages. + type: integer + send-bootstrap-interval-in-sec: + description: |- + Simple Protocol only config, use to control the behavior of sending bootstrap message. + Note: When one of the following conditions is set to negative value, + bootstrap sending function will be disabled. + SendBootstrapIntervalInSec is the interval in seconds to send bootstrap message. + type: integer + send-bootstrap-to-all-partition: + description: |- + SendBootstrapToAllPartition determines whether to send bootstrap message to all partitions. + If set to false, bootstrap message will only be sent to the first partition of each topic. + Default value is true. + type: boolean terminator: description: Terminator is NOT available when the downstream is DB. type: string @@ -574,6 +613,10 @@ definitions: type: boolean "null": type: string + output_handle_key: + type: boolean + output_old_value: + type: boolean quote: type: string type: object @@ -693,8 +736,14 @@ definitions: type: object v2.CloudStorageConfig: properties: + file_cleanup_cron_spec: + type: string + file_expiration_days: + type: integer file_size: type: integer + flush_concurrency: + type: integer flush_interval: type: string output_column_id: @@ -712,6 +761,8 @@ definitions: type: boolean enable_tidb_extension: type: boolean + encoding_format: + type: string max_batch_size: type: integer type: object @@ -728,17 +779,34 @@ definitions: type: object v2.ConsistentConfig: properties: + compression: + type: string + encoding_worker_num: + type: integer + flush_concurrency: + type: integer flush_interval: type: integer + flush_worker_num: + type: integer level: type: string max_log_size: type: integer + memory_usage: + $ref: '#/definitions/v2.ConsistentMemoryUsage' + meta_flush_interval: + type: integer storage: type: string use_file_backend: type: boolean type: object + v2.ConsistentMemoryUsage: + properties: + memory_quota_percentage: + type: integer + type: object v2.DispatchRule: properties: columns: @@ -785,30 +853,10 @@ definitions: type: object v2.FilterConfig: properties: - do_dbs: - description: DoDBs is an allowlist of schemas. - items: - type: string - type: array - do_tables: - description: DoTables is an allowlist of tables. - items: - $ref: '#/definitions/v2.Table' - type: array event_filters: items: $ref: '#/definitions/v2.EventFilterRule' type: array - ignore_dbs: - description: IgnoreDBs is a blocklist of schemas. - items: - type: string - type: array - ignore_tables: - description: IgnoreTables is a blocklist of tables. - items: - $ref: '#/definitions/v2.Table' - type: array ignore_txn_start_ts: items: type: integer @@ -842,6 +890,8 @@ definitions: integrity_check_level: type: string type: object + v2.JSONDuration: + type: object v2.KafkaConfig: properties: auto_create_topic: @@ -1046,12 +1096,16 @@ definitions: type: boolean case_sensitive: type: boolean + changefeed_error_stuck_duration: + $ref: '#/definitions/v2.JSONDuration' check_gc_safe_point: type: boolean consistent: $ref: '#/definitions/v2.ConsistentConfig' enable_sync_point: type: boolean + enable_table_monitor: + type: boolean filter: $ref: '#/definitions/v2.FilterConfig' force_replicate: @@ -1068,10 +1122,15 @@ definitions: $ref: '#/definitions/v2.ChangefeedSchedulerConfig' sink: $ref: '#/definitions/v2.SinkConfig' + sql_mode: + description: 'Deprecated: we don''t use this field since v8.0.0.' + type: string sync_point_interval: type: string sync_point_retention: type: string + synced_status: + $ref: '#/definitions/v2.SyncedStatusConfig' type: object v2.ResumeChangefeedConfig: properties: @@ -1130,10 +1189,14 @@ definitions: items: $ref: '#/definitions/v2.ColumnSelector' type: array + content_compatible: + type: boolean csv: $ref: '#/definitions/v2.CSVConfig' date_separator: type: string + debezium_disable_schema: + type: boolean delete_only_output_handle_key_columns: type: boolean dispatchers: @@ -1162,19 +1225,43 @@ definitions: type: boolean schema_registry: type: string + send_bootstrap_in_msg_count: + type: integer + send_bootstrap_interval_in_sec: + type: integer + send_bootstrap_to_all_partition: + type: boolean terminator: type: string transaction_atomicity: type: string type: object - v2.Table: + v2.SyncedStatus: properties: - database_name: - description: Schema is the name of the schema (database) containing this table. + info: + type: string + last_synced_ts: type: string - table_name: - description: Name is the unqualified table name. + now_ts: type: string + puller_resolved_ts: + type: string + sink_checkpoint_ts: + type: string + synced: + type: boolean + type: object + v2.SyncedStatusConfig: + properties: + checkpoint_interval: + description: |- + The maximum interval between latest checkpoint ts and now or + between latest sink's checkpoint ts and puller's checkpoint ts required to reach synced state + type: integer + synced_check_interval: + description: The minimum interval between the latest synced ts and now required + to reach synced state + type: integer type: object info: contact: {} @@ -1894,6 +1981,40 @@ paths: tags: - changefeed - v2 + /api/v2/changefeeds/{changefeed_id}/synced: + get: + consumes: + - application/json + description: get the synced status of a changefeed + parameters: + - description: changefeed_id + in: path + name: changefeed_id + required: true + type: string + - description: default + in: query + name: namespace + type: string + produces: + - application/json + responses: + "200": + description: OK + schema: + $ref: '#/definitions/v2.SyncedStatus' + "400": + description: Bad Request + schema: + $ref: '#/definitions/model.HTTPError' + "500": + description: Internal Server Error + schema: + $ref: '#/definitions/model.HTTPError' + summary: Get synced status + tags: + - changefeed + - v2 /api/v2/health: get: description: Check the health status of a TiCDC cluster