diff --git a/Makefile b/Makefile index 4d1c39caa18..57f9fe36f9d 100644 --- a/Makefile +++ b/Makefile @@ -346,7 +346,7 @@ check: check-copyright generate_mock go-generate fmt check-static tidy terror_ch @git --no-pager diff --exit-code || (echo "Please add changed files!" && false) fast_check: check-copyright fmt check-static tidy terror_check errdoc \ - check-merge-conflicts check-ticdc-dashboard check-diff-line-width check-makefiles \ + check-merge-conflicts check-ticdc-dashboard check-diff-line-width swagger-spec check-makefiles \ check_cdc_integration_test check_dm_integration_test check_engine_integration_test @git --no-pager diff --exit-code || (echo "Please add changed files!" && false) @@ -359,6 +359,9 @@ else go tool cover -html "$(TEST_DIR)/all_cov.out" -o "$(TEST_DIR)/all_cov.html" endif +swagger-spec: tools/bin/swag + tools/bin/swag init --exclude dm,engine --parseVendor -generalInfo cdc/api/v1/api.go --output docs/swagger + unit_test_coverage: grep -vE ".*.pb.go|$(CDC_PKG)/testing_utils/.*|$(CDC_PKG)/cdc/sink/simple_mysql_tester.go|.*.__failpoint_binding__.go" "$(TEST_DIR)/cov.unit.out" > "$(TEST_DIR)/unit_cov.out" go tool cover -html "$(TEST_DIR)/unit_cov.out" -o "$(TEST_DIR)/unit_cov.html" diff --git a/docs/swagger/docs.go b/docs/swagger/docs.go index ca7a3347066..5e8e929f2e2 100644 --- a/docs/swagger/docs.go +++ b/docs/swagger/docs.go @@ -1103,6 +1103,57 @@ var doc = `{ } } }, + "/api/v2/changefeeds/{changefeed_id}/synced": { + "get": { + "description": "get the synced status of a changefeed", + "consumes": [ + "application/json" + ], + "produces": [ + "application/json" + ], + "tags": [ + "changefeed", + "v2" + ], + "summary": "Get synced status", + "parameters": [ + { + "type": "string", + "description": "changefeed_id", + "name": "changefeed_id", + "in": "path", + "required": true + }, + { + "type": "string", + "description": "default", + "name": "namespace", + "in": "query" + } + ], + "responses": { + "200": { + "description": "OK", + "schema": { + "$ref": "#/definitions/v2.SyncedStatus" + } + }, + "400": { + "description": "Bad Request", + "schema": { + "$ref": "#/definitions/model.HTTPError" + } + }, + "500": { + "description": "Internal Server Error", + "schema": { + "$ref": "#/definitions/model.HTTPError" + } + } + } + } + }, "/api/v2/health": { "get": { "description": "Check the health status of a TiCDC cluster", @@ -1370,9 +1421,18 @@ var doc = `{ "config.CloudStorageConfig": { "type": "object", "properties": { + "file-cleanup-cron-spec": { + "type": "string" + }, + "file-expiration-days": { + "type": "integer" + }, "file-size": { "type": "integer" }, + "flush-concurrency": { + "type": "integer" + }, "flush-interval": { "type": "string" }, @@ -1742,10 +1802,10 @@ var doc = `{ "description": "SendTimeout specifies the timeout for a message that has not been acknowledged by the server since sent.\nSend and SendAsync returns an error after timeout.\ndefault: 30s", "type": "integer" }, - "tls-certificate-path": { + "tls-certificate-file": { "type": "string" }, - "tls-private-key-path": { + "tls-key-file-path": { "type": "string" }, "tls-trust-certs-file-path": { @@ -1768,7 +1828,6 @@ var doc = `{ "$ref": "#/definitions/config.CloudStorageConfig" }, "column-selectors": { - "description": "ColumnSelectors is Deprecated.", "type": "array", "items": { "$ref": "#/definitions/config.ColumnSelector" @@ -2314,9 +2373,18 @@ var doc = `{ "v2.CloudStorageConfig": { "type": "object", "properties": { + "file_cleanup_cron_spec": { + "type": "string" + }, + "file_expiration_days": { + "type": "integer" + }, "file_size": { "type": "integer" }, + "flush_concurrency": { + "type": "integer" + }, "flush_interval": { "type": "string" }, @@ -2368,15 +2436,33 @@ var doc = `{ "v2.ConsistentConfig": { "type": "object", "properties": { + "compression": { + "type": "string" + }, + "encoding_worker_num": { + "type": "integer" + }, + "flush_concurrency": { + "type": "integer" + }, "flush_interval": { "type": "integer" }, + "flush_worker_num": { + "type": "integer" + }, "level": { "type": "string" }, "max_log_size": { "type": "integer" }, + "memory_usage": { + "$ref": "#/definitions/v2.ConsistentMemoryUsage" + }, + "meta_flush_interval": { + "type": "integer" + }, "storage": { "type": "string" }, @@ -2385,6 +2471,17 @@ var doc = `{ } } }, + "v2.ConsistentMemoryUsage": { + "type": "object", + "properties": { + "event_cache_percentage": { + "type": "integer" + }, + "memory_quota_percentage": { + "type": "integer" + } + } + }, "v2.DispatchRule": { "type": "object", "properties": { @@ -2537,6 +2634,9 @@ var doc = `{ } } }, + "v2.JSONDuration": { + "type": "object" + }, "v2.KafkaConfig": { "type": "object", "properties": { @@ -2847,6 +2947,9 @@ var doc = `{ "case_sensitive": { "type": "boolean" }, + "changefeed_error_stuck_duration": { + "$ref": "#/definitions/v2.JSONDuration" + }, "check_gc_safe_point": { "type": "boolean" }, @@ -2880,11 +2983,17 @@ var doc = `{ "sink": { "$ref": "#/definitions/v2.SinkConfig" }, + "sql_mode": { + "type": "string" + }, "sync_point_interval": { "type": "string" }, "sync_point_retention": { "type": "string" + }, + "synced_status": { + "$ref": "#/definitions/v2.SyncedStatusConfig" } } }, @@ -3031,6 +3140,42 @@ var doc = `{ } } }, + "v2.SyncedStatus": { + "type": "object", + "properties": { + "info": { + "type": "string" + }, + "last_synced_ts": { + "type": "string" + }, + "now_ts": { + "type": "string" + }, + "puller_resolved_ts": { + "type": "string" + }, + "sink_checkpoint_ts": { + "type": "string" + }, + "synced": { + "type": "boolean" + } + } + }, + "v2.SyncedStatusConfig": { + "type": "object", + "properties": { + "checkpoint_interval": { + "description": "The maximum interval between latest checkpoint ts and now or\nbetween latest sink's checkpoint ts and puller's checkpoint ts required to reach synced state", + "type": "integer" + }, + "synced_check_interval": { + "description": "The minimum interval between the latest synced ts and now required to reach synced state", + "type": "integer" + } + } + }, "v2.Table": { "type": "object", "properties": { diff --git a/docs/swagger/swagger.json b/docs/swagger/swagger.json index d748dd19560..2dede1a0e12 100644 --- a/docs/swagger/swagger.json +++ b/docs/swagger/swagger.json @@ -1084,6 +1084,57 @@ } } }, + "/api/v2/changefeeds/{changefeed_id}/synced": { + "get": { + "description": "get the synced status of a changefeed", + "consumes": [ + "application/json" + ], + "produces": [ + "application/json" + ], + "tags": [ + "changefeed", + "v2" + ], + "summary": "Get synced status", + "parameters": [ + { + "type": "string", + "description": "changefeed_id", + "name": "changefeed_id", + "in": "path", + "required": true + }, + { + "type": "string", + "description": "default", + "name": "namespace", + "in": "query" + } + ], + "responses": { + "200": { + "description": "OK", + "schema": { + "$ref": "#/definitions/v2.SyncedStatus" + } + }, + "400": { + "description": "Bad Request", + "schema": { + "$ref": "#/definitions/model.HTTPError" + } + }, + "500": { + "description": "Internal Server Error", + "schema": { + "$ref": "#/definitions/model.HTTPError" + } + } + } + } + }, "/api/v2/health": { "get": { "description": "Check the health status of a TiCDC cluster", @@ -1351,9 +1402,18 @@ "config.CloudStorageConfig": { "type": "object", "properties": { + "file-cleanup-cron-spec": { + "type": "string" + }, + "file-expiration-days": { + "type": "integer" + }, "file-size": { "type": "integer" }, + "flush-concurrency": { + "type": "integer" + }, "flush-interval": { "type": "string" }, @@ -1723,10 +1783,10 @@ "description": "SendTimeout specifies the timeout for a message that has not been acknowledged by the server since sent.\nSend and SendAsync returns an error after timeout.\ndefault: 30s", "type": "integer" }, - "tls-certificate-path": { + "tls-certificate-file": { "type": "string" }, - "tls-private-key-path": { + "tls-key-file-path": { "type": "string" }, "tls-trust-certs-file-path": { @@ -1749,7 +1809,6 @@ "$ref": "#/definitions/config.CloudStorageConfig" }, "column-selectors": { - "description": "ColumnSelectors is Deprecated.", "type": "array", "items": { "$ref": "#/definitions/config.ColumnSelector" @@ -2295,9 +2354,18 @@ "v2.CloudStorageConfig": { "type": "object", "properties": { + "file_cleanup_cron_spec": { + "type": "string" + }, + "file_expiration_days": { + "type": "integer" + }, "file_size": { "type": "integer" }, + "flush_concurrency": { + "type": "integer" + }, "flush_interval": { "type": "string" }, @@ -2349,15 +2417,33 @@ "v2.ConsistentConfig": { "type": "object", "properties": { + "compression": { + "type": "string" + }, + "encoding_worker_num": { + "type": "integer" + }, + "flush_concurrency": { + "type": "integer" + }, "flush_interval": { "type": "integer" }, + "flush_worker_num": { + "type": "integer" + }, "level": { "type": "string" }, "max_log_size": { "type": "integer" }, + "memory_usage": { + "$ref": "#/definitions/v2.ConsistentMemoryUsage" + }, + "meta_flush_interval": { + "type": "integer" + }, "storage": { "type": "string" }, @@ -2366,6 +2452,17 @@ } } }, + "v2.ConsistentMemoryUsage": { + "type": "object", + "properties": { + "event_cache_percentage": { + "type": "integer" + }, + "memory_quota_percentage": { + "type": "integer" + } + } + }, "v2.DispatchRule": { "type": "object", "properties": { @@ -2518,6 +2615,9 @@ } } }, + "v2.JSONDuration": { + "type": "object" + }, "v2.KafkaConfig": { "type": "object", "properties": { @@ -2828,6 +2928,9 @@ "case_sensitive": { "type": "boolean" }, + "changefeed_error_stuck_duration": { + "$ref": "#/definitions/v2.JSONDuration" + }, "check_gc_safe_point": { "type": "boolean" }, @@ -2861,11 +2964,17 @@ "sink": { "$ref": "#/definitions/v2.SinkConfig" }, + "sql_mode": { + "type": "string" + }, "sync_point_interval": { "type": "string" }, "sync_point_retention": { "type": "string" + }, + "synced_status": { + "$ref": "#/definitions/v2.SyncedStatusConfig" } } }, @@ -3012,6 +3121,42 @@ } } }, + "v2.SyncedStatus": { + "type": "object", + "properties": { + "info": { + "type": "string" + }, + "last_synced_ts": { + "type": "string" + }, + "now_ts": { + "type": "string" + }, + "puller_resolved_ts": { + "type": "string" + }, + "sink_checkpoint_ts": { + "type": "string" + }, + "synced": { + "type": "boolean" + } + } + }, + "v2.SyncedStatusConfig": { + "type": "object", + "properties": { + "checkpoint_interval": { + "description": "The maximum interval between latest checkpoint ts and now or\nbetween latest sink's checkpoint ts and puller's checkpoint ts required to reach synced state", + "type": "integer" + }, + "synced_check_interval": { + "description": "The minimum interval between the latest synced ts and now required to reach synced state", + "type": "integer" + } + } + }, "v2.Table": { "type": "object", "properties": { diff --git a/docs/swagger/swagger.yaml b/docs/swagger/swagger.yaml index ce21f335acb..d11127767ff 100644 --- a/docs/swagger/swagger.yaml +++ b/docs/swagger/swagger.yaml @@ -19,8 +19,14 @@ definitions: type: object config.CloudStorageConfig: properties: + file-cleanup-cron-spec: + type: string + file-expiration-days: + type: integer file-size: type: integer + flush-concurrency: + type: integer flush-interval: type: string output-column-id: @@ -291,9 +297,9 @@ definitions: Send and SendAsync returns an error after timeout. default: 30s type: integer - tls-certificate-path: + tls-certificate-file: type: string - tls-private-key-path: + tls-key-file-path: type: string tls-trust-certs-file-path: type: string @@ -313,7 +319,6 @@ definitions: cloud-storage-config: $ref: '#/definitions/config.CloudStorageConfig' column-selectors: - description: ColumnSelectors is Deprecated. items: $ref: '#/definitions/config.ColumnSelector' type: array @@ -693,8 +698,14 @@ definitions: type: object v2.CloudStorageConfig: properties: + file_cleanup_cron_spec: + type: string + file_expiration_days: + type: integer file_size: type: integer + flush_concurrency: + type: integer flush_interval: type: string output_column_id: @@ -728,17 +739,36 @@ definitions: type: object v2.ConsistentConfig: properties: + compression: + type: string + encoding_worker_num: + type: integer + flush_concurrency: + type: integer flush_interval: type: integer + flush_worker_num: + type: integer level: type: string max_log_size: type: integer + memory_usage: + $ref: '#/definitions/v2.ConsistentMemoryUsage' + meta_flush_interval: + type: integer storage: type: string use_file_backend: type: boolean type: object + v2.ConsistentMemoryUsage: + properties: + event_cache_percentage: + type: integer + memory_quota_percentage: + type: integer + type: object v2.DispatchRule: properties: columns: @@ -842,6 +872,8 @@ definitions: integrity_check_level: type: string type: object + v2.JSONDuration: + type: object v2.KafkaConfig: properties: auto_create_topic: @@ -1046,6 +1078,8 @@ definitions: type: boolean case_sensitive: type: boolean + changefeed_error_stuck_duration: + $ref: '#/definitions/v2.JSONDuration' check_gc_safe_point: type: boolean consistent: @@ -1068,10 +1102,14 @@ definitions: $ref: '#/definitions/v2.ChangefeedSchedulerConfig' sink: $ref: '#/definitions/v2.SinkConfig' + sql_mode: + type: string sync_point_interval: type: string sync_point_retention: type: string + synced_status: + $ref: '#/definitions/v2.SyncedStatusConfig' type: object v2.ResumeChangefeedConfig: properties: @@ -1167,6 +1205,33 @@ definitions: transaction_atomicity: type: string type: object + v2.SyncedStatus: + properties: + info: + type: string + last_synced_ts: + type: string + now_ts: + type: string + puller_resolved_ts: + type: string + sink_checkpoint_ts: + type: string + synced: + type: boolean + type: object + v2.SyncedStatusConfig: + properties: + checkpoint_interval: + description: |- + The maximum interval between latest checkpoint ts and now or + between latest sink's checkpoint ts and puller's checkpoint ts required to reach synced state + type: integer + synced_check_interval: + description: The minimum interval between the latest synced ts and now required + to reach synced state + type: integer + type: object v2.Table: properties: database_name: @@ -1894,6 +1959,40 @@ paths: tags: - changefeed - v2 + /api/v2/changefeeds/{changefeed_id}/synced: + get: + consumes: + - application/json + description: get the synced status of a changefeed + parameters: + - description: changefeed_id + in: path + name: changefeed_id + required: true + type: string + - description: default + in: query + name: namespace + type: string + produces: + - application/json + responses: + "200": + description: OK + schema: + $ref: '#/definitions/v2.SyncedStatus' + "400": + description: Bad Request + schema: + $ref: '#/definitions/model.HTTPError' + "500": + description: Internal Server Error + schema: + $ref: '#/definitions/model.HTTPError' + summary: Get synced status + tags: + - changefeed + - v2 /api/v2/health: get: description: Check the health status of a TiCDC cluster