Skip to content

Commit 1575123

Browse files
subkanthisubkanthi
authored
Merge pull request #71 from Altinity/develop
Release 0.2.0
2 parents 6e2051b + 62f89dc commit 1575123

File tree

75 files changed

+1492
-1776
lines changed

Some content is hidden

Large Commits have some content hidden by default. Use the searchbox below for content that may be hidden.

75 files changed

+1492
-1776
lines changed

‎.github/workflows/maven.yml

Lines changed: 2 additions & 2 deletions
Original file line numberDiff line numberDiff line change
@@ -5,9 +5,9 @@ name: Java CI with Maven
55

66
on:
77
push:
8-
branches: [ main ]
8+
branches: [ main, develop]
99
pull_request:
10-
branches: [ main ]
10+
branches: [ main , develop]
1111

1212
jobs:
1313
build:

‎README.md

Lines changed: 14 additions & 3 deletions
Original file line numberDiff line numberDiff line change
@@ -18,18 +18,29 @@ The connector is tested with the following converters
1818
- Kafka Offset management in ClickHouse
1919
- Increased Parallelism(Customize thread pool for JDBC connections)
2020

21-
22-
2321
### Grafana Dashboard
24-
![](doc/img/Grafana_dashboard.png) \
22+
![](doc/img/Grafana_dashboard.png)
23+
2524
# Source Databases
2625
- MySQL (Debezium)
2726
- PostgreSQL (Debezium) (Testing in progress)
2827

28+
### Quick Start
29+
Docker image for Sink connector `altinity/clickhouse-sink-connector:latest`
30+
31+
https://hub.docker.com/r/altinity/clickhouse-sink-connector
32+
33+
### MySQL:
34+
```bash
35+
cd deploy/docker
36+
./start-docker-compose.sh
37+
```
38+
For detailed instructions [Setup](doc/setup.md)
2939
## Documentation
3040
- [Data Types](doc/DataTypes.md)
3141
- [Architecture](doc/architecture.md)
3242
- [Local Setup - Docker Compose](doc/setup.md)
43+
- [Debezium Setup](doc/debezium_setup.md)
3344
- [Kubernetes Setup](doc/k8s_pipeline_setup.md)
3445
- [Sink Configuration](doc/sink_configuration.md)
3546
- [Testing](doc/TESTING.md)

‎deploy/debezium-connector-setup-schema-registry.sh

Lines changed: 5 additions & 2 deletions
Original file line numberDiff line numberDiff line change
@@ -47,9 +47,12 @@ if [[ $1 == "postgres" ]]; then
4747

4848
curl --request POST --url "${CONNECTORS_MANAGEMENT_URL}" --header 'Content-Type: application/json' --data @payload.json
4949
exit
50+
elif [[ $1 == "mysql_external" ]]; then
51+
echo "MySQL external database"
52+
HOST="host.docker.internal"
5053
else
5154
echo "MySQL Database"
52-
SNAPSHOT_MODE="initial"
55+
SNAPSHOT_MODE="schema_only_recovery"
5356
fi
5457

5558

@@ -117,7 +120,7 @@ else
117120
"connector.class": "${CONNECTOR_CLASS}",
118121
"tasks.max": "1",
119122
"snapshot.mode": "${SNAPSHOT_MODE}",
120-
"snapshot.locking.mode": "minimal",
123+
"snapshot.locking.mode": "none",
121124
"snapshot.delay.ms": 10000,
122125
"include.schema.changes":"true",
123126
"include.schema.comments": "true",

‎deploy/docker/docker-compose-airline-data.override.yaml

Lines changed: 33 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -0,0 +1,33 @@
1+
version: "2.3"
2+
3+
# Ubuntu , set this for redpanda to start
4+
# https://sort.veritas.com/public/documents/HSO/2.0/linux/productguides/html/hfo_admin_ubuntu/ch04s03.htm
5+
6+
# Clickhouse Table Schema
7+
# create table test(id int, message String) ENGINE=MergeTree() PRIMARY KEY id;
8+
9+
services:
10+
11+
mysql-master:
12+
container_name: mysql-master
13+
image: docker.io/bitnami/mysql:latest
14+
restart: "no"
15+
ports:
16+
- "3306:3306"
17+
environment:
18+
- MYSQL_ROOT_PASSWORD=root
19+
- MYSQL_DATABASE=test
20+
- MYSQL_REPLICATION_MODE=master
21+
- MYSQL_REPLICATION_USER=repl_user
22+
- ALLOW_EMPTY_PASSWORD=yes
23+
volumes:
24+
- ./mysqld.cnf:/opt/bitnami/mysql/conf/my_custom.cnf
25+
- ../sql/ontime_1.sql:/docker-entrypoint-initdb.d/ontime_1.sql
26+
- ../sql/ontime_2.sql:/docker-entrypoint-initdb.d/ontime_2.sql
27+
- ../sql/ontime_3.sql:/docker-entrypoint-initdb.d/ontime_3.sql
28+
- ../sql/ontime_4.sql:/docker-entrypoint-initdb.d/ontime_4.sql
29+
healthcheck:
30+
test: [ 'CMD', '/opt/bitnami/scripts/mysql/healthcheck.sh' ]
31+
interval: 15s
32+
timeout: 5s
33+
retries: 6

‎deploy/docker/docker-compose-apicurio-schema-registry.override.yaml

Lines changed: 63 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -0,0 +1,63 @@
1+
services:
2+
3+
4+
schemaregistry:
5+
container_name: schemaregistry
6+
image: apicurio/apicurio-registry-mem:latest-release
7+
#image: confluentinc/cp-schema-registry:latest
8+
restart: "no"
9+
ports:
10+
- "8080:8080"
11+
environment:
12+
- KAFKA_BOOTSTRAP_SERVERS=kafka:9092
13+
# environment:
14+
# - SCHEMA_REGISTRY_KAFKASTORE_BOOTSTRAP_SERVERS=kafka:9092
15+
# - SCHEMA_REGISTRY_HOST_NAME=localhost
16+
# - SCHEMA_REGISTRY_LISTENERS=http://localhost:8081
17+
# - SCHEMA_REGISTRY_DEBUG=true
18+
19+
depends_on:
20+
- kafka
21+
22+
debezium:
23+
container_name: debezium
24+
#image: debezium/connect:2.0
25+
build:
26+
context: ../../docker/debezium_jmx
27+
args:
28+
DEBEZIUM_VERSION: 1.9.5.Final
29+
JMX_AGENT_VERSION: 0.15.0
30+
restart: "no"
31+
ports:
32+
- "8083:8083"
33+
- "1976:1976"
34+
35+
environment:
36+
# Where to find Kafka broker
37+
- BOOTSTRAP_SERVERS=kafka:9092
38+
# Maps to rest.advertised.host.name
39+
#- ADVERTISED_HOST_NAME=kafka
40+
# Maps to rest.advertised.port
41+
#- ADVERTISED_PORT=9092
42+
- GROUP_ID=1
43+
- CONFIG_STORAGE_TOPIC=config-storage-topic-debezium
44+
- OFFSET_STORAGE_TOPIC=offset-storage-topic-debezium
45+
- STATUS_STORAGE_TOPIC=status-storage-topic-debezium
46+
- LOG_LEVEL=INFO
47+
- ENABLE_APICURIO_CONVERTERS=true
48+
- KEY_CONVERTER=io.apicurio.registry.utils.converter.AvroConverter
49+
- VALUE_CONVERTER=io.apicurio.registry.utils.converter.AvroConverter
50+
- CONNECT_KEY_CONVERTER=io.apicurio.registry.utils.converter.AvroConverter
51+
- CONNECT_KEY_CONVERTER_APICURIO.REGISTRY_URL=http://schemaregistry:8080/apis/registry/v2
52+
- CONNECT_KEY_CONVERTER_APICURIO_REGISTRY_AUTO-REGISTER=true
53+
- CONNECT_KEY_CONVERTER_APICURIO_REGISTRY_FIND-LATEST=true
54+
- CONNECT_VALUE_CONVERTER=io.apicurio.registry.utils.converter.AvroConverter
55+
- CONNECT_VALUE_CONVERTER_APICURIO_REGISTRY_URL=http://schemaregistry:8080/apis/registry/v2
56+
- CONNECT_VALUE_CONVERTER_APICURIO_REGISTRY_AUTO-REGISTER=true
57+
- CONNECT_VALUE_CONVERTER_APICURIO_REGISTRY_FIND-LATEST=true
58+
- KAFKA_OPTS=-javaagent:/kafka/etc/jmx_prometheus_javaagent.jar=8080:/kafka/etc/config.yml
59+
- JMXHOST=localhost
60+
- JMXPORT=1976
61+
#- LOG_LEVEL=DEBUG
62+
depends_on:
63+
- kafka

‎deploy/docker/docker-compose-graalvm.yaml renamed to ‎deploy/docker/docker-compose-graalvm.override.yaml

File renamed without changes.

‎deploy/docker/docker-compose-kafka.yml renamed to ‎deploy/docker/docker-compose-kafka.override.yml

File renamed without changes.

‎deploy/docker/docker-compose-apicurio-schema-registry.yaml renamed to ‎deploy/docker/docker-compose-mysql-server.yaml

Lines changed: 52 additions & 59 deletions
Original file line numberDiff line numberDiff line change
@@ -1,4 +1,4 @@
1-
version: "2.3"
1+
version: "3.4"
22

33
# Ubuntu , set this for redpanda to start
44
# https://sort.veritas.com/public/documents/HSO/2.0/linux/productguides/html/hfo_admin_ubuntu/ch04s03.htm
@@ -8,36 +8,36 @@ version: "2.3"
88

99
services:
1010

11-
mysql-master:
12-
# mysql --host=127.0.0.1 --port=3306 --user=root --password=root --database=test
13-
# SHOW VARIABLES LIKE 'server_id';
14-
# SHOW VARIABLES LIKE 'log_bin';
15-
# SHOW MASTER STATUS;
16-
container_name: mysql-master
17-
image: docker.io/bitnami/mysql:latest
18-
# command:
19-
# --default-authentication-plugin=mysql_native_password
20-
# --server-id=1
21-
# --binlog_do_db=test
22-
restart: "no"
23-
ports:
24-
- "3306:3306"
25-
environment:
26-
- MYSQL_ROOT_PASSWORD=root
27-
- MYSQL_DATABASE=test
28-
- MYSQL_REPLICATION_MODE=master
29-
- MYSQL_REPLICATION_USER=repl_user
30-
- ALLOW_EMPTY_PASSWORD=yes
31-
volumes:
32-
- ./mysqld.cnf:/opt/bitnami/mysql/conf/my_custom.cnf
33-
- ../sql/init_mysql.sql:/docker-entrypoint-initdb.d/init_mysql.sql
34-
healthcheck:
35-
test: [ 'CMD', '/opt/bitnami/scripts/mysql/healthcheck.sh' ]
36-
interval: 15s
37-
timeout: 5s
38-
retries: 6
39-
#security_opt:
40-
# - seccomp:unconfined
11+
# mysql-master:
12+
# # mysql --host=127.0.0.1 --port=3306 --user=root --password=root --database=test
13+
# # SHOW VARIABLES LIKE 'server_id';
14+
# # SHOW VARIABLES LIKE 'log_bin';
15+
# # SHOW MASTER STATUS;
16+
# container_name: mysql-master
17+
# image: docker.io/bitnami/mysql:latest
18+
## command:
19+
## --default-authentication-plugin=mysql_native_password
20+
## --server-id=1
21+
## --binlog_do_db=test
22+
# restart: "no"
23+
# ports:
24+
# - "3306:3306"
25+
# environment:
26+
# - MYSQL_ROOT_PASSWORD=root
27+
# - MYSQL_DATABASE=test
28+
# - MYSQL_REPLICATION_MODE=master
29+
# - MYSQL_REPLICATION_USER=repl_user
30+
# - ALLOW_EMPTY_PASSWORD=yes
31+
# volumes:
32+
# - ./mysqld.cnf:/opt/bitnami/mysql/conf/my_custom.cnf
33+
# - ../sql/init_mysql.sql:/docker-entrypoint-initdb.d/init_mysql.sql
34+
# healthcheck:
35+
# test: [ 'CMD', '/opt/bitnami/scripts/mysql/healthcheck.sh' ]
36+
# interval: 15s
37+
# timeout: 5s
38+
# retries: 6
39+
# security_opt:
40+
# - seccomp:unconfined
4141

4242
# mysql-slave:
4343
# # mysql --host=127.0.0.1 --port=3306 --user=root --password=root --database=test
@@ -79,34 +79,36 @@ services:
7979

8080
schemaregistry:
8181
container_name: schemaregistry
82-
image: apicurio/apicurio-registry-mem:latest-release
83-
#image: confluentinc/cp-schema-registry:latest
82+
#image: apicurio/apicurio-registry-mem:latest-release
83+
image: confluentinc/cp-schema-registry:latest
8484
restart: "no"
8585
ports:
86-
- "8080:8080"
86+
- "8081:8081"
87+
#environment:
88+
# - KAFKA_BOOTSTRAP_SERVERS=kafka:9092
8789
environment:
88-
- KAFKA_BOOTSTRAP_SERVERS=kafka:9092
89-
# environment:
90-
# - SCHEMA_REGISTRY_KAFKASTORE_BOOTSTRAP_SERVERS=kafka:9092
91-
# - SCHEMA_REGISTRY_HOST_NAME=localhost
92-
# - SCHEMA_REGISTRY_LISTENERS=http://localhost:8081
93-
# - SCHEMA_REGISTRY_DEBUG=true
90+
- SCHEMA_REGISTRY_KAFKASTORE_BOOTSTRAP_SERVERS=kafka:9092
91+
- SCHEMA_REGISTRY_HOST_NAME=schemaregistry
92+
- SCHEMA_REGISTRY_LISTENERS=http://0.0.0.0:8081
93+
- SCHEMA_REGISTRY_DEBUG=true
9494

9595
depends_on:
9696
- kafka
9797

9898
debezium:
9999
container_name: debezium
100-
#image: debezium/connect:2.0
100+
#image: debezium/connect:1.9.5.Final
101101
build:
102102
context: ../../docker/debezium_jmx
103103
args:
104-
DEBEZIUM_VERSION: 2.0
104+
DEBEZIUM_VERSION: 1.9.5.Final
105105
JMX_AGENT_VERSION: 0.15.0
106106
restart: "no"
107107
ports:
108108
- "8083:8083"
109109
- "1976:1976"
110+
extra_hosts:
111+
- "host.docker.internal:host-gateway"
110112

111113
environment:
112114
# Where to find Kafka broker
@@ -120,20 +122,11 @@ services:
120122
- OFFSET_STORAGE_TOPIC=offset-storage-topic-debezium
121123
- STATUS_STORAGE_TOPIC=status-storage-topic-debezium
122124
- LOG_LEVEL=INFO
123-
- ENABLE_APICURIO_CONVERTERS=true
124-
- KEY_CONVERTER=io.apicurio.registry.utils.converter.AvroConverter
125-
- VALUE_CONVERTER=io.apicurio.registry.utils.converter.AvroConverter
126-
- CONNECT_KEY_CONVERTER=io.apicurio.registry.utils.converter.AvroConverter
127-
- CONNECT_KEY_CONVERTER_APICURIO.REGISTRY_URL=http://schemaregistry:8080/apis/registry/v2
128-
- CONNECT_KEY_CONVERTER_APICURIO_REGISTRY_AUTO-REGISTER=true
129-
- CONNECT_KEY_CONVERTER_APICURIO_REGISTRY_FIND-LATEST=true
130-
- CONNECT_VALUE_CONVERTER=io.apicurio.registry.utils.converter.AvroConverter
131-
- CONNECT_VALUE_CONVERTER_APICURIO_REGISTRY_URL=http://schemaregistry:8080/apis/registry/v2
132-
- CONNECT_VALUE_CONVERTER_APICURIO_REGISTRY_AUTO-REGISTER=true
133-
- CONNECT_VALUE_CONVERTER_APICURIO_REGISTRY_FIND-LATEST=true
134-
- KAFKA_OPTS=-javaagent:/kafka/etc/jmx_prometheus_javaagent.jar=8080:/kafka/etc/config.yml
135-
- JMXHOST=localhost
136-
- JMXPORT=1976
125+
- KEY_CONVERTER=io.confluent.connect.avro.AvroConverter
126+
- VALUE_CONVERTER=io.confluent.connect.avro.AvroConverter
127+
# - KAFKA_OPTS=-javaagent:/kafka/etc/jmx_prometheus_javaagent.jar=8080:/kafka/etc/config.yml
128+
# - JMXHOST=localhost
129+
# - JMXPORT=1976
137130
#- LOG_LEVEL=DEBUG
138131
depends_on:
139132
- kafka
@@ -283,19 +276,19 @@ services:
283276
build:
284277
context: ../../docker/grafana
285278
args:
286-
GRAFANA_VERSION: 7.5.5
279+
GRAFANA_VERSION: latest
287280
#container_name: grafana
288281
#image: grafana/grafana
289282
restart: "no"
290283
#volumes:
291284
# - ../config/grafana/dashboards:/etc/grafana/provisioning/dashboards
292285
ports:
293286
- "3000:3000"
294-
links:
295-
- prometheus
296287
environment:
297288
- DS_PROMETHEUS=prometheus
298289
- GF_USERS_DEFAULT_THEME=light
290+
depends_on:
291+
- prometheus
299292
#### END OF MONITORING ###
300293

301294
bash:

‎deploy/docker/docker-compose-no-mysql.override.yaml

Lines changed: 10 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -0,0 +1,10 @@
1+
version: "3.4"
2+
3+
4+
# Useful to connect to a local MySQL instance
5+
services:
6+
mysql-master:
7+
container_name: mysql-master
8+
image: alpine
9+
ports:
10+
- "3307:3307"

0 commit comments

Comments
 (0)