Skip to content

Commit b4f2ae7

Browse files
rerorerorerorero
committed
add e2e test
1 parent 312f8b2 commit b4f2ae7

File tree

5 files changed

+143
-0
lines changed

5 files changed

+143
-0
lines changed

‎.github/workflows/ci.yaml

Lines changed: 5 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -33,6 +33,11 @@ jobs:
3333
- name: assemble
3434
run: ./gradlew shadowJar
3535

36+
- name: e2e test with docker
37+
shell: bash
38+
working-directory: ./e2e
39+
run: ./test.sh
40+
3641
- name: Cleanup Gradle Cache
3742
# Remove some files from the Gradle cache, so they aren't cached by GitHub Actions.
3843
run: |

‎e2e/Dockerfile

Lines changed: 7 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -0,0 +1,7 @@
1+
FROM confluentinc/cp-kafka-connect:6.1.1
2+
3+
ENV CONNECT_PLUGIN_PATH="/usr/share/java,/usr/share/confluent-hub-components"
4+
5+
COPY --chown=appuser:appuser build/libs/kafka-smt-payload-basis-router.jar /usr/share/java
6+
7+
RUN confluent-hub install --no-prompt confluentinc/kafka-connect-datagen:latest

‎e2e/config.json

Lines changed: 20 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -0,0 +1,20 @@
1+
{
2+
"name": "datagen-pageviews",
3+
"config": {
4+
"connector.class": "io.confluent.kafka.connect.datagen.DatagenConnector",
5+
"kafka.topic": "pageviews-vault",
6+
"quickstart": "pageviews",
7+
"key.converter": "io.confluent.connect.avro.AvroConverter",
8+
"value.converter": "io.confluent.connect.avro.AvroConverter",
9+
"key.converter.schema.registry.url": "http://schema-registry:8081",
10+
"value.converter.schema.registry.url": "http://schema-registry:8081",
11+
"max.interval": 300,
12+
"iterations": 20,
13+
"tasks.max": "1",
14+
15+
"transforms": "route",
16+
17+
"transforms.route.type": "com.github.rerorero.kafka.smt.PayloadBasisRouter$Value",
18+
"transforms.route.replacement": "topic-{$.userid}"
19+
}
20+
}

‎e2e/docker-compose.yaml

Lines changed: 96 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -0,0 +1,96 @@
1+
---
2+
version: '3'
3+
4+
services:
5+
zookeeper:
6+
image: confluentinc/cp-zookeeper:6.1.1
7+
hostname: zookeeper
8+
container_name: zookeeper
9+
ports:
10+
- '32181:32181'
11+
environment:
12+
ZOOKEEPER_CLIENT_PORT: 32181
13+
ZOOKEEPER_TICK_TIME: 2000
14+
extra_hosts:
15+
- "moby:127.0.0.1"
16+
17+
kafka:
18+
image: confluentinc/cp-server:6.1.1
19+
hostname: kafka
20+
container_name: kafka
21+
ports:
22+
- '9092:9092'
23+
- '29092:29092'
24+
depends_on:
25+
- zookeeper
26+
environment:
27+
KAFKA_BROKER_ID: 1
28+
KAFKA_ZOOKEEPER_CONNECT: zookeeper:32181
29+
KAFKA_LISTENER_SECURITY_PROTOCOL_MAP: PLAINTEXT:PLAINTEXT,PLAINTEXT_HOST:PLAINTEXT
30+
KAFKA_INTER_BROKER_LISTENER_NAME: PLAINTEXT
31+
KAFKA_ADVERTISED_LISTENERS: PLAINTEXT://kafka:29092,PLAINTEXT_HOST://localhost:9092
32+
KAFKA_AUTO_CREATE_TOPICS_ENABLE: "true"
33+
KAFKA_OFFSETS_TOPIC_REPLICATION_FACTOR: 1
34+
KAFKA_METRIC_REPORTERS: io.confluent.metrics.reporter.ConfluentMetricsReporter
35+
KAFKA_OFFSETS_TOPIC_REPLICATION_FACTOR: 1
36+
KAFKA_GROUP_INITIAL_REBALANCE_DELAY_MS: 0
37+
KAFKA_CONFLUENT_LICENSE_TOPIC_REPLICATION_FACTOR: 1
38+
KAFKA_CONFLUENT_BALANCER_TOPIC_REPLICATION_FACTOR: 1
39+
KAFKA_TRANSACTION_STATE_LOG_MIN_ISR: 1
40+
KAFKA_TRANSACTION_STATE_LOG_REPLICATION_FACTOR: 1
41+
CONFLUENT_METRICS_REPORTER_BOOTSTRAP_SERVERS: kafka:29092
42+
CONFLUENT_METRICS_REPORTER_ZOOKEEPER_CONNECT: zookeeper:32181
43+
CONFLUENT_METRICS_REPORTER_TOPIC_REPLICAS: 1
44+
CONFLUENT_METRICS_ENABLE: 'true'
45+
CONFLUENT_SUPPORT_CUSTOMER_ID: 'anonymous'
46+
extra_hosts:
47+
- "moby:127.0.0.1"
48+
49+
schema-registry:
50+
image: confluentinc/cp-schema-registry:6.1.1
51+
hostname: schema-registry
52+
container_name: schema-registry
53+
depends_on:
54+
- zookeeper
55+
- kafka
56+
ports:
57+
- '8081:8081'
58+
environment:
59+
SCHEMA_REGISTRY_HOST_NAME: schema-registry
60+
SCHEMA_REGISTRY_KAFKASTORE_CONNECTION_URL: zookeeper:32181
61+
extra_hosts:
62+
- "moby:127.0.0.1"
63+
64+
connect:
65+
image: datagen-with-router:0.1.0
66+
build:
67+
context: ..
68+
dockerfile: e2e/Dockerfile
69+
container_name: connect
70+
restart: always
71+
ports:
72+
- "8083:8083"
73+
depends_on:
74+
- zookeeper
75+
- kafka
76+
- schema-registry
77+
environment:
78+
CONNECT_BOOTSTRAP_SERVERS: kafka:29092
79+
CONNECT_REST_PORT: 8083
80+
CONNECT_REST_ADVERTISED_HOST_NAME: connect
81+
CONNECT_GROUP_ID: compose-connect-group
82+
CONNECT_CONFIG_STORAGE_TOPIC: docker-connect-configs
83+
CONNECT_CONFIG_STORAGE_REPLICATION_FACTOR: 1
84+
CONNECT_OFFSET_FLUSH_INTERVAL_MS: 10000
85+
CONNECT_OFFSET_STORAGE_TOPIC: docker-connect-offsets
86+
CONNECT_OFFSET_STORAGE_REPLICATION_FACTOR: 1
87+
CONNECT_STATUS_STORAGE_TOPIC: docker-connect-status
88+
CONNECT_STATUS_STORAGE_REPLICATION_FACTOR: 1
89+
CONNECT_KEY_CONVERTER: org.apache.kafka.connect.storage.StringConverter
90+
CONNECT_VALUE_CONVERTER: io.confluent.connect.avro.AvroConverter
91+
CONNECT_VALUE_CONVERTER_SCHEMA_REGISTRY_URL: 'http://schema-registry:8081'
92+
CONNECT_INTERNAL_KEY_CONVERTER: org.apache.kafka.connect.json.JsonConverter
93+
CONNECT_INTERNAL_VALUE_CONVERTER: org.apache.kafka.connect.json.JsonConverter
94+
CONNECT_ZOOKEEPER_CONNECT: 'zookeeper:32181'
95+
CONNECT_PLUGIN_PATH: "/usr/share/java,/usr/share/confluent-hub-components"
96+
CONNECT_LOG4J_LOGGERS: "org.apache.kafka.connect.runtime.rest=WARN,org.reflections=ERROR"

‎e2e/test.sh

Lines changed: 15 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -0,0 +1,15 @@
1+
#!/bin/bash
2+
3+
set -eux
4+
5+
# start services
6+
docker-compose up -d --build connect
7+
trap 'docker-compose down' EXIT
8+
9+
sleep 60
10+
# start connect
11+
curl -X POST -H "Content-Type: application/json" --data @config.json http://localhost:8083/connectors
12+
sleep 20
13+
14+
# Count the number of topics that has a prefix 'topic-User_'
15+
[ "$(docker-compose exec -T kafka kafka-topics --list --bootstrap-server kafka:29092 | grep -c topic-User_)" -gt "5" ]

0 commit comments

Comments
 (0)