Skip to content

Commit 3430cb8

Browse files
rmoffrmoff
committed
tweaks.
1 parent 1ea69f5 commit 3430cb8

File tree

6 files changed

+59
-140
lines changed

6 files changed

+59
-140
lines changed

‎mqtt-tracker/copy_from_ccloud.sh‎

Lines changed: 13 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -0,0 +1,13 @@
1+
#!/bin/bash
2+
3+
source .env
4+
5+
kafkacat -b $CCLOUD_BROKER_HOST \
6+
-X security.protocol=SASL_SSL -X sasl.mechanisms=PLAIN \
7+
-X sasl.username="$CCLOUD_API_KEY" -X sasl.password="$CCLOUD_API_SECRET" \
8+
-X ssl.ca.location=/usr/local/etc/openssl/cert.pem -X api.version.request=true \
9+
-X auto.offset.reset=earliest
10+
-G copy_to_local_00 data_mqtt -K: | \
11+
kafkacat -b localhost:9092,localhost:19092,localhost:29092 \
12+
-t data_mqtt_ccloud_kc_00 \
13+
-K: -P

‎mqtt-tracker/create_mqtt_source_ccloud.sh‎

100644100755
File mode changed.

‎mqtt-tracker/create_mqtt_source_local.sh‎

100644100755
File mode changed.

‎mqtt-tracker/create_replicator_source.sh‎

Lines changed: 18 additions & 5 deletions
Original file line numberDiff line numberDiff line change
@@ -2,8 +2,19 @@
22

33
source .env
44

5-
curl -i -X PUT -H "Accept:application/json" \
6-
-H "Content-Type:application/json" http://localhost:58083/connectors/replicator-source2/config \
5+
echo "Waiting for Kafka Connect to start listening on localhost:58083 ⏳"
6+
while : ; do
7+
curl_status=$(curl -s -o /dev/null -w %{http_code} http://localhost:58083/connectors)
8+
echo -e $(date) " Kafka Connect listener HTTP state: " $curl_status " (waiting for 200)"
9+
if [ $curl_status -eq 200 ] ; then
10+
break
11+
fi
12+
sleep 5
13+
done
14+
#
15+
epoch=$(date +%s)
16+
curl -s -X PUT -H "Accept:application/json" \
17+
-H "Content-Type:application/json" http://localhost:58083/connectors/replicator-source/config \
718
-d '
819
{
920
"connector.class": "io.confluent.connect.replicator.ReplicatorSourceConnector",
@@ -14,10 +25,12 @@ curl -i -X PUT -H "Accept:application/json" \
1425
"src.kafka.security.protocol": "SASL_SSL",
1526
"src.kafka.sasl.mechanism": "PLAIN",
1627
"src.kafka.sasl.jaas.config": "org.apache.kafka.common.security.plain.PlainLoginModule required username=\"'$CCLOUD_API_KEY'\" password=\"'$CCLOUD_API_SECRET'\";",
28+
"src.consumer.group.id": "replicator-'$epoch'",
1729
"dest.kafka.bootstrap.servers": "kafka-1:39092,kafka-2:49092,kafka-3:59092",
1830
"topic.whitelist": "data_mqtt",
19-
"topic.rename.format":"${topic}-ccloud2",
31+
"topic.rename.format":"${topic}-ccloud-'$epoch'",
2032
"confluent.license":"",
2133
"confluent.topic.bootstrap.servers":"kafka-1:39092,kafka-2:49092,kafka-3:59092",
22-
"confluent.topic.replication.factor":1
23-
}'
34+
"confluent.topic.replication.factor":1,
35+
"offset.start":"consumer"
36+
}' | jq '.'

‎mqtt-tracker/docker-compose.yml‎

Lines changed: 18 additions & 15 deletions
Original file line numberDiff line numberDiff line change
@@ -282,18 +282,21 @@ services:
282282
283283
sleep infinity
284284
285-
# kafkacat:
286-
# image: edenhill/kafkacat:1.5.0
287-
# container_name: kafkacat
288-
# # depends_on:
289-
# # - kafka
290-
# command:
291-
# - bash
292-
# - -c
293-
# - |
294-
# echo '{"batt":100,"lon":-1.8029050435887677,"acc":30,"p":98.641334533691406,"bs":1,"vel":0,"vac":12,"lat":53.925261804884229,"t":"t","conn":"w","tst":1569319041,"alt":97,"_type":"location","tid":"FF"}' | \
295-
# kafkacat -b ${CCLOUD_BROKER_HOST}
296-
# -X security.protocol=SASL_SSL -X sasl.mechanisms=PLAIN \
297-
# -X sasl.username="${CCLOUD_API_KEY}" -X sasl.password="${CCLOUD_API_SECRET}" \
298-
# -X ssl.ca.location=/usr/local/etc/openssl/cert.pem -X api.version.request=true
299-
# -P -t data_mqtt
285+
kafkacat:
286+
image: edenhill/kafkacat:1.5.0
287+
container_name: kafkacat
288+
volumes:
289+
- ./data:/data
290+
depends_on:
291+
- kafka-1
292+
- kafka-2
293+
- kafka-3
294+
command:
295+
- -b
296+
- kafka-1:39092,kafka-2:49092,kafka-3:59092
297+
- -t
298+
- "data_mqtt_sample"
299+
- "-K:"
300+
- -P
301+
- -l
302+
- "/data/mqtt_data.kcat"

‎mqtt-tracker/docker-compose_ccloud.yml‎

Lines changed: 10 additions & 120 deletions
Original file line numberDiff line numberDiff line change
@@ -75,128 +75,18 @@ services:
7575
#
7676
sleep infinity
7777
78-
# elasticsearch:
79-
# image: docker.elastic.co/elasticsearch/elasticsearch:6.7.0
80-
# container_name: elasticsearch
81-
# ports:
82-
# - 9200:9200
83-
# environment:
84-
# xpack.security.enabled: "false"
85-
# ES_JAVA_OPTS: "-Xms1g -Xmx1g"
86-
# command:
87-
# - bash
88-
# - -c
89-
# - |
90-
# /usr/local/bin/docker-entrypoint.sh &
91-
# echo "Waiting for Elasticsearch to start ⏳"
92-
# while : ; do
93-
# curl_status=$$(curl -s -o /dev/null -w %{http_code} http://localhost:9200/)
94-
# echo -e $$(date) " Elasticsearch listener HTTP state: " $$curl_status " (waiting for 200)"
95-
# if [ $$curl_status -eq 200 ] ; then
96-
# break
97-
# fi
98-
# sleep 5
99-
# done
100-
# echo -e "\n--\n+> Creating Elasticsearch dynamic mapping"
101-
# curl -XPUT "http://localhost:9200/_template/kafkaconnect/" -H 'Content-Type: application/json' -d'
102-
# {
103-
# "index_patterns": "*",
104-
# "settings": {
105-
# "number_of_shards": 1,
106-
# "number_of_replicas": 0
107-
# },
108-
# "mappings": {
109-
# "dynamic_templates": [
110-
# {
111-
# "dates": {
112-
# "match": "*_TS",
113-
# "mapping": {
114-
# "type": "date"
115-
# }
116-
# }
117-
# },
118-
# {
119-
# "heights": {
120-
# "match": "HEIGHT",
121-
# "mapping": {
122-
# "type": "float"
123-
# }
124-
# }
125-
# },
126-
# {
127-
# "locations": {
128-
# "match": "LOCATION",
129-
# "mapping": {
130-
# "type": "geo_point"
131-
# }
132-
# }
133-
# }
134-
# ]
135-
# }
136-
# }'
137-
# sleep infinity
138-
139-
14078
141-
# kibana:
142-
# image: docker.elastic.co/kibana/kibana:6.7.0
143-
# container_name: kibana
144-
# depends_on:
145-
# - elasticsearch
146-
# ports:
147-
# - 5601:5601
148-
# environment:
149-
# xpack.security.enabled: "false"
150-
# discovery.type: "single-node"
79+
# kafkacat:
80+
# image: edenhill/kafkacat:1.5.0
81+
# container_name: kafkacat
82+
#
15183
# command:
15284
# - bash
15385
# - -c
15486
# - |
155-
# /usr/local/bin/kibana-docker &
156-
# #
157-
# echo "Waiting for Kibana to start ⏳"
158-
# while : ; do
159-
# curl_status=$$(curl -s -o /dev/null -w %{http_code} http://localhost:5601/api/kibana/settings)
160-
# echo -e $$(date) " Kibana listener HTTP state: " $$curl_status " (waiting for 200)"
161-
# if [ $$curl_status -eq 200 ] ; then
162-
# break
163-
# fi
164-
# sleep 5
165-
# done
166-
# #
167-
# echo "Waiting for Kibana API to be available ⏳"
168-
# while : ; do
169-
# kibana_status=$$(curl -s 'http://localhost:5601/api/kibana/settings')
170-
# echo -e $$(date) " Kibana API response: " $$kibana_status
171-
# if [ $$kibana_status != "Kibana server is not ready yet" ] ; then
172-
# break
173-
# fi
174-
# sleep 5
175-
# done
176-
# #
177-
# sleep 60
178-
# echo -e "\n--\n+> Setup Kibana objects"
179-
180-
# echo -e "\n--\n+> Opt out of Kibana telemetry"
181-
# curl 'http://localhost:5601/api/telemetry/v1/optIn' -H 'kbn-xsrf: nevergonnagiveyouup' -H 'content-type: application/json' -H 'accept: application/json' --data-binary '{"enabled":false}' --compressed
182-
183-
# echo -e "\n--\n+> Create Kibana index patterns"
184-
# curl -XPOST 'http://localhost:5601/api/saved_objects/index-pattern/runner_location_idx' \
185-
# -H 'kbn-xsrf: nevergonnagiveyouup' \
186-
# -H 'Content-Type: application/json' \
187-
# -d '{"attributes":{"title":"runner_location","timeFieldName":"EVENT_TS"}}'
188-
189-
# curl -XPOST 'http://localhost:5601/api/saved_objects/index-pattern/runner_status_idx' \
190-
# -H 'kbn-xsrf: nevergonnagiveyouup' \
191-
# -H 'Content-Type: application/json' \
192-
# -d '{"attributes":{"title":"runner_status","timeFieldName":"EVENT_TS"}}'
193-
194-
# echo -e "\n--\n+> Set default Kibana index"
195-
# curl -XPOST 'http://localhost:5601/api/kibana/settings' \
196-
# -H 'kbn-xsrf: nevergonnagiveyouup' \
197-
# -H 'content-type: application/json' \
198-
# -d '{"changes":{"defaultIndex":"runner_status_idx"}}'
199-
200-
# echo -e "\n--\n+> Import Kibana objects"
201-
202-
# sleep infinity
87+
# f
88+
# kafkacat -b ${CCLOUD_BROKER_HOST}
89+
# -X security.protocol=SASL_SSL -X sasl.mechanisms=PLAIN \
90+
# -X sasl.username="${CCLOUD_API_KEY}" -X sasl.password="${CCLOUD_API_SECRET}" \
91+
# -X ssl.ca.location=/usr/local/etc/openssl/cert.pem -X api.version.request=true
92+
# -P -t data_mqtt

0 commit comments

Comments
 (0)