Skip to content

Commit 498fd92

Browse files
rmoffrmoff
committed
WIP
1 parent 3430cb8 commit 498fd92

20 files changed

+2145
-148
lines changed

‎mqtt-tracker/copy_from_ccloud.sh‎

100644100755
Lines changed: 11 additions & 4 deletions
Original file line numberDiff line numberDiff line change
@@ -2,12 +2,19 @@
22

33
source .env
44

5+
while [ 1 -eq 1 ]
6+
do
57
kafkacat -b $CCLOUD_BROKER_HOST \
68
-X security.protocol=SASL_SSL -X sasl.mechanisms=PLAIN \
79
-X sasl.username="$CCLOUD_API_KEY" -X sasl.password="$CCLOUD_API_SECRET" \
810
-X ssl.ca.location=/usr/local/etc/openssl/cert.pem -X api.version.request=true \
9-
-X auto.offset.reset=earliest
10-
-G copy_to_local_00 data_mqtt -K: | \
11+
-X auto.offset.reset=earliest \
12+
-K: \
13+
-G asgard03_copy_to_local_05 data_mqtt | \
1114
kafkacat -b localhost:9092,localhost:19092,localhost:29092 \
12-
-t data_mqtt_ccloud_kc_00 \
13-
-K: -P
15+
-t data_mqtt-import \
16+
-K: -P -T
17+
18+
sleep 30
19+
20+
done

‎mqtt-tracker/create_es_sink.sh‎

Lines changed: 7 additions & 24 deletions
Original file line numberDiff line numberDiff line change
@@ -2,24 +2,6 @@
22

33
source .env
44

5-
curl -i -X PUT -H "Content-Type:application/json" \
6-
http://localhost:8083/connectors/sink-elastic-runner_status-00/config \
7-
-d '{
8-
"connector.class": "io.confluent.connect.elasticsearch.ElasticsearchSinkConnector",
9-
"connection.url": "'$ELASTIC_URL'",
10-
"connection.username": "'$ELASTIC_USERNAME'",
11-
"connection.password": "'$ELASTIC_PASSWORD'",
12-
"type.name": "type.name=kafkaconnect",
13-
"behavior.on.malformed.documents": "warn",
14-
"topics": "RUNNER_STATUS",
15-
"key.ignore": "false",
16-
"schema.ignore": "true",
17-
"key.converter": "org.apache.kafka.connect.storage.StringConverter",
18-
"transforms": "addTS",
19-
"transforms.addTS.type": "org.apache.kafka.connect.transforms.InsertField$Value",
20-
"transforms.addTS.timestamp.field": "EVENT_TS"
21-
}'
22-
235
curl -i -X PUT -H "Content-Type:application/json" \
246
http://localhost:8083/connectors/sink-elastic-runner_location-00/config \
257
-d '{
@@ -29,11 +11,12 @@ curl -i -X PUT -H "Content-Type:application/json" \
2911
"connection.password": "'$ELASTIC_PASSWORD'",
3012
"type.name": "type.name=kafkaconnect",
3113
"behavior.on.malformed.documents": "warn",
32-
"topics": "RUNNER_LOCATION",
14+
"errors.tolerance": "all",
15+
"errors.log.enable":true,
16+
"errors.log.include.messages":true,
17+
"topics.regex": "RUNNER_LOCATION_.*",
3318
"key.ignore": "true",
3419
"schema.ignore": "true",
35-
"key.converter": "org.apache.kafka.connect.storage.StringConverter",
36-
"transforms": "addTS",
37-
"transforms.addTS.type": "org.apache.kafka.connect.transforms.InsertField$Value",
38-
"transforms.addTS.timestamp.field": "EVENT_TS"
39-
}'
20+
"key.converter": "org.apache.kafka.connect.storage.StringConverter"
21+
}'
22+

‎mqtt-tracker/create_kibana_metadata.sh‎

Lines changed: 13 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -0,0 +1,13 @@
1+
#!/bin/bash
2+
3+
echo -e "\n--\n+> Create Kibana index patterns"
4+
curl -XPOST 'http://localhost:5601/api/saved_objects/index-pattern/runner_location' \
5+
-H 'kbn-xsrf: nevergonnagiveyouup' \
6+
-H 'Content-Type: application/json' \
7+
-d '{"attributes":{"title":"runner_location*","timeFieldName":"EVENT_TIME_EPOCH_MS_TS"}}'
8+
9+
echo -e "\n--\n+> Set default Kibana index"
10+
curl -XPOST 'http://localhost:5601/api/kibana/settings' \
11+
-H 'kbn-xsrf: nevergonnagiveyouup' \
12+
-H 'content-type: application/json' \
13+
-d '{"changes":{"defaultIndex":"runner_location"}}'

‎mqtt-tracker/create_replicator_source.sh‎

Lines changed: 4 additions & 2 deletions
Original file line numberDiff line numberDiff line change
@@ -14,7 +14,7 @@ done
1414
#
1515
epoch=$(date +%s)
1616
curl -s -X PUT -H "Accept:application/json" \
17-
-H "Content-Type:application/json" http://localhost:58083/connectors/replicator-source/config \
17+
-H "Content-Type:application/json" "http://localhost:58083/connectors/replicator-source"$epoch"/config" \
1818
-d '
1919
{
2020
"connector.class": "io.confluent.connect.replicator.ReplicatorSourceConnector",
@@ -28,9 +28,11 @@ curl -s -X PUT -H "Accept:application/json" \
2828
"src.consumer.group.id": "replicator-'$epoch'",
2929
"dest.kafka.bootstrap.servers": "kafka-1:39092,kafka-2:49092,kafka-3:59092",
3030
"topic.whitelist": "data_mqtt",
31-
"topic.rename.format":"${topic}-ccloud-'$epoch'",
31+
"topic.rename.format":"${topic}-import",
3232
"confluent.license":"",
3333
"confluent.topic.bootstrap.servers":"kafka-1:39092,kafka-2:49092,kafka-3:59092",
3434
"confluent.topic.replication.factor":1,
3535
"offset.start":"consumer"
3636
}' | jq '.'
37+
38+
# "topic.rename.format":"${topic}-ccloud-'$epoch'",

‎mqtt-tracker/data/ddl/users.sql‎

Lines changed: 26 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -0,0 +1,26 @@
1+
GRANT REPLICATION SLAVE, REPLICATION CLIENT ON *.* TO 'replicator' IDENTIFIED BY 'replpass';
2+
GRANT SELECT, RELOAD, SHOW DATABASES, REPLICATION SLAVE, REPLICATION CLIENT ON *.* TO 'debezium' IDENTIFIED BY 'dbz';
3+
4+
CREATE DATABASE demo;
5+
GRANT ALL PRIVILEGES ON demo.* TO 'mysqluser'@'%';
6+
7+
use demo;
8+
9+
CREATE TABLE USERS (USERID VARCHAR(255),
10+
EMAIL VARCHAR(255),
11+
SHARE_LOCATION_OPTIN BOOLEAN,
12+
PRIVACY_LOCATION_LAT DOUBLE,
13+
PRIVACY_LOCATION_LON DOUBLE,
14+
PRIVACY_ZONE_KM INT,
15+
CREATE_TS TIMESTAMP DEFAULT CURRENT_TIMESTAMP ,
16+
UPDATE_TS TIMESTAMP DEFAULT CURRENT_TIMESTAMP ON UPDATE CURRENT_TIMESTAMP
17+
);
18+
19+
INSERT INTO USERS (USERID, EMAIL, SHARE_LOCATION_OPTIN, PRIVACY_LOCATION_LAT, PRIVACY_LOCATION_LON, PRIVACY_ZONE_KM)
20+
VALUES ('rmoff','[email protected]', TRUE, 52.924651779196246,-1.7996157809808713, 1);
21+
22+
INSERT INTO USERS (USERID, EMAIL, SHARE_LOCATION_OPTIN)
23+
VALUES ('tom','[email protected]', FALSE);
24+
25+
INSERT INTO USERS (USERID, EMAIL, SHARE_LOCATION_OPTIN)
26+
VALUES ('isla','[email protected]', FALSE);

‎mqtt-tracker/data/dummy_data.kcat‎

Lines changed: 15 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -0,0 +1,15 @@
1+
owntracks/race-write/ivor:{"batt":100,"lon":-1.8125752571133549,"acc":65,"p":98.489105224609375,"bs":3,"vac":10,"lat":53.955233261289684,"t":"t","conn":"w","tst":1569334836,"alt":98,"_type":"location","tid":"C1"}
2+
owntracks/race-write/hugh:{"cog":193,"batt":45,"lon":-78.74988541880019,"acc":16,"p":100.14521789550781,"bs":1,"vel":0,"vac":3,"lat":35.66231724270073,"conn":"w","tst":1569330852,"tid":"RM","_type":"location","alt":104}
3+
owntracks/race-write/rick:{"batt":100,"lon":-1.8125821847113142,"acc":65,"bs":2,"p":98.428581237792969,"vac":10,"lat":53.955247763099669,"conn":"w","tst":1569334160,"alt":98,"_type":"location","tid":"A4"}
4+
owntracks/tiqmyral/rmoff:{"batt":97,"lon":-1.8125818890032339,"acc":200,"p":98.689468383789062,"bs":1,"vel":0,"vac":93,"lat":53.92535400390625,"t":"u","conn":"w","tst":1569316069,"alt":97,"_type":"location","tid":"FF"}
5+
owntracks/race-write/hugh:{"cog":24,"batt":96,"lon":-1.5314248613277124,"acc":10,"p":99.762825012207031,"bs":2,"vel":130,"vac":4,"lat":53.645720385215718,"conn":"m","tst":1569689674,"tid":"RM","_type":"location","alt":52}
6+
owntracks/race-write/ivor:{"cog":19,"batt":96,"lon":-1.5301563222094541,"acc":10,"p":99.829292297363281,"bs":2,"vel":134,"vac":6,"lat":53.647593543379635,"conn":"m","tst":1569689680,"tid":"RM","_type":"location","alt":51}
7+
owntracks/race-write/rick:{"cog":15,"batt":96,"lon":-1.5292111820733143,"acc":10,"p":99.908317565917969,"bs":2,"vel":129,"vac":4,"lat":53.64943998483686,"conn":"m","tst":1569689686,"tid":"RM","_type":"location","alt":42}
8+
owntracks/race-write/ivor:{"cog":10,"batt":96,"lon":-1.5284748555519652,"acc":10,"p":99.969696044921875,"bs":2,"vel":131,"vac":4,"lat":53.651334385427887,"conn":"m","tst":1569689692,"tid":"RM","_type":"location","alt":39}
9+
owntracks/race-write/hugh:{"cog":3,"batt":96,"lon":-1.5280530303414841,"acc":10,"p":100.00237274169922,"bs":2,"vel":125,"vac":6,"lat":53.653228086413485,"conn":"m","tst":1569689698,"tid":"RM","_type":"location","alt":37}
10+
owntracks/race-write/ivor:{"cog":349,"batt":96,"lon":-1.5283093337051832,"acc":5,"p":100.01512145996094,"bs":2,"vel":125,"vac":3,"lat":53.655085952807283,"conn":"m","tst":1569689704,"tid":"RM","_type":"location","alt":36}
11+
owntracks/race-write/ivor:{"cog":336,"batt":96,"lon":-1.5292491484563615,"acc":5,"p":100.01145172119141,"bs":2,"vel":125,"vac":3,"lat":53.656871188023594,"conn":"m","tst":1569689710,"tid":"RM","_type":"location","alt":39}
12+
owntracks/race-write/rmoff:{"cog":327,"batt":96,"lon":-1.5307541122276733,"acc":5,"p":100.00465393066406,"bs":2,"vel":120,"vac":3,"lat":53.658466731924534,"conn":"m","tst":1569689716,"tid":"RM","_type":"location","alt":42}
13+
owntracks/race-write/rmoff:{"cog":326,"batt":96,"lon":-1.5326987256024216,"acc":5,"p":99.988372802734375,"bs":2,"vel":119,"vac":4,"lat":53.660191805074049,"conn":"m","tst":1569689723,"tid":"RM","_type":"location","alt":40}
14+
owntracks/race-write/rmoff:{"cog":326,"batt":96,"lon":-1.5346830820408368,"acc":5,"p":99.980300903320312,"bs":2,"vel":119,"vac":6,"lat":53.661916155530776,"conn":"m","tst":1569689730,"tid":"RM","_type":"location","alt":36}
15+
owntracks/race-write/rmoff:{"cog":326,"batt":96,"lon":-1.536664103040333,"acc":5,"p":99.97674560546875,"bs":2,"vel":118,"vac":6,"lat":53.663641107896694,"conn":"m","tst":1569689737,"tid":"RM","_type":"location","alt":35}

0 commit comments

Comments
 (0)