Skip to content
This repository was archived by the owner on Jul 17, 2024. It is now read-only.

Commit 9077f86

Browse files
TheWall89TheWall89
committed
Put every Kafka mirror in a systemd unit for better management.
1 parent 85d94a3 commit 9077f86

12 files changed

+111
-18
lines changed

‎roles/dcm_service/files/create_dcm_service.sh‎

Lines changed: 14 additions & 7 deletions
Original file line numberDiff line numberDiff line change
@@ -1,14 +1,21 @@
11
#!/bin/bash
22

3-
echo [Unit] | sudo tee -a /etc/systemd/system/dcm.service > /dev/null
4-
echo Description=DCM REST Client | sudo tee -a /etc/systemd/system/dcm.service > /dev/null
3+
# TODO we should change this so it reflects latest changes
4+
5+
echo "[Unit]" | sudo tee -a /etc/systemd/system/dcm.service > /dev/null
6+
echo "Description=DCM REST Client" | sudo tee -a /etc/systemd/system/dcm.service > /dev/null
7+
echo "Requires=kafka-mirror-french-site.service kafka-mirror-greek-site.service kafka-mirror-italian-site.service kafka-mirror-spanish-site.service" | sudo tee -a /etc/systemd/system/dcm.service > /dev/null
8+
echo "After=kafka-mirror-french-site.service kafka-mirror-greek-site.service kafka-mirror-italian-site.service kafka-mirror-spanish-site.service" | sudo tee -a /etc/systemd/system/dcm.service > /dev/null
59
echo | sudo tee -a /etc/systemd/system/dcm.service > /dev/null
6-
echo [Service] | sudo tee -a /etc/systemd/system/dcm.service > /dev/null
7-
echo User=$1 | sudo tee -a /etc/systemd/system/dcm.service > /dev/null
8-
echo ExecStart=/usr/bin/sudo /usr/bin/python3 /usr/bin/dcm/dcm-python/dcm_rest_client.py --dcm_ip_address $2 --spanish_site_plugin_ip_port $3 --italian_site_plugin_ip_port $4 --french_site_plugin_ip_port $5 --greek_site_plugin_ip_port $6 --port 8090 --log info | sudo tee -a /etc/systemd/system/dcm.service > /dev/null
10+
echo "[Service]" | sudo tee -a /etc/systemd/system/dcm.service > /dev/null
11+
echo "User=$1" | sudo tee -a /etc/systemd/system/dcm.service > /dev/null
12+
echo "ExecStart=/usr/bin/sudo /usr/bin/python3 /usr/bin/dcm/dcm-python/dcm_rest_client.py --dcm_ip_address $2 --spanish_site_plugin_ip_port $3 --italian_site_plugin_ip_port $4 --french_site_plugin_ip_port $5 --greek_site_plugin_ip_port $6 --port 8090 --log info" | sudo tee -a /etc/systemd/system/dcm.service > /dev/null
13+
echo "TimeoutSec=10" | sudo tee -a /etc/systemd/system/dcm.service > /dev/null
14+
echo "Restart=on-failure" | sudo tee -a /etc/systemd/system/dcm.service > /dev/null
15+
echo "RestartSec=1" | sudo tee -a /etc/systemd/system/dcm.service > /dev/null
916
echo | sudo tee -a /etc/systemd/system/dcm.service > /dev/null
10-
echo [Install] | sudo tee -a /etc/systemd/system/dcm.service > /dev/null
11-
echo WantedBy=default.target | sudo tee -a /etc/systemd/system/dcm.service > /dev/null
17+
echo "[Install]" | sudo tee -a /etc/systemd/system/dcm.service > /dev/null
18+
echo "WantedBy=default.target" | sudo tee -a /etc/systemd/system/dcm.service > /dev/null
1219

1320
sudo chmod 664 /etc/systemd/system/dcm.service
1421
sudo systemctl daemon-reload

‎roles/preparation/files/create_kafka_mirror.sh‎

Lines changed: 8 additions & 6 deletions
Original file line numberDiff line numberDiff line change
@@ -1,9 +1,11 @@
1-
#!/bin/bash
2-
# Arguments:
3-
# 1-site
4-
# 2-topic name
1+
#!/usr/bin/env bash
52

63
SITE=$1
7-
TOPIC=$2
84

9-
/opt/kafka/bin/kafka-run-class.sh kafka.tools.MirrorMaker --consumer.config /usr/bin/dcm/${SITE}_consumer.config --num.streams 1 --producer.config /usr/bin/dcm/producer.config --whitelist \'${TOPIC}\' &
5+
/opt/kafka/bin/kafka-run-class.sh \
6+
kafka.tools.MirrorMaker \
7+
--abort.on.send.failure "true" \
8+
--consumer.config "/usr/bin/dcm/${SITE}_consumer.config" \
9+
--num.streams 4 \
10+
--producer.config /usr/bin/dcm/producer.config \
11+
--whitelist '^.*\.application_metric\..*$,^.*\.infrastructure_metric\..*$'

‎roles/preparation/files/french_consumer.config‎

Lines changed: 6 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -0,0 +1,6 @@
1+
bootstrap.servers=BROKER_IP:9092
2+
group.id=dp-MirrorMaker-group
3+
exclude.internal.topics=true
4+
#mirror.topics.whitelist=.*
5+
client.id=mirror_maker_consumer
6+
enable.auto.commit=false

‎roles/preparation/files/greek_consumer.config‎

Lines changed: 6 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -0,0 +1,6 @@
1+
bootstrap.servers=BROKER_IP:9092
2+
group.id=dp-MirrorMaker-group
3+
exclude.internal.topics=true
4+
#mirror.topics.whitelist=.*
5+
client.id=mirror_maker_consumer
6+
enable.auto.commit=false

‎roles/preparation/files/italian_consumer.config‎

Lines changed: 6 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -0,0 +1,6 @@
1+
bootstrap.servers=BROKER_IP:9092
2+
group.id=dp-MirrorMaker-group
3+
exclude.internal.topics=true
4+
#mirror.topics.whitelist=.*
5+
client.id=mirror_maker_consumer
6+
enable.auto.commit=false

‎roles/preparation/files/kafka-mirror-services/kafka-mirror-french-site.service‎

Lines changed: 12 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -0,0 +1,12 @@
1+
[Unit]
2+
Description=Kafka MirrorMaker for French Site
3+
PartOf=dcm.service
4+
5+
[Service]
6+
ExecStart=/usr/bin/dcm/create_kafka_mirror.sh french
7+
TimeoutSec=10
8+
Restart=on-failure
9+
RestartSec=1
10+
11+
[Install]
12+
WantedBy=default.target

‎roles/preparation/files/kafka-mirror-services/kafka-mirror-greek-site.service‎

Lines changed: 12 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -0,0 +1,12 @@
1+
[Unit]
2+
Description=Kafka MirrorMaker for Greek Site
3+
PartOf=dcm.service
4+
5+
[Service]
6+
ExecStart=/usr/bin/dcm/create_kafka_mirror.sh greek
7+
TimeoutSec=10
8+
Restart=on-failure
9+
RestartSec=1
10+
11+
[Install]
12+
WantedBy=default.target

‎roles/preparation/files/kafka-mirror-services/kafka-mirror-italian-site.service‎

Lines changed: 12 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -0,0 +1,12 @@
1+
[Unit]
2+
Description=Kafka MirrorMaker for Italian Site
3+
PartOf=dcm.service
4+
5+
[Service]
6+
ExecStart=/usr/bin/dcm/create_kafka_mirror.sh italian
7+
TimeoutSec=10
8+
Restart=on-failure
9+
RestartSec=1
10+
11+
[Install]
12+
WantedBy=default.target

‎roles/preparation/files/kafka-mirror-services/kafka-mirror-spanish-site.service‎

Lines changed: 12 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -0,0 +1,12 @@
1+
[Unit]
2+
Description=Kafka MirrorMaker for Spanish Site
3+
PartOf=dcm.service
4+
5+
[Service]
6+
ExecStart=/usr/bin/dcm/create_kafka_mirror.sh spanish
7+
TimeoutSec=10
8+
Restart=on-failure
9+
RestartSec=1
10+
11+
[Install]
12+
WantedBy=default.target

‎roles/preparation/files/producer.config‎

Lines changed: 4 additions & 1 deletion
Original file line numberDiff line numberDiff line change
@@ -1,4 +1,7 @@
11
bootstrap.servers=BROKER_IP:9092
2-
acks=1
2+
acks=-1
33
batch.size=100
44
client.id=mirror_maker_producer
5+
6+
max.in.flight.requests.per.connection=1
7+
retries=1000

0 commit comments

Comments
 (0)