-
Notifications
You must be signed in to change notification settings - Fork 11
/
.aliases
279 lines (224 loc) · 10.1 KB
/
.aliases
1
2
3
4
5
6
7
8
9
10
11
12
13
14
15
16
17
18
19
20
21
22
23
24
25
26
27
28
29
30
31
32
33
34
35
36
37
38
39
40
41
42
43
44
45
46
47
48
49
50
51
52
53
54
55
56
57
58
59
60
61
62
63
64
65
66
67
68
69
70
71
72
73
74
75
76
77
78
79
80
81
82
83
84
85
86
87
88
89
90
91
92
93
94
95
96
97
98
99
100
101
102
103
104
105
106
107
108
109
110
111
112
113
114
115
116
117
118
119
120
121
122
123
124
125
126
127
128
129
130
131
132
133
134
135
136
137
138
139
140
141
142
143
144
145
146
147
148
149
150
151
152
153
154
155
156
157
158
159
160
161
162
163
164
165
166
167
168
169
170
171
172
173
174
175
176
177
178
179
180
181
182
183
184
185
186
187
188
189
190
191
192
193
194
195
196
197
198
199
200
201
202
203
204
205
206
207
208
209
210
211
212
213
214
215
216
217
218
219
220
221
222
223
224
225
226
227
228
229
230
231
232
233
234
235
236
237
238
239
240
241
242
243
244
245
246
247
248
249
250
251
252
253
254
255
256
257
258
259
260
261
262
263
264
265
266
267
268
269
270
271
272
273
274
275
#!/bin/bash
#Shell Colour constants for use in 'echo -e'
# shellcheck disable=SC2034
{
RED='\033[1;31m'
GREEN='\033[1;32m'
YELLOW='\033[1;33m'
BLUE='\033[1;34m'
LGREY='\e[37m'
DGREY='\e[90m'
NC='\033[0m' # No Color
}
# docker-compose
kListTopics() {
docker exec -i -t kafka bash -c "unset JMX_PORT; /opt/kafka/bin/kafka-topics.sh --list --zookeeper zookeeper:2181/kafka"
}
kConsumeTopic() {
if [ -n "$1" ]; then
topics="$1"
else
selectedTopics="$( \
kListTopics \
| fzf --multi \
)"
# shellcheck disable=SC2039
topics="$(echo -e "${selectedTopics}" | paste -s -d,)"
fi
echo "Consuming from topic(s) [$topics] (comma delimited list of topics)"
docker exec -i -t kafka bash -c "unset JMX_PORT; /opt/kafka/bin/kafka-console-consumer.sh --bootstrap-server localhost:9092 --property print.key=true --property key.separator=\"|\" --whitelist $topics"
}
kDeleteTopic() {
if [ -n "$1" ]; then
topicName="$1"
echo "Deleting topic ${topicName}"
docker exec -i -t kafka bash -c "unset JMX_PORT; /opt/kafka/bin/kafka-topics.sh --delete --zookeeper zookeeper:2181/kafka --topic ${topicName}"
else
echo "ERROR: No topic name supplied"
fi
}
kCreateTopic() {
topicName="$1"
partitionCount="$2"
if [ "${partitionCount}x" = "x" ]; then
partitionCount=1
fi
echo "Creating topic [$topicName] with partition count [$partitionCount]"
docker exec -i -t kafka bash -c "unset JMX_PORT; /opt/kafka/bin/kafka-topics.sh --create --zookeeper zookeeper:2181/kafka --replication-factor 1 --partitions $partitionCount --topic $topicName"
}
execSQL() {
service="$1"
mysqlArgs="$2"
sql="$3"
docker exec -it "${service}" bash -c "echo \"${sql}\" | mysql ${mysqlArgs}"
}
# Kafka aliases
# We need to unset the JMX_PORT because these scripts share a script with the Kafka start script. The shared script uses the JMX_PORT, so no more than one of these scripts can be run from the same address
alias createStatisticsTopic='docker exec -i -t kafka bash -c "unset JMX_PORT; bin/kafka-topics.sh --create --zookeeper zookeeper:2181/kafka --replication-factor 1 --partitions 1 --topic statistics"'
alias kproduce='docker exec -i -t kafka bash -c "unset JMX_PORT; bin/kafka-console-producer.sh --broker-list localhost:9092 --topic statistics"'
alias kconsume='kConsumeTopic'
# consume from all topics
alias kconsumeAll='echo "Consuming from all topics..."; docker exec -i -t kafka bash -c "unset JMX_PORT; /opt/kafka/bin/kafka-console-consumer.sh --bootstrap-server localhost:9092 --blacklist noSuchTopic --from-beginning --zookeeper zookeeper:2181/kafka"'
alias kconsumeEvents='echo "Consuming from topics statisticEvents-Count & statisticEvents-Value ..."; docker exec -i -t kafka bash -c "unset JMX_PORT; /opt/kafka/bin/kafka-console-consumer.sh --bootstrap-server localhost:9092 --from-beginning --zookeeper zookeeper:2181/kafka --whitelist statisticEvents-Count,statisticEvents-Value"'
alias ktopics='kListTopics'
alias kcreate='kCreateTopic'
alias kdelete='kDeleteTopic'
alias kpurge='kPurgeTopic'
# bash shells in containers
# ~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~
alias kbash='docker exec -i -t kafka bash'
alias zkbash='docker exec -i -t zookeeper bash'
alias hbasebash='docker exec -i -t hbase bash'
alias stroombash='docker exec -it --user stroom stroom bash'
alias statsbash='docker exec -it --user stroomstats stroom-stats bash'
alias stroomdbbash='docker exec -it stroom-db bash'
alias nginxbash='docker exec -it nginx bash'
# A generic one that uses FZF to fuzzy find from a list of running containers (but only runs as root).
# shellcheck disable=SC1083
alias dbash='service=$(docker ps --format '{{.Names}}' | fzf); docker exec -it $service bash'
# shellcheck disable=SC1083
alias dsh='service=$(docker ps --format '{{.Names}}' | fzf); docker exec -it $service sh'
docker_logs() {
local service
service=$(docker ps --format '{{.Names}}' | fzf)
if [[ -n "${service}" ]]; then
docker logs "$@" "${service}"
fi
}
# shellcheck disable=SC1083
#alias dlog='service=$(docker ps --format '{{.Names}}' | fzf); docker logs "$@" $service'
alias dlog='docker_logs'
alias dlogf='docker_logs -f --since 0m'
# Zookeeper CLI
# ~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~
alias zkcli='docker exec -i -t zookeeper bin/zkCli.sh -server 127.0.0.1:2181'
getZkProp() {
local -r ZK_SERVER="127.0.0.1:2181/propertyService"
local -r ZK_SERVICE_NAME="zookeeper"
# for some reason zkCli outputs ^M chars at the end of each line so we need to remove them
# or they cause loads of problems later on
local -r prop=$( \
docker exec -t ${ZK_SERVICE_NAME} bin/zkCli.sh -server ${ZK_SERVER} ls / | \
sed 's///g' | \
tail -n1 | \
sed -E 's/\[(.*?)\]/\1/' | \
sed 's/, /\n/g' | \
fzf)
local curr_val
curr_val=$( \
docker exec -t ${ZK_SERVICE_NAME} bin/zkCli.sh -server ${ZK_SERVER} get "/${prop}" | \
sed 's///g' | \
grep -B 1 cZxid | \
grep -v cZxid)
#echo "[${curr_val}]"
echo -e "${BLUE}${prop}${NC}=[${GREEN}${curr_val}${NC}]"
echo
echo -e "${YELLOW}Enter the new property value:${NC}"
read new_val
#echo "[${new_val}]"
echo
echo -e "Changing ${BLUE}${prop}${NC} from [${GREEN}${curr_val}${NC}] to [${GREEN}${new_val}${NC}]"
docker exec -i -t ${ZK_SERVICE_NAME} bin/zkCli.sh -server ${ZK_SERVER} set "/${prop}" "${new_val}" > /dev/null
local curr_val
curr_val=$( \
docker exec -t ${ZK_SERVICE_NAME} bin/zkCli.sh -server ${ZK_SERVER} get "/${prop}" | \
sed 's///g' | \
grep -B 1 cZxid | \
grep -v cZxid)
echo
echo -e "${BLUE}${prop}${NC}=[${GREEN}${curr_val}${NC}]"
echo
echo "Done"
}
setZkProp() {
local -r ZK_SERVER="127.0.0.1:2181/propertyService"
local -r ZK_SERVICE_NAME="zookeeper"
if [[ $# -ne 2 ]]; then
echo "Invalid arguments" >&2
else
local -r prop=$1
local -r new_val=$2
docker exec -i -t ${ZK_SERVICE_NAME} bin/zkCli.sh -server ${ZK_SERVER} set "/${prop}" "${new_val}" > /dev/null
local curr_val
curr_val=$( \
docker exec -t ${ZK_SERVICE_NAME} bin/zkCli.sh -server ${ZK_SERVER} get "/${prop}" | \
sed 's///g' | \
grep -B 1 cZxid | \
grep -v cZxid)
echo
echo -e "${BLUE}${prop}${NC}=[${GREEN}${curr_val}${NC}]"
echo
echo "Done"
fi
}
alias zkgetprop='getZkProp'
alias zksetprop='setZkProp'
#
# HBase
# ~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~
alias hbasecli='docker exec -i -t hbase hbase/bin/hbase shell'
hbaseTableCount() {
tableName=$1
echo "Getting count of table $tableName"
cmd="count '${tableName}'"
docker exec -i -t hbase bash -c "echo \"${cmd}\" | hbase/bin/hbase shell"
}
alias hbasecount='hbaseTableCount'
# MySQL
# ~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~
# Use FZF to pick a DB to open a sql shell on
alias db='theDb=$(echo "annotations\nauth\nconfig\ndatameta\nexplorer\nprocess\nstats\nstroom" | fzf); docker exec -it stroom-all-dbs mysql -h"localhost" -P"3307" -u"${theDb}user" -p"stroompassword1" ${theDb}'
# Connect to the DB as root
alias rootdb='docker exec -it stroom-all-dbs mysql -h"localhost" -P"3307" -u"root" -p"my-secret-pw"'
# Connect to the DB as a user
alias stroomdb='docker exec -it stroom-all-dbs mysql -h"localhost" -P"3307" -u"stroomuser" -p"stroompassword1" stroom'
alias statsdb='docker exec -it stroom-all-dbs mysql -h"localhost" -P"3307" -u"statsuser" -p"stroompassword1" stats'
alias authdb='docker exec -it stroom-all-dbs mysql -h"localhost" -P"3307" -u"authuser" -p"stroompassword1" auth'
alias annotationsdb='docker exec -it stroom-all-dbs mysql -h"localhost" -P"3307" -u"annotationsuser" -p"stroompassword1" annotations'
alias configdb='docker exec -it stroom-all-dbs mysql -h"localhost" -P"3307" -u"configuser" -p"stroompassword1" config'
alias explorerdb='docker exec -it stroom-all-dbs mysql -h"localhost" -P"3307" -u"exploreruser" -p"stroompassword1" explorer'
alias datametadb='docker exec -it stroom-all-dbs mysql -h"localhost" -P"3307" -u"datametauser" -p"stroompassword1" datameta'
# Performa database dump of a named database
alias stroomdb_dump='docker exec -it stroom-all-dbs mysqldump --databases stroom -h"localhost" -P"3307" -u"root" -p"my-secret-pw" | grep -v "Using a password"'
alias stroomstatsdb_dump='docker exec -it stroom-all-dbs mysqldump --databases stats -h"localhost" -P"3307" -u"root" -p"my-secret-pw" | grep -v "Using a password"'
alias stroomauthdb_dump='docker exec -it stroom-all-dbs mysqldump --databases auth -h"localhost" -P"3307" -u"root" -p"my-secret-pw" | grep -v "Using a password"'
#TODO not working
alias stroomstatsdb_counts='execSQL stroom-all-dbs "-h\"localhost\" -P\"3307\" -u\"statsuser\" -p\"stroompassword1\" stats" "select \"key\", count(*) from SQL_STAT_KEY union select \"val\", count(*) from SQL_STAT_VAL union select \"src\", count(*) from SQL_STAT_VAL_SRC;" '
# Pipe the output of the dump commands to a file on the host
# To re-import the database dumps, use something like this
#cat ~/stroomDb.backup.02032018.sql | docker exec -i stroom-db mysql -u"root" -p"my-secret-pw"
# docker status
docker_summary() {
echo -e "\e[96mSummary\e[0m" \
&& docker system df \
&& echo
}
docker_images() {
echo -e "\e[96mImages\e[0m" \
&& docker images \
&& echo ""
}
docker_all_containers() {
echo -e "\e[96mAll containers\e[0m" \
&& docker ps -a -s \
&& echo ""
}
docker_running_containers() {
echo -e "\e[96mRunning containers\e[0m" \
&& docker ps \
&& echo ""
}
docker_volumes() {
echo -e "\e[96mVolumes\e[0m" \
&& docker volume ls \
&& echo ""
}
alias ds='docker_summary && docker_images && docker_volumes && docker_all_containers && docker_running_containers'
alias dps='docker ps -a --format "table {{.ID}}\t{{.Image}}\t{{.Names}}\t{{.Status}}"'
alias dpss='docker ps -a --format "table {{.Names}}\t{{.Status}}"'
# top for docker containers
alias ctop='docker run --rm -ti --name=ctop -v /var/run/docker.sock:/var/run/docker.sock quay.io/vektorlab/ctop:latest'