Skip to content

Commit 2aeaa14

Browse files
BorisYourichBorisYourich
committed
Finished Galaxy Monitoring with Telegraf and Grafana
1 parent ee0bc68 commit 2aeaa14

File tree

126 files changed

+6985
-1
lines changed

Some content is hidden

Large Commits have some content hidden by default. Use the searchbox below for content that may be hidden.

126 files changed

+6985
-1
lines changed

‎README.md

Lines changed: 1 addition & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -89,3 +89,4 @@ On Debian 11 the default version of `ansible` is 2.10.08 which fails to extract
8989

9090
Use `sudo apt install python3-docker` before running the `usegalaxy.rabbitmqserver` role.
9191

92+
For influxdb apt key error: https://www.influxdata.com/blog/linux-package-signing-key-rotation/

‎galaxy.yml

Lines changed: 1 addition & 1 deletion
Original file line numberDiff line numberDiff line change
@@ -47,7 +47,7 @@
4747
- usegalaxy_eu.rabbitmqserver
4848
- galaxyproject.gxadmin
4949
- galaxyproject.cvmfs
50-
- galaxyproject.gxadmin
50+
- dj-wasabi.telegraf
5151
post_tasks:
5252
- name: Setup gxadmin cleanup task
5353
ansible.builtin.cron:

‎group_vars/all.yml

Lines changed: 26 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -10,3 +10,29 @@ galaxy_db_name: galaxy
1010
cvmfs_role: client
1111
galaxy_cvmfs_repos_enabled: config-repo
1212
cvmfs_quota_limit: 500
13+
14+
# Telegraf
15+
telegraf_agent_package_state: latest
16+
17+
# Configure the output to point to an InfluxDB
18+
# running on localhost, and # place data in the
19+
# database "telegraf" which will be created if need be.
20+
telegraf_agent_output:
21+
- type: influxdb
22+
config:
23+
- urls = ["http://127.0.0.1:8086"]
24+
- database = "telegraf"
25+
26+
# The default plugins, applied to any telegraf-configured host
27+
telegraf_plugins_default:
28+
- plugin: cpu
29+
- plugin: disk
30+
- plugin: kernel
31+
- plugin: processes
32+
- plugin: io
33+
- plugin: mem
34+
- plugin: system
35+
- plugin: swap
36+
- plugin: net
37+
- plugin: netstat
38+

‎group_vars/dbservers.yml

Lines changed: 6 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -2,9 +2,15 @@
22
# PostgreSQL
33
postgresql_objects_users:
44
- name: "{{ galaxy_user_name }}"
5+
- name: telegraf
56
postgresql_objects_databases:
67
- name: "{{ galaxy_db_name }}"
78
owner: "{{ galaxy_user_name }}"
9+
postgresql_objects_privileges:
10+
- database: galaxy
11+
roles: telegraf
12+
privs: SELECT
13+
objs: ALL_IN_SCHEMA
814

915
# PostgreSQL Backups
1016
postgresql_backup_dir: /data/backups

‎group_vars/galaxyservers.yml

Lines changed: 19 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -169,6 +169,9 @@ galaxy_config:
169169
celery_conf:
170170
result_backend: "redis://localhost:6379/0"
171171
enable_celery_tasks: true
172+
# Monitoring
173+
statsd_host: localhost
174+
statsd_influxdb: true
172175
gravity:
173176
process_manager: systemd
174177
galaxy_root: "{{ galaxy_root }}/server"
@@ -373,3 +376,19 @@ flower_ui_users:
373376

374377
flower_environment_variables:
375378
GALAXY_CONFIG_FILE: "{{ galaxy_config_file }}"
379+
380+
# Telegraf
381+
telegraf_plugins_extra:
382+
listen_galaxy_routes:
383+
plugin: "statsd"
384+
config:
385+
- service_address = ":8125"
386+
- metric_separator = "."
387+
- allowed_pending_messages = 10000
388+
monitor_galaxy_queue:
389+
plugin: "exec"
390+
config:
391+
- commands = ["/usr/bin/env PGDATABASE=galaxy /usr/local/bin/gxadmin iquery queue-overview --short-tool-id"]
392+
- timeout = "10s"
393+
- data_format = "influx"
394+
- interval = "15s"

‎group_vars/monitoring.yml

Lines changed: 17 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -0,0 +1,17 @@
1+
grafana_url: "https://{{ inventory_hostname }}/grafana/"
2+
3+
grafana_security:
4+
# Please change at least the password to something more suitable
5+
admin_user: admin
6+
admin_password: password
7+
8+
# These datasources will be automatically included into Grafana
9+
grafana_datasources:
10+
- name: Galaxy
11+
type: influxdb
12+
access: proxy
13+
url: http://127.0.0.1:8086
14+
isDefault: true
15+
version: 1
16+
editable: false
17+
database: telegraf

‎hosts

Lines changed: 2 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -2,3 +2,5 @@
22
usegalaxy-test.cerit-sc.cz ansible_connection=local ansible_user=debian
33
[dbservers:children]
44
galaxyservers
5+
[monitoring]
6+
usegalaxy-test.cerit-sc.cz ansible_connection=local ansible_user=debian

‎influxdata-archive_compat.key

Lines changed: 29 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -0,0 +1,29 @@
1+
-----BEGIN PGP PUBLIC KEY BLOCK-----
2+
3+
mQINBGPIEycBEACpG4qSjhxA6fh4QJVJxFVBvCFt9tVx/hDbKH0Ryy9iilyMeReC
4+
AS1/CZnSv/fhDNKmVPckf6on72z/ODwZcVfMV6DHkxmZ6x/tQrS6CWfKkupsON2H
5+
KS3t4HUivahwHPlWtbfDqsWNwTAsZqklKpJQWY2ADPwurkbCmtYSjsgbLuWe23Pd
6+
nJpLTHtlChM0ntW/l7Le1zYjGPUGoxMJgjg1YG8fi2l/zS0Of8bdQ26ps+WRvrSQ
7+
RKhfAkfIgUiCXxBpDlN1spN73ZlAkaSb+myTfEKyJR55Yt9pHfkDdJh26RVgE1+N
8+
GuLmm6oidaD9lTlNJ9P8wlLzoof3xJXYprgLLz/HmgtawnJ+DxFIXoXNNpUmhORJ
9+
6Hb2Z5IKIyGIwXhQVe2Lw7B8awBNV99zUw517Wuax3RYx7Hwhntz9gFxS4GRxaCo
10+
uLCFQ0AgDCkMHyEHufQo1XdjIB7fz6U551y5GMQw6/rjMnUM9ZI68SQ/FWou2cQf
11+
533PyayvWOYQM4pP7ZmbzyCd393XlMaPWA5dyUOqv7Vcmv0IsAbncX6/KJmZAhKG
12+
qu19xb6rv3ab2RbcU422guK3C/h/URPZJbSjf2w4jUV5UDe2veZg6BEVn7Sk5bW0
13+
ceX8n0GVbPNG7CvRduJPjXNzsz3FzmUS8QFFde3H5gl1T0f6GcfhmKgKEQARAQAB
14+
tDdJbmZsdXhEYXRhIFBhY2thZ2UgU2lnbmluZyBLZXkgPHN1cHBvcnRAaW5mbHV4
15+
ZGF0YS5jb20+iQJVBBMBCAA/BgsJCAcDAgYVCAIJCgsEFgIDAQIeAQIXgAUJBaOk
16+
/BYhBJ1TnZDTMo3H1sjTudj/jh99+LB+BQJjyB9PAhsDAAoJENj/jh99+LB+klgQ
17+
AKOKdwTyKOr6+mnRrACz5U3EFxfAXXFGan9Ka7Nzgz4K+FOnTtT1gWwqrPPmTKQk
18+
epNUMcelfX1kCA08yCm0nyw2niqxES40W33ergKUj6jlDx7UQYXWsDQGD9IKksa8
19+
MWfZlJ3zlrsGKXA4oa+kfY+vltWDVP8WhLcQzm2LywbKvr3WgY80GZbnRjoekiBK
20+
oMKztQVMJG5yNZBo9B4JrqB3wMpnXZxEtqZcBPsJJdXTFKHsQ7kB9TMNorbUvDNH
21+
ohwsprgMw84vHikEk9jyCypXpYq/E/wvkM0CeIUJ36S2vGvACib7BiY6Xv0BQbM4
22+
rWq2Rrjag1y5vVAF9gJkeo/3rhM6lE1ahDCRq0QcBMVzbxiE+3COIzRPmz14J3Yn
23+
0pkvzlVkNj5UZR8q91ESl+UxkFCP1wzcXgs0dpJWirQIOZ9E2eYv3LcjE68xjW1k
24+
c5q1GOGvJI7aXADxUZ4lFbz+NUb4Ts4HXHc8gV1Gm0vvmIqv2YfAvL5DXbKLdZxh
25+
73CxKvBMmTXIEQ+vQJ3p1ZnUnb+l6DoxEFWg/hXHmE5jY3P6HIVFdliXF5FEs1lr
26+
9snU2Pn1BDL+TBN7SX0QbKqArWA4qyn6eGH8Z1ULoUVBPCjwC9QuInp/9fqifFYo
27+
OM3A51MDGyc/HCVG6jNJEI5h71QGHlPfyQybpjy7rQSe
28+
=YwXc
29+
-----END PGP PUBLIC KEY BLOCK-----

‎monitoring.yml

Lines changed: 5 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -0,0 +1,5 @@
1+
- hosts: monitoring
2+
become: true
3+
roles:
4+
- usegalaxy_eu.influxdb
5+
- cloudalchemy.grafana

‎requirements.yml

Lines changed: 8 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -38,3 +38,11 @@
3838
version: 1.0.2
3939
- src: galaxyproject.gxadmin
4040
version: 0.0.12
41+
- src: usegalaxy_eu.influxdb
42+
version: v6.0.7
43+
# Monitoring
44+
- name: dj-wasabi.telegraf
45+
src: https://github.com/dj-wasabi/ansible-telegraf
46+
version: 6f6fdf7f5ead491560783d52528b79e9e088bd5b
47+
- src: cloudalchemy.grafana
48+
version: 0.14.2

‎roles/cloudalchemy.grafana/.ansible-lint

Lines changed: 3 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -0,0 +1,3 @@
1+
---
2+
skip_list:
3+
- '602'

‎roles/cloudalchemy.grafana/.github/lock.yml

Lines changed: 2 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -0,0 +1,2 @@
1+
---
2+
_extends: auto-maintenance

‎roles/cloudalchemy.grafana/.github/stale.yml

Lines changed: 2 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -0,0 +1,2 @@
1+
---
2+
_extends: auto-maintenance

‎roles/cloudalchemy.grafana/.gitignore

Lines changed: 14 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -0,0 +1,14 @@
1+
*.swp
2+
*.swo
3+
*.idea
4+
.vagrant/
5+
*.retry
6+
*.log
7+
*.swp
8+
*.swo
9+
*.idea
10+
.molecule
11+
.cache
12+
__pycache__/
13+
.pytest_cache
14+
.tox

‎roles/cloudalchemy.grafana/.travis.yml

Lines changed: 27 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -0,0 +1,27 @@
1+
---
2+
sudo: required
3+
language: python
4+
cache: pip
5+
services:
6+
- docker
7+
env:
8+
- ANSIBLE=2.5
9+
- ANSIBLE=2.6
10+
- ANSIBLE=2.7
11+
matrix:
12+
fast_finish: true
13+
install:
14+
- pip install tox-travis git-semver
15+
script:
16+
- tox
17+
deploy:
18+
provider: script
19+
skip_cleanup: true
20+
script: .travis/releaser.sh
21+
on:
22+
branch: master
23+
branches:
24+
only:
25+
- master
26+
notifications:
27+
webhooks: https://galaxy.ansible.com/api/v1/notifications/

‎roles/cloudalchemy.grafana/.travis/releaser.sh

Lines changed: 71 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -0,0 +1,71 @@
1+
#!/bin/bash
2+
#
3+
# Copyright (C) 2018 Pawel Krupa (@paulfantom) - All Rights Reserved
4+
# Permission to copy and modify is granted under the MIT license
5+
#
6+
# Script to automatically do a couple of things:
7+
# - generate a new tag according to semver (https://semver.org/)
8+
# - generate CHANGELOG.md by using https://github.com/skywinder/github-changelog-generator
9+
# - sync CHANGELOG with GitHub releases by using https://github.com/mattbrictson/chandler
10+
#
11+
# Tags are generated by searching for a keyword in last commit message. Keywords are:
12+
# - [patch] or [fix] to bump patch number
13+
# - [minor], [feature] or [feat] to bump minor number
14+
# - [major] or [breaking change] to bump major number
15+
# All keywords MUST be surrounded with square braces.
16+
#
17+
# Script uses git mechanisms for locking, so it can be used in parallel builds
18+
#
19+
# Requirements:
20+
# - GH_TOKEN variable set with GitHub token. Access level: repo.public_repo
21+
# - docker
22+
# - git-semver python package (pip install git-semver)
23+
24+
# Exit when latest commit is tagged
25+
[[ $(git tag --points-at) ]] && exit 0
26+
27+
# Some basic variables
28+
GIT_MAIL="[email protected]"
29+
GIT_USER="cloudalchemybot"
30+
ORGANIZATION=$(echo "$TRAVIS_REPO_SLUG" | awk -F '/' '{print $1}')
31+
PROJECT=$(echo "$TRAVIS_REPO_SLUG" | awk -F '/' '{print $2}')
32+
GALAXY_URL="https://galaxy.ansible.com/${ORGANIZATION}/${PROJECT#ansible-}"
33+
34+
# Git config
35+
git config --global user.email "${GIT_MAIL}"
36+
git config --global user.name "${GIT_USER}"
37+
GIT_URL=$(git config --get remote.origin.url)
38+
GIT_URL=${GIT_URL#*//}
39+
40+
# Generate TAG
41+
GIT_TAG=none
42+
echo "Last commit message: $TRAVIS_COMMIT_MESSAGE"
43+
case "${TRAVIS_COMMIT_MESSAGE}" in
44+
*"[patch]"*|*"[fix]"* ) GIT_TAG=$(git semver --next-patch) ;;
45+
*"[minor]"*|*"[feat]"*|*"[feature]"* ) GIT_TAG=$(git semver --next-minor) ;;
46+
*"[major]"*|*"[breaking change]"* ) GIT_TAG=$(git semver --next-major) ;;
47+
*) echo "Keyword not detected. Doing nothing" ;;
48+
esac
49+
if [ "$GIT_TAG" != "none" ]; then
50+
echo "Assigning new tag: $GIT_TAG"
51+
git tag "$GIT_TAG" -a -m "Automatic tag generation for travis build no. $TRAVIS_BUILD_NUMBER"
52+
git push "https://${GH_TOKEN}:@${GIT_URL}" --tags || exit 0
53+
fi
54+
55+
# Generate CHANGELOG.md
56+
git checkout master
57+
git pull
58+
docker run -it --rm -v "$(pwd)":/usr/local/src/your-app ferrarimarco/github-changelog-generator:1.14.3 \
59+
-u "${ORGANIZATION}" -p "${PROJECT}" --token "${GH_TOKEN}" \
60+
--release-url "${GALAXY_URL}" \
61+
--unreleased-label "**Next release**" --no-compare-link
62+
63+
git add CHANGELOG.md
64+
git commit -m '[ci skip] Automatic changelog update'
65+
66+
git push "https://${GH_TOKEN}:@${GIT_URL}" || exit 0
67+
68+
# Sync changelog to github releases
69+
if [ "$GIT_TAG" != "none" ]; then
70+
docker run -e CHANDLER_GITHUB_API_TOKEN="${GH_TOKEN}" -v "$(pwd)":/chandler -ti whizark/chandler push "${GIT_TAG}"
71+
fi

‎roles/cloudalchemy.grafana/.yamllint

Lines changed: 14 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -0,0 +1,14 @@
1+
extends: default
2+
ignore: |
3+
.travis/
4+
.travis.yml
5+
meta/
6+
7+
rules:
8+
braces:
9+
max-spaces-inside: 1
10+
level: error
11+
brackets:
12+
max-spaces-inside: 1
13+
level: error
14+
line-length: disable

0 commit comments

Comments
 (0)