diff --git a/.gitignore b/.gitignore new file mode 100644 index 0000000..0b24208 --- /dev/null +++ b/.gitignore @@ -0,0 +1 @@ +dist/history_influxdb_local.conf diff --git a/DEVELOPMENT.md b/DEVELOPMENT.md index 8bcaf21..a33b8b5 100644 --- a/DEVELOPMENT.md +++ b/DEVELOPMENT.md @@ -1,8 +1,8 @@ # Local Development -You will need Zabbix sources for compiling the module. For running the module you have two options: -- compile whole Zabbix source code and run from there (follow https://www.zabbix.com/developers) -- install Zabbix from distributed packages and just put compiled module and module's config in the modules path, then restart Zabbix as usual +You will need Zabbix sources for compiling the module. For running the zabbix-server you have two options: +- compile Zabbix source code and run from there (follow https://www.zabbix.com/developers) +- install Zabbix from distributed packages and just link compiled module and module's config in the modules path, then restart Zabbix as usual (you need Zabbix source code only to compile module) We are using the second option. Our local development environment is: - VirtualBox with Ubuntu 18.04 @@ -24,7 +24,7 @@ https://www.zabbix.com/documentation/3.4/manual/installation/install#installing_ Download the zabbix source code from https://www.zabbix.com/download_sources and place it in the home dir (or any other workdir you like). -Run git clone of this repository inside of the zabbix sources under `src/modules` +Get a copy of this module's sources - run `git clone` inside of the zabbix sources under `src/modules` ``` $ cd zabbix-/src/modules/ @@ -37,7 +37,7 @@ Use option `ForceModuleDebugLogging=1` in the local config, this will enforce de Also put all necessities for your InfluxDB connection in the local config. -We create symlink `/usr/lib/zabbix/modules` pointing to module's `dist/`. +Create symlink `/usr/lib/zabbix/modules` pointing to module's `dist/` to allow zabbix-server read the module. ``` # sudo ln -s ~/zabbix-/src/modules/zabbix-history-influxdb/dist /usr/lib/zabbix/modules @@ -45,7 +45,7 @@ We create symlink `/usr/lib/zabbix/modules` pointing to module's `dist/`. ## Similarly to regular installation -Now edit the main Zabbix server configuration file, usually in `/etc/zabbix/zabbix_server.conf` and change modules section near the end to point on your module: +Enable the module in the main Zabbix server configuration file (usually in `/etc/zabbix/zabbix_server.conf`) - change modules section near the end to point on your module: ``` LoadModulePath=/usr/lib/zabbix/modules @@ -53,18 +53,20 @@ LoadModulePath=/usr/lib/zabbix/modules LoadModule=history_influxdb.so ``` -Restart Zabbix server daemon, usually: +Restart Zabbix server daemon: ``` # systemctl restart zabbix-server ``` -Open Zabbix log, usually `/var/log/zabbix_server.log`. You now should see all the debug messages from the module, including potential errors. +Running `tail -f /var/log/zabbix_server.log` should list all the debug messages from the module, including potential errors. # Compiling -You will need the packages +Finally if you change any module source, you need to compile it with `make` command. + +For compiler to work you will need, apart from Zabbix sources, these packages ``` # apt install gcc libcurl4-openssl-dev libpcre3-dev libevent-dev diff --git a/README.md b/README.md index ccc39dc..6d420d2 100644 --- a/README.md +++ b/README.md @@ -17,6 +17,8 @@ - `InfluxDBSSLInsecure=0` - `InfluxDBUser=` - `InfluxDBPassword=` + - `ZabbixMajorVersion=4` + - `DatabaseEngine=mysql` This is what you get in Grafana: diff --git a/dist/history_influxdb.conf b/dist/history_influxdb.conf index 19e9d79..eec3974 100644 --- a/dist/history_influxdb.conf +++ b/dist/history_influxdb.conf @@ -66,3 +66,19 @@ InfluxDBName=zabbix # # Default: # ForceModuleDebugLogging=0 + +### Option: ZabbixMajorVersion +# Flag to provide compatibility between Zabbix 3 and 4. +# From Zabbix 3 to 4 table groups has been renamed to hstgrp for example. +# To enable pre Zabbix 4 set this to 3 +# Only values 3 and 4 are allowed here +# +# Default: +# ZabbixMajorVersion=4 + +### Option: DatabaseEngine +# Provide compatibility with MySQL and PostgreSQL engines. +# Value can only be 'mysql' or 'postgresql' (lowercase) +# +# Default: +# DatabaseEngine=mysql diff --git a/dist/history_influxdb.so b/dist/history_influxdb.so old mode 100644 new mode 100755 index 4cd7c79..363f1bf Binary files a/dist/history_influxdb.so and b/dist/history_influxdb.so differ diff --git a/src/history_influxdb.c b/src/history_influxdb.c index 5cc0a2b..4661aed 100644 --- a/src/history_influxdb.c +++ b/src/history_influxdb.c @@ -141,7 +141,13 @@ int zbx_module_init(void) zbx_snprintf(influxdb_write_url, CURL_LEN, "%s://%s:%s/write?db=%s&u=%s&p=%s", CONFIG_INFLUXDB_PROTOCOL, CONFIG_INFLUXDB_ADDRESS, CONFIG_INFLUXDB_PORT, CONFIG_INFLUXDB_NAME, CONFIG_INFLUXDB_USER, CONFIG_INFLUXDB_PWD); } + if(CONFIG_DATABASE_ENGINE == NULL){ + zbx_error("DatabaseEngine missconfigured expected one of (mysql, postgresql), but found %s", PARSE_DATABASE_ENGINE); + exit(EXIT_FAILURE); + } zabbix_log(LOG_LEVEL_INFORMATION, "[%s] Initialised History InfluxDB module, target: %s", MODULE_NAME, influxdb_write_url); + zabbix_log(LOG_LEVEL_INFORMATION, "[%s] Database Engine used: %s", MODULE_NAME, PARSE_DATABASE_ENGINE); + zabbix_log(LOG_LEVEL_INFORMATION, "[%s] Using compatibility with Zabbix %d", MODULE_NAME, CONFIG_ZABBIX_MAJOR_VERSION); return ZBX_MODULE_OK; } @@ -214,54 +220,139 @@ char *itemid_to_influx_data(zbx_uint64_t itemid) DB_RESULT result; DB_ROW row; char *ret_string; +// const char *cut_fnc; +// const char *agg_fnc; +// const char *agg_sep; + static char query_str[3000]; + + switch((int)(uintptr_t)CONFIG_DATABASE_ENGINE) { + case DATABASE_ENGINE_POSTGRESQL: + // prepare query for PostgreSQL + zbx_snprintf(query_str, sizeof(query_str), + "SELECT " + // item name with $1 - $9 replaced and escaped ',' and ' ' + "replace(replace(replace(" + "coalesce(" + // replace all $1 - $9 in item name with key parameters + "replace(replace(replace(replace(replace(replace(replace(replace(replace(i.name," + " '$1', split_part(substring(i.key_ FROM '\\[(.+)\\]'), ',', 1))," + " '$2', split_part(substring(i.key_ FROM '\\[(.+)\\]'), ',', 2))," + " '$3', split_part(substring(i.key_ FROM '\\[(.+)\\]'), ',', 3))," + " '$4', split_part(substring(i.key_ FROM '\\[(.+)\\]'), ',', 4))," + " '$5', split_part(substring(i.key_ FROM '\\[(.+)\\]'), ',', 5))," + " '$6', split_part(substring(i.key_ FROM '\\[(.+)\\]'), ',', 6))," + " '$7', split_part(substring(i.key_ FROM '\\[(.+)\\]'), ',', 7))," + " '$8', split_part(substring(i.key_ FROM '\\[(.+)\\]'), ',', 8))," + " '$9', split_part(substring(i.key_ FROM '\\[(.+)\\]'), ',', 9))," + // or use plain item name if no variables to replace + "i.name" + "), " + "' ', '\\ '), '\"', '\\\"'), ',', '\\,')" + " || " + // host name with escaped ',' and ' ' + "',host_name=' || " + "replace(replace((" + "select h.name from hosts h where h.hostid=i.hostid" + "), ' ', '\\ '), ',', '\\,')" + " || " + // host groups with escaped ',' and ' ' joined with '|' + "',host_groups=' || " + "replace(replace((" + "select string_agg(g.name, '|') " + "from %s g " + "inner join hosts_groups hg on hg.groupid = g.groupid " + "where hg.hostid=i.hostid" + "), ' ', '\\ '), ',', '\\,')" + // " || " + // item_key with escaped ',' and ' ' + // "',item_key=' || " + // "replace(replace((" + // "i.key_" + // "), ' ', '\\ '), ',', '\\,')" + " || " + // applications + "coalesce(" + "',applications=' || replace(replace((" + "select string_agg(a.name, '|') " + "from applications a " + "inner join items_applications ia on ia.applicationid = a.applicationid " + "where ia.itemid=i.itemid" + "), ' ', '\\ '), ',', '\\,'), " + "'') " + "FROM items i WHERE i.itemid=" ZBX_FS_UI64, + // Zabbix 3 vs Zabbix 4 table name + (CONFIG_ZABBIX_MAJOR_VERSION > (int*) 3) ? "hstgrp": "groups", itemid); + break; + + case DATABASE_ENGINE_MYSQL: + // prepare query for MySQL + zbx_snprintf(query_str, sizeof(query_str), + "SELECT CONCAT(" + // item name with $1 - $9 replaced and ',', '"' and ' ' escaped + "replace(replace(replace(" + "coalesce(" + // replace all $1 - $9 in item name with key parameters + // magic line explained: + // +------------------------------------------------+-----------------------------------------------+---------------------------------------------------------------------------------------------------------------------------+ + // | key_ | name | substring(i.key_, position('[' in i.key_)+1, length(i.key_) - position('[' in i.key_) - position(']' in reverse(i.key_))) | + // +------------------------------------------------+-----------------------------------------------+---------------------------------------------------------------------------------------------------------------------------+ + // | web.test.rspcode[CURL mockapp - http,Homepage] | Response code for step "$2" of scenario "$1". | CURL mockapp - http,Homepage | + // +------------------------------------------------+-----------------------------------------------+---------------------------------------------------------------------------------------------------------------------------+ + // find first '[' and last ']' in the key_ and for what is between use substring_index() to extract correct parameter + "replace(replace(replace(replace(replace(replace(replace(replace(replace(i.name," + " '$1', substring_index(substring(i.key_, position('[' in i.key_)+1, length(i.key_) - position('[' in i.key_) - position(']' in reverse(i.key_))), ',', 1))," + " '$2', substring_index(substring(i.key_, position('[' in i.key_)+1, length(i.key_) - position('[' in i.key_) - position(']' in reverse(i.key_))), ',', 2))," + " '$3', substring_index(substring(i.key_, position('[' in i.key_)+1, length(i.key_) - position('[' in i.key_) - position(']' in reverse(i.key_))), ',', 3))," + " '$4', substring_index(substring(i.key_, position('[' in i.key_)+1, length(i.key_) - position('[' in i.key_) - position(']' in reverse(i.key_))), ',', 4))," + " '$5', substring_index(substring(i.key_, position('[' in i.key_)+1, length(i.key_) - position('[' in i.key_) - position(']' in reverse(i.key_))), ',', 5))," + " '$6', substring_index(substring(i.key_, position('[' in i.key_)+1, length(i.key_) - position('[' in i.key_) - position(']' in reverse(i.key_))), ',', 6))," + " '$7', substring_index(substring(i.key_, position('[' in i.key_)+1, length(i.key_) - position('[' in i.key_) - position(']' in reverse(i.key_))), ',', 7))," + " '$8', substring_index(substring(i.key_, position('[' in i.key_)+1, length(i.key_) - position('[' in i.key_) - position(']' in reverse(i.key_))), ',', 8))," + " '$9', substring_index(substring(i.key_, position('[' in i.key_)+1, length(i.key_) - position('[' in i.key_) - position(']' in reverse(i.key_))), ',', 9))," + // or use plain item name if no variables to replace + "i.name" + "), " + "' ', '\\\\ '), '\"', '\\\\\"'), ',', '\\\\,')" + ", " + // host name with escaped ',' and ' ' + "concat(',host_name=', replace(replace((" + "select h.name from hosts h where h.hostid=i.hostid" + "), ' ', '\\\\ '), ',', '\\\\,'))" + ", " + // host groups with escaped ',' and ' ' joined with '|' + "concat(',host_groups=', replace(replace((" + "select group_concat(g.name SEPARATOR '|') " + "from %s g " + "inner join hosts_groups hg on hg.groupid = g.groupid " + "where hg.hostid=i.hostid" + "), ' ', '\\\\ '), ',', '\\\\,'))" + // ", " + // item_key with escaped ',' and ' ' + // "concat(',item_key=', replace(replace((" + // "i.key_" + // "), ' ', '\\\\ '), ',', '\\\\,'))" + ", " + // applications + "coalesce(concat(',applications=', replace(replace((" + "select group_concat(a.name SEPARATOR '|') " + "from applications a " + "inner join items_applications ia on ia.applicationid = a.applicationid " + "where ia.itemid=i.itemid" + "), ' ', '\\\\ '), ',', '\\\\,')), " + "'') " + ") FROM items i WHERE i.itemid=" ZBX_FS_UI64, + + // Zabbix 3 vs Zabbix 4 table name + (CONFIG_ZABBIX_MAJOR_VERSION > (int*) 3) ? "hstgrp": "groups", itemid); + break; - result = DBselect("SELECT replace(replace(" - "coalesce(" - "replace(" - "replace(" - "replace(" - "replace(" - "replace(" - "replace(" - "replace(" - "replace(" - "replace(" - "i.name, '$1'," - "split_part(substring(i.key_ FROM '\\[(.+)\\]'), ',', 1)), '$2'," - "split_part(substring(i.key_ FROM '\\[(.+)\\]'), ',', 2)), '$3'," - "split_part(substring(i.key_ FROM '\\[(.+)\\]'), ',', 3)), '$4'," - "split_part(substring(i.key_ FROM '\\[(.+)\\]'), ',', 4)), '$5'," - "split_part(substring(i.key_ FROM '\\[(.+)\\]'), ',', 5)), '$6'," - "split_part(substring(i.key_ FROM '\\[(.+)\\]'), ',', 6)), '$7'," - "split_part(substring(i.key_ FROM '\\[(.+)\\]'), ',', 7)), '$8'," - "split_part(substring(i.key_ FROM '\\[(.+)\\]'), ',', 8)), '$9'," - "split_part(substring(i.key_ FROM '\\[(.+)\\]'), ',', 9))," - "i.name" - "), ',', '\\,'), ' ', '\\ ') ||" - - "',host_name=' || replace(replace((" - "select h.name from hosts h where h.hostid=i.hostid" - "), ' ', '\\ '), ',', '\\,') ||" - - "',host_groups=' || coalesce(replace((" - "select string_agg(g.name, '|') " - "from groups g " - "inner join hosts_groups hg on hg.groupid = g.groupid " - "where hg.hostid=i.hostid" - "), ' ', '\\ '),'') ||" - - // "',item_key=' || replace(replace((" - // "i.key_" - // "), ' ', '\\ '), ',', '\\,') ||" - - "coalesce(',applications=' || replace(replace((" - "select string_agg(a.name, '|') " - "from applications a " - "inner join items_applications ia on ia.applicationid = a.applicationid " - "where ia.itemid=i.itemid" - "), ' ', '\\ '), ',', '\\,'),'') " - - "from items i where i.itemid=" ZBX_FS_UI64, itemid); + default: + THIS_SHOULD_NEVER_HAPPEN; + } + + // log for debugging the query + // zabbix_log(MODULE_LOG_LEVEL, "[%s] itemid_to_influx_data query: %s", MODULE_NAME, query_str); + result = DBselect("%s", query_str); if (NULL != (row = DBfetch(result))) { @@ -272,6 +363,9 @@ char *itemid_to_influx_data(zbx_uint64_t itemid) ret_string = NULL; } DBfree_result(result); + + // log for debugging the query + // zabbix_log(MODULE_LOG_LEVEL, "[%s] itemid_to_influx_data result: %s", MODULE_NAME, ret_string); return ret_string; } diff --git a/src/load_config.c b/src/load_config.c index 4c612cd..a8aaba3 100644 --- a/src/load_config.c +++ b/src/load_config.c @@ -13,6 +13,9 @@ char *CONFIG_INFLUXDB_USER = NULL; char *CONFIG_INFLUXDB_PWD = NULL; int *CONFIG_INFLUXDB_SSL_INSECURE = NULL; int *CONFIG_FORCE_MODULE_DEBUG = NULL; +int *CONFIG_ZABBIX_MAJOR_VERSION = NULL; +int *CONFIG_DATABASE_ENGINE = NULL; +char *PARSE_DATABASE_ENGINE = NULL; /********************************************************************* @@ -46,6 +49,10 @@ void zbx_module_load_config(void) PARM_OPT, 0, 1}, {"ForceModuleDebugLogging", &CONFIG_FORCE_MODULE_DEBUG, TYPE_INT, PARM_OPT, 0, 1}, + {"ZabbixMajorVersion", &CONFIG_ZABBIX_MAJOR_VERSION, TYPE_INT, + PARM_OPT, 3, 4}, + {"DatabaseEngine", &PARSE_DATABASE_ENGINE, TYPE_STRING, + PARM_OPT, 0, 0}, {NULL} }; @@ -54,7 +61,9 @@ void zbx_module_load_config(void) CONFIG_INFLUXDB_ADDRESS = zbx_strdup(CONFIG_INFLUXDB_ADDRESS, "localhost"); CONFIG_INFLUXDB_PORT = zbx_strdup(CONFIG_INFLUXDB_PORT, "8086"); CONFIG_INFLUXDB_PROTOCOL = zbx_strdup(CONFIG_INFLUXDB_PROTOCOL, "http"); - CONFIG_FORCE_MODULE_DEBUG = 0; + CONFIG_FORCE_MODULE_DEBUG = (int*) 0; + CONFIG_ZABBIX_MAJOR_VERSION = (int*) 4; + PARSE_DATABASE_ENGINE = zbx_strdup(PARSE_DATABASE_ENGINE, "mysql"); // load main config file @@ -64,6 +73,14 @@ void zbx_module_load_config(void) // load local config file if present parse_cfg_file(MODULE_LOCAL_CONFIG_FILE, module_cfg, ZBX_CFG_FILE_OPTIONAL, ZBX_CFG_STRICT); + // parse database engine + if (strcmp(PARSE_DATABASE_ENGINE, "mysql") == 0) { + CONFIG_DATABASE_ENGINE = (int*) DATABASE_ENGINE_MYSQL; + } + else if (strcmp(PARSE_DATABASE_ENGINE, "postgresql") == 0) { + CONFIG_DATABASE_ENGINE = (int*) DATABASE_ENGINE_POSTGRESQL; + } + // clean up path variables zbx_free(MODULE_CONFIG_FILE); zbx_free(MODULE_LOCAL_CONFIG_FILE); diff --git a/src/load_config.h b/src/load_config.h index a2c414b..432c07f 100644 --- a/src/load_config.h +++ b/src/load_config.h @@ -15,6 +15,9 @@ #define CONFIG_DISABLE 0 #define CONFIG_ENABLE 1 +#define DATABASE_ENGINE_POSTGRESQL 1 +#define DATABASE_ENGINE_MYSQL 2 + extern char *CONFIG_LOAD_MODULE_PATH; @@ -29,6 +32,9 @@ extern char *CONFIG_INFLUXDB_USER; extern char *CONFIG_INFLUXDB_PWD; extern int *CONFIG_INFLUXDB_SSL_INSECURE; extern int *CONFIG_FORCE_MODULE_DEBUG; +extern int *CONFIG_ZABBIX_MAJOR_VERSION; +extern int *CONFIG_DATABASE_ENGINE; +extern char *PARSE_DATABASE_ENGINE; #endif /* __ZABBIX_LOAD_CONFIG_H */