Added new prometheus metrics mimicking current ones

include/MySQL_HostGroups_Manager.h

@@ -10,6 +10,9 @@
 #include "thread.h"
 #include "wqueue.h"
 
+// Headers for declaring Prometheus counters
+#include <prometheus/counter.h>
+#include <prometheus/gauge.h>
 
 #include "ev.h"
 
@@ -352,6 +355,49 @@ class MySQL_HostGroups_Manager {
 		unsigned long long access_denied_max_connections;
 		unsigned long long access_denied_max_user_connections;
 		unsigned long long select_for_update_or_equivalent;
+
+		//////////////////////////////////////////////////////
+		///              Prometheus Metrics                ///
+		//////////////////////////////////////////////////////
+
+		/// Prometheus client metrics
+		prometheus::Counter* p_client_connections_created { nullptr };
+		prometheus::Counter* p_client_connections_aborted { nullptr };
+		prometheus::Gauge*   p_client_connections { nullptr };
+		/// Prometheus server metrics
+		prometheus::Counter* p_server_connections_aborted { nullptr };
+		prometheus::Counter* p_server_connections_created { nullptr };
+		prometheus::Counter* p_server_connections_delayed { nullptr };
+		prometheus::Gauge*   p_server_connections_connected { nullptr };
+		/// Prometheus access metrics
+		prometheus::Counter* p_access_denied_wrong_password { nullptr };
+		prometheus::Counter* p_access_denied_max_connections { nullptr };
+		prometheus::Counter* p_access_denied_max_user_connections { nullptr };
+		/// TODO: Prometheus myconnpoll metrics
+		prometheus::Counter* p_myconnpoll_get { nullptr };
+		prometheus::Counter* p_myconnpoll_get_ok { nullptr };
+		prometheus::Counter* p_myconnpoll_get_ping { nullptr };
+		prometheus::Counter* p_myconnpoll_push { nullptr };
+		prometheus::Counter* p_myconnpoll_reset { nullptr };
+		prometheus::Counter* p_myconnpoll_destroy { nullptr };
+		/// Prometheus backend metrics
+		prometheus::Counter* p_backend_change_user { nullptr };
+		prometheus::Counter* p_backend_init_db { nullptr };
+		prometheus::Counter* p_backend_set_names { nullptr };
+		/// Prometheus frontend metrics
+		prometheus::Counter* p_frontend_init_db { nullptr };
+		prometheus::Counter* p_frontend_set_names { nullptr };
+		prometheus::Counter* p_frontend_use_db { nullptr };
+		/// Prometheus ** metrics
+		prometheus::Counter* p_autocommit_cnt { nullptr };
+		prometheus::Counter* p_commit_cnt { nullptr };
+		prometheus::Counter* p_rollback_cnt { nullptr };
+		prometheus::Counter* p_autocommit_cnt_filtered { nullptr };
+		prometheus::Counter* p_commit_cnt_filtered { nullptr };
+		prometheus::Counter* p_rollback_cnt_filtered { nullptr };
+		prometheus::Counter* p_select_for_update_or_equivalent { nullptr };
+
+		//////////////////////////////////////////////////////
 	} status;
 	wqueue<MySQL_Connection *> queue;
 	MySQL_HostGroups_Manager();

include/MySQL_Monitor.hpp

@@ -1,10 +1,15 @@
 #ifndef __CLASS_MYSQL_MONITOR_H
 #define __CLASS_MYSQL_MONITOR_H
+#include "prometheus/gauge.h"
 #include "proxysql.h"
 #include "cpp.h"
 #include "thread.h"
 #include "wqueue.h"
 
+// Headers for declaring Prometheus counters
+#include <prometheus/counter.h>
+#include <prometheus/gauge.h>
+
 //#define MONITOR_SQLITE_TABLE_MYSQL_SERVER_CONNECT "CREATE TABLE mysql_server_connect (hostname VARCHAR NOT NULL , port INT NOT NULL DEFAULT 3306 , time_since INT NOT NULL DEFAULT 0 , time_until INT NOT NULL DEFAULT 0 , connect_success_count INT NOT NULL DEFAULT 0 , connect_success_first INT NOT NULL DEFAULT 0 , connect_success_last INT NOT NULL DEFAULT 0 , connect_success_time_min INT NOT NULL DEFAULT 0 , connect_success_time_max INT NOT NULL DEFAULT 0 , connect_success_time_total INT NOT NULL DEFAULT 0 , connect_failure_count INT NOT NULL DEFAULT 0 , connect_failure_first INT NOT NULL DEFAULT 0 , connect_failure_last INT NOT NULL DEFAULT 0 , PRIMARY KEY (hostname, port))"
 
 //#define MONITOR_SQLITE_TABLE_MYSQL_SERVER_PING "CREATE TABLE mysql_server_ping (hostname VARCHAR NOT NULL , port INT NOT NULL DEFAULT 3306 , time_since INT NOT NULL DEFAULT 0 , time_until INT NOT NULL DEFAULT 0 , ping_success_count INT NOT NULL DEFAULT 0 , ping_success_first INT NOT NULL DEFAULT 0, ping_success_last INT NOT NULL DEFAULT 0 , ping_success_time_min INT NOT NULL DEFAULT 0 , ping_success_time_max INT NOT NULL DEFAULT 0 , ping_success_time_total INT NOT NULL DEFAULT 0 , ping_failure_count INT NOT NULL DEFAULT 0 , ping_failure_first INT NOT NULL DEFAULT 0 , ping_failure_last INT NOT NULL DEFAULT 0 , PRIMARY KEY (hostname, port))"
@@ -254,6 +259,19 @@ class MySQL_Monitor {
 	unsigned long long read_only_check_ERR;
 	unsigned long long replication_lag_check_OK;
 	unsigned long long replication_lag_check_ERR;
+	struct {
+		prometheus::Gauge* p_num_threads { nullptr };
+		prometheus::Gauge* p_aux_threads { nullptr };
+		prometheus::Counter* p_started_threads { nullptr };
+		prometheus::Counter* p_connect_check_OK { nullptr };
+		prometheus::Counter* p_connect_check_ERR { nullptr };
+		prometheus::Counter* p_ping_check_OK { nullptr };
+		prometheus::Counter* p_ping_check_ERR { nullptr };
+		prometheus::Counter* p_read_only_check_OK { nullptr };
+		prometheus::Counter* p_read_only_check_ERR { nullptr };
+		prometheus::Counter* p_replication_lag_check_OK { nullptr };
+		prometheus::Counter* p_replication_lag_check_ERR { nullptr };
+	} metrics;
 	std::unique_ptr<wqueue<WorkItem*>> queue;
 	MySQL_Monitor_Connection_Pool *My_Conn_Pool;
 	bool shutdown;

include/MySQL_Thread.h

@@ -9,6 +9,10 @@
 #endif // IDLE_THREADS
 #include <atomic>
 
+// Headers for declaring Prometheus counters
+#include <prometheus/counter.h>
+#include <prometheus/gauge.h>
+
 #define MIN_POLL_LEN 8
 #define MIN_POLL_DELETE_RATIO  8
 #define MY_EPOLL_THREAD_MAXEVENTS 128
@@ -394,6 +398,59 @@ class MySQL_Threads_Handler
 	} variables;
 	struct {
 		unsigned int mirror_sessions_current;
+
+		//////////////////////////////////////////////////////
+		///              Prometheus Metrics                ///
+		//////////////////////////////////////////////////////
+
+		prometheus::Gauge* p_active_transations { nullptr };
+		prometheus::Gauge* p_non_idle_client_connections { nullptr };
+		prometheus::Counter* p_queries_backends_bytes_recv { nullptr };
+		prometheus::Counter* p_queries_backends_bytes_sent { nullptr };
+		prometheus::Counter* p_queries_frontends_bytes_recv { nullptr };
+		prometheus::Counter* p_queries_frontends_bytes_sent { nullptr };
+		prometheus::Counter* p_query_processor_time { nullptr };
+		prometheus::Counter* p_backend_query_time { nullptr };
+		prometheus::Gauge* p_mysql_backend_buffers_bytes { nullptr };
+		prometheus::Gauge* p_mysql_frontend_buffers_bytes { nullptr };
+		prometheus::Gauge* p_mysql_session_internal_bytes { nullptr };
+		prometheus::Counter* p_total_backend_stmt_prepare { nullptr };
+		prometheus::Counter* p_total_backend_stmt_execute { nullptr };
+		prometheus::Counter* p_total_backend_stmt_close { nullptr };
+		prometheus::Counter* p_total_frontend_stmt_prepare { nullptr };
+		prometheus::Counter* p_total_frontend_stmt_execute { nullptr };
+		prometheus::Counter* p_total_frontend_stmt_close { nullptr };
+		prometheus::Gauge* p_mirror_sessions_current { nullptr };
+		prometheus::Gauge* p_total_mirror_queue { nullptr };
+		prometheus::Counter* p_total_queries { nullptr };
+		prometheus::Counter* p_slow_queries { nullptr };
+		prometheus::Counter* p_gtid_queries { nullptr };
+		prometheus::Counter* p_gtid_session_collected { nullptr };
+		prometheus::Counter* p_servers_table_version { nullptr };
+		prometheus::Gauge* p_num_threads { nullptr };
+		prometheus::Counter* p_ConnPool_get_conn_latency_awareness { nullptr };
+		prometheus::Counter* p_ConnPool_get_conn_immediate { nullptr };
+		prometheus::Counter* p_ConnPool_get_conn_success { nullptr };
+		prometheus::Counter* p_ConnPool_get_conn_failure { nullptr };
+		prometheus::Counter* p_generated_error_packets { nullptr };
+		prometheus::Counter* p_max_connect_timeout { nullptr };
+		prometheus::Counter* p_backend_lagging_during_query { nullptr };
+		prometheus::Counter* p_backend_offline_during_query { nullptr };
+		prometheus::Counter* p_queries_with_max_lag_ms { nullptr };
+		prometheus::Counter* p_queries_with_max_lag_ms__delayed { nullptr };
+		prometheus::Counter* p_queries_with_max_lag_ms__total_wait_time_us { nullptr };
+		prometheus::Counter* p_mysql_unexpected_frontend_com_quit { nullptr };
+		prometheus::Counter* p_get_hostgroup_locked { nullptr };
+		prometheus::Counter* p_hostgroup_locked_set_cmds { nullptr };
+		prometheus::Counter* p_hostgroup_locked_queries { nullptr };
+		prometheus::Counter* p_mysql_unexpected_frontend_packets { nullptr };
+		prometheus::Counter* p_aws_aurora_replicas_skipped_during_query { nullptr };
+		prometheus::Counter* p_automatic_detected_sqli { nullptr };
+		prometheus::Counter* p_whitelisted_sqli_fingerprint { nullptr };
+		prometheus::Counter* p_killed_connections { nullptr };
+		prometheus::Counter* p_killed_queries { nullptr };
+
+		//////////////////////////////////////////////////////
 	} status_variables;
 	unsigned int num_threads;
 	proxysql_mysql_thread_t *mysql_threads;

lib/MySQL_HostGroups_Manager.cpp

@@ -1,9 +1,12 @@
+#include "prometheus/counter.h"
+#include "prometheus/gauge.h"
 #include "proxysql.h"
 #include "cpp.h"
 #include "SpookyV2.h"
 
 #include "MySQL_PreparedStatement.h"
 #include "MySQL_Data_Stream.h"
+#include <memory>
 
 #define char_malloc (char *)malloc
 #define itostr(__s, __i)  { __s=char_malloc(32); sprintf(__s, "%lld", __i); }
@@ -855,6 +858,7 @@ void MySrvC::connect_error(int err_num) {
 	// as a single connection failure won't make a significant difference
 	__sync_fetch_and_add(&connect_ERR,1);
 	__sync_fetch_and_add(&MyHGM->status.server_connections_aborted,1);
+	MyHGM->status.p_server_connections_aborted->Increment();
 	if (err_num >= 1048 && err_num <= 1052)
 		return;
 	if (err_num >= 1054 && err_num <= 1075)
@@ -1020,6 +1024,68 @@ MySQL_HostGroups_Manager::MySQL_HostGroups_Manager() {
 		rand_del[7] = 0;
 	}
 	pthread_mutex_init(&mysql_errors_mutex, NULL);
+
+	// Create and register prometheus metrics
+	auto& new_gauge_family {
+		prometheus::BuildGauge()
+			.Name("MySQL_HostGroups_Manager_Gauges")
+			.Register(*GloVars.prometheus_registry)
+	};
+	auto& new_counter_family {
+		prometheus::BuildCounter()
+			.Name("MySQL_HostGroups_Manager_Counters")
+			.Register(*GloVars.prometheus_registry)
+	};
+
+	// Server_Connections counters
+	this->status.p_server_connections_connected =
+		std::addressof(new_gauge_family.Add({{ "id", "Server_Connections_connected" }}));
+	this->status.p_server_connections_created =
+		std::addressof(new_counter_family.Add({{ "id", "Server_Connections_created" }}));
+	this->status.p_server_connections_delayed =
+		std::addressof(new_counter_family.Add({{ "id", "Server_Connections_delayed" }}));
+	this->status.p_server_connections_aborted =
+		std::addressof(new_counter_family.Add({{ "id", "Server_Connections_aborted" }}));
+
+	// Client_Connections counters
+	this->status.p_client_connections_created =
+		std::addressof(new_counter_family.Add({{ "id", "Client_Connections_created" }}));
+	this->status.p_client_connections_aborted =
+		std::addressof(new_counter_family.Add({{ "id", "Client_Connections_aborted" }}));
+	this->status.p_client_connections =
+		std::addressof(new_gauge_family.Add({{ "id", "Client_Connections_connected" }}));
+
+	// Com_* counters
+	this->status.p_autocommit_cnt =
+		std::addressof(new_counter_family.Add({{ "id", "Com_autocommit" }}));
+	this->status.p_autocommit_cnt_filtered =
+		std::addressof(new_counter_family.Add({{ "id", "Com_autocommit_filtered" }}));
+	this->status.p_rollback_cnt =
+		std::addressof(new_counter_family.Add({{ "id", "Com_rollback" }}));
+	this->status.p_rollback_cnt_filtered =
+		std::addressof(new_counter_family.Add({{ "id", "Com_rollback_filtered" }}));
+	this->status.p_backend_change_user =
+		std::addressof(new_counter_family.Add({{ "id", "Com_backend_change_user" }}));
+	this->status.p_backend_init_db =
+		std::addressof(new_counter_family.Add({{ "id", "Com_backend_init_db" }}));
+	this->status.p_backend_set_names =
+		std::addressof(new_counter_family.Add({{ "id", "Com_backend_set_names" }}));
+	this->status.p_frontend_init_db =
+		std::addressof(new_counter_family.Add({{ "id", "Com_frontend_init_db" }}));
+	this->status.p_frontend_set_names =
+		std::addressof(new_counter_family.Add({{ "id", "Com_frontend_set_names" }}));
+	this->status.p_frontend_use_db =
+		std::addressof(new_counter_family.Add({{ "id", "Com_frontend_use_db" }}));
+	this->status.p_select_for_update_or_equivalent =
+		std::addressof(new_counter_family.Add({{ "id", "Selects_for_update__autocommit0" }}));
+
+	// Access_* errors
+	this->status.p_access_denied_wrong_password =
+		std::addressof(new_counter_family.Add({{ "id", "Access_Denied_Wrong_Password" }}));
+	this->status.p_access_denied_max_connections =
+		std::addressof(new_counter_family.Add({{ "id", "Access_Denied_Max_Connections" }}));
+	this->status.p_access_denied_max_user_connections =
+		std::addressof(new_counter_family.Add({{ "id", "Access_Denied_Max_User_Connections" }}));
 }
 
 void MySQL_HostGroups_Manager::init() {
@@ -2743,6 +2809,7 @@ MySQL_Connection * MySrvConnList::get_random_MyConn(MySQL_Session *sess, bool ff
 					conn = new MySQL_Connection();
 					conn->parent=mysrvc;
 					__sync_fetch_and_add(&MyHGM->status.server_connections_created, 1);
+					MyHGM->status.p_server_connections_created->Increment();
 					proxy_debug(PROXY_DEBUG_MYSQL_CONNPOOL, 7, "Returning MySQL Connection %p, server %s:%d\n", conn, conn->parent->address, conn->parent->port);
 				} else {
 					// we may consider creating a new connection
@@ -2752,6 +2819,7 @@ MySQL_Connection * MySrvConnList::get_random_MyConn(MySQL_Session *sess, bool ff
 						conn = new MySQL_Connection();
 						conn->parent=mysrvc;
 						__sync_fetch_and_add(&MyHGM->status.server_connections_created, 1);
+						MyHGM->status.p_server_connections_created->Increment();
 						proxy_debug(PROXY_DEBUG_MYSQL_CONNPOOL, 7, "Returning MySQL Connection %p, server %s:%d\n", conn, conn->parent->address, conn->parent->port);
 					} else {
 						conn=(MySQL_Connection *)conns->remove_index_fast(i);
@@ -2774,11 +2842,13 @@ MySQL_Connection * MySrvConnList::get_random_MyConn(MySQL_Session *sess, bool ff
 		_myhgc->new_connections_now++;
 		if (_myhgc->new_connections_now > (unsigned int) mysql_thread___throttle_connections_per_sec_to_hostgroup) {
 			__sync_fetch_and_add(&MyHGM->status.server_connections_delayed, 1);
+			MyHGM->status.p_server_connections_delayed->Increment();
 			return NULL;
 		} else {
 			conn = new MySQL_Connection();
 			conn->parent=mysrvc;
 			__sync_fetch_and_add(&MyHGM->status.server_connections_created, 1);
+			MyHGM->status.p_server_connections_created->Increment();
 			proxy_debug(PROXY_DEBUG_MYSQL_CONNPOOL, 7, "Returning MySQL Connection %p, server %s:%d\n", conn, conn->parent->address, conn->parent->port);
 			return  conn;
 		}

lib/MySQL_Monitor.cpp

@@ -608,6 +608,38 @@ MySQL_Monitor::MySQL_Monitor() {
 		num_threads=16;	// limit to 16
 	}
 */
+
+	// Create and register prometheus metrics families
+	auto& new_gauge_family {
+		prometheus::BuildGauge()
+			.Name("MySQL_Monitor_Gauges")
+			.Register(*GloVars.prometheus_registry)
+	};
+	auto& new_counter_family {
+		prometheus::BuildCounter()
+			.Name("MySQL_Monitor_Counters")
+			.Register(*GloVars.prometheus_registry)
+	};
+
+	// Initialize prometheus metrics
+	this->metrics.p_num_threads =
+		std::addressof(new_gauge_family.Add({{ "id", "MySQL_Monitor_Workers" }}));
+	this->metrics.p_aux_threads =
+		std::addressof(new_gauge_family.Add({{ "id", "MySQL_Monitor_Workers_Aux" }}));
+	this->metrics.p_started_threads =
+		std::addressof(new_counter_family.Add({{ "id", "MySQL_Monitor_Workers_Started" }}));
+	this->metrics.p_connect_check_OK =
+		std::addressof(new_counter_family.Add({{ "id", "MySQL_Monitor_connect_check_OK" }}));
+	this->metrics.p_connect_check_ERR =
+		std::addressof(new_counter_family.Add({{ "id", "MySQL_Monitor_connect_check_ERR" }}));
+	this->metrics.p_ping_check_OK =
+		std::addressof(new_counter_family.Add({{ "id", "MySQL_Monitor_ping_check_OK" }}));
+	this->metrics.p_ping_check_ERR =
+		std::addressof(new_counter_family.Add({{ "id", "MySQL_Monitor_ping_check_ERR" }}));
+	this->metrics.p_replication_lag_check_OK =
+		std::addressof(new_counter_family.Add({{ "id", "MySQL_Monitor_read_only_check_OK" }}));
+	this->metrics.p_replication_lag_check_ERR =
+		std::addressof(new_counter_family.Add({{ "id", "MySQL_Monitor_read_only_check_ERR" }}));
 };
 
 MySQL_Monitor::~MySQL_Monitor() {
@@ -3055,6 +3087,7 @@ void * MySQL_Monitor::run() {
 		threads[i]->start(2048,false);
 	}
 	started_threads += num_threads;
+	this->metrics.p_started_threads->Increment(num_threads);
 	pthread_attr_t attr;
 	pthread_attr_init(&attr);
 	pthread_attr_setstacksize (&attr, 2048*1024);
@@ -3104,6 +3137,7 @@ void * MySQL_Monitor::run() {
 				unsigned int threads_min = (unsigned int)mysql_thread___monitor_threads_min;
 				if (old_num_threads < threads_min) {
 					num_threads = threads_min;
+					this->metrics.p_num_threads->Set(static_cast<double>(threads_min));
 					threads= (ConsumerThread **)realloc(threads, sizeof(ConsumerThread *)*num_threads);
 					started_threads += (num_threads - old_num_threads);
 					for (unsigned int i = old_num_threads ; i < num_threads ; i++) {
@@ -3132,6 +3166,7 @@ void * MySQL_Monitor::run() {
 				if (new_threads) {
 					unsigned int old_num_threads = num_threads;
 					num_threads += new_threads;
+					this->metrics.p_num_threads->Increment(num_threads);
 					threads= (ConsumerThread **)realloc(threads, sizeof(ConsumerThread *)*num_threads);
 					started_threads += new_threads;
 					for (unsigned int i = old_num_threads ; i < num_threads ; i++) {