From 989687c68aa77941426e0032ec9260963b2e1824 Mon Sep 17 00:00:00 2001 From: Rahim Kanji Date: Mon, 30 Oct 2023 13:08:24 +0500 Subject: [PATCH] Implemented memory usage monitoring for the prepared statement cache with the following changes: * Added two new variables to the 'stats_memory_metrics' table: prepare_statement_metadata_memory prepare_statement_backend_memory * Introduced corresponding variables in the Prometheus exporter to measure memory usage of prepare statement metadata and prepare statement backend. --- include/MySQL_PreparedStatement.h | 3 + include/proxysql_admin.h | 2 + lib/MySQL_PreparedStatement.cpp | 56 +++++++ lib/MySQL_Session.cpp | 1 + lib/MySQL_Thread.cpp | 2 +- lib/ProxySQL_Admin.cpp | 36 +++++ .../test_prepare_statement_memory_usage-t.cpp | 151 ++++++++++++++++++ 7 files changed, 250 insertions(+), 1 deletion(-) create mode 100644 test/tap/tests/test_prepare_statement_memory_usage-t.cpp diff --git a/include/MySQL_PreparedStatement.h b/include/MySQL_PreparedStatement.h index 344eb945e2..af3776fcb3 100644 --- a/include/MySQL_PreparedStatement.h +++ b/include/MySQL_PreparedStatement.h @@ -60,6 +60,7 @@ class MySQL_STMT_Global_info { uint16_t warning_count; MYSQL_FIELD **fields; char* first_comment; + uint64_t total_mem_usage; // struct { // int cache_ttl; // int timeout; @@ -70,6 +71,7 @@ class MySQL_STMT_Global_info { MySQL_STMT_Global_info(uint64_t id, char *u, char *s, char *q, unsigned int ql, char *fc, MYSQL_STMT *stmt, uint64_t _h); void update_metadata(MYSQL_STMT *stmt); ~MySQL_STMT_Global_info(); + void calculate_mem_usage(); }; @@ -264,6 +266,7 @@ class MySQL_STMT_Manager_v14 { MySQL_STMT_Global_info * add_prepared_statement(char *u, char *s, char *q, unsigned int ql, char *fc, MYSQL_STMT *stmt, bool lock=true); void get_metrics(uint64_t *c_unique, uint64_t *c_total, uint64_t *stmt_max_stmt_id, uint64_t *cached, uint64_t *s_unique, uint64_t *s_total); SQLite3_result * get_prepared_statements_global_infos(); + void get_memory_usage(uint64_t& prep_stmt_metadata_mem_usage, uint64_t& prep_stmt_backend_mem_usage); }; #endif /* CLASS_MYSQL_PREPARED_STATEMENT_H */ diff --git a/include/proxysql_admin.h b/include/proxysql_admin.h index 12037861cf..644d351b40 100644 --- a/include/proxysql_admin.h +++ b/include/proxysql_admin.h @@ -79,6 +79,8 @@ struct p_admin_gauge { stack_memory_mysql_threads, stack_memory_admin_threads, stack_memory_cluster_threads, + prepare_stmt_metadata_memory_bytes, + prepare_stmt_backend_memory_bytes, // stmt metrics stmt_client_active_total, stmt_client_active_unique, diff --git a/lib/MySQL_PreparedStatement.cpp b/lib/MySQL_PreparedStatement.cpp index f98eb3ea1e..f3d82ea084 100644 --- a/lib/MySQL_PreparedStatement.cpp +++ b/lib/MySQL_PreparedStatement.cpp @@ -140,6 +140,7 @@ MySQL_STMT_Global_info::MySQL_STMT_Global_info(uint64_t id, char *fc, MYSQL_STMT *stmt, uint64_t _h) { pthread_rwlock_init(&rwlock_, NULL); + total_mem_usage = 0; statement_id = id; ref_count_client = 0; ref_count_server = 0; @@ -294,6 +295,33 @@ MySQL_STMT_Global_info::MySQL_STMT_Global_info(uint64_t id, memset(params[i], 0, sizeof(MYSQL_BIND)); } } + + calculate_mem_usage(); +} + +void MySQL_STMT_Global_info::calculate_mem_usage() { + total_mem_usage = sizeof(MySQL_STMT_Global_info) + + (num_params * (sizeof(MYSQL_BIND) + sizeof(MYSQL_BIND*))) + + (num_columns * (sizeof(MYSQL_FIELD) + sizeof(MYSQL_FIELD*))) + + query_length + 1;// + + //(ref_count_client * 24) + + //(ref_count_server * 24); + + if (username) total_mem_usage += strlen(username) + 1; + if (schemaname) total_mem_usage += strlen(schemaname) + 1; + if (first_comment) total_mem_usage += strlen(first_comment) + 1; + if (digest_text) total_mem_usage += strlen(digest_text) + 1; + + for (uint16_t i = 0; i < num_columns; i++) { + const MYSQL_FIELD* fd = fields[i]; + if (fd->name) total_mem_usage += strlen(fd->name) + 1; + if (fd->org_name) total_mem_usage += strlen(fd->org_name) + 1; + if (fd->table) total_mem_usage += strlen(fd->table) + 1; + if (fd->org_table) total_mem_usage += strlen(fd->org_table) + 1; + if (fd->db) total_mem_usage += strlen(fd->db) + 1; + if (fd->catalog) total_mem_usage += strlen(fd->catalog) + 1; + if (fd->def) total_mem_usage += strlen(fd->def) + 1; + } } void MySQL_STMT_Global_info::update_metadata(MYSQL_STMT *stmt) { @@ -478,6 +506,7 @@ void MySQL_STMT_Global_info::update_metadata(MYSQL_STMT *stmt) { } } // till here is copied from constructor + calculate_mem_usage(); } pthread_rwlock_unlock(&rwlock_); } @@ -892,6 +921,33 @@ MySQL_STMT_Global_info *MySQL_STMT_Manager_v14::add_prepared_statement( return ret; } + +void MySQL_STMT_Manager_v14::get_memory_usage(uint64_t& prep_stmt_metadata_mem_usage, uint64_t& prep_stmt_backend_mem_usage) { + prep_stmt_backend_mem_usage = 0; + prep_stmt_metadata_mem_usage = sizeof(MySQL_STMT_Manager_v14); + rdlock(); + prep_stmt_metadata_mem_usage += map_stmt_id_to_info.size() * (sizeof(uint64_t) + sizeof(MySQL_STMT_Global_info*)); + prep_stmt_metadata_mem_usage += map_stmt_hash_to_info.size() * (sizeof(uint64_t) + sizeof(MySQL_STMT_Global_info*)); + prep_stmt_metadata_mem_usage += free_stmt_ids.size() * (sizeof(uint64_t)); + for (const auto& keyval : map_stmt_id_to_info) { + const MySQL_STMT_Global_info* stmt_global_info = keyval.second; + prep_stmt_metadata_mem_usage += stmt_global_info->total_mem_usage; + prep_stmt_metadata_mem_usage += stmt_global_info->ref_count_server * + ((stmt_global_info->num_params * sizeof(MYSQL_BIND)) + + (stmt_global_info->num_columns * sizeof(MYSQL_FIELD))) + 16; + prep_stmt_metadata_mem_usage += stmt_global_info->ref_count_client * + ((stmt_global_info->num_params * sizeof(MYSQL_BIND)) + + (stmt_global_info->num_columns * sizeof(MYSQL_FIELD))) + 16; + + // backend + prep_stmt_backend_mem_usage += stmt_global_info->ref_count_server * (sizeof(MYSQL_STMT) + + 56 + //sizeof(MADB_STMT_EXTENSION) + (stmt_global_info->num_params * sizeof(MYSQL_BIND)) + + (stmt_global_info->num_columns * sizeof(MYSQL_BIND))); + } + unlock(); +} + void MySQL_STMT_Manager_v14::get_metrics(uint64_t *c_unique, uint64_t *c_total, uint64_t *stmt_max_stmt_id, uint64_t *cached, uint64_t *s_unique, uint64_t *s_total) { diff --git a/lib/MySQL_Session.cpp b/lib/MySQL_Session.cpp index b65cc171c6..7a298ba5fa 100644 --- a/lib/MySQL_Session.cpp +++ b/lib/MySQL_Session.cpp @@ -4326,6 +4326,7 @@ bool MySQL_Session::handler_rc0_PROCESSING_STMT_PREPARE(enum session_status& st, stmt_info->digest_text=strdup(CurrentQuery.QueryParserArgs.digest_text); stmt_info->digest=CurrentQuery.QueryParserArgs.digest; // copy digest stmt_info->MyComQueryCmd=CurrentQuery.MyComQueryCmd; // copy MyComQueryCmd + stmt_info->calculate_mem_usage(); } } global_stmtid=stmt_info->statement_id; diff --git a/lib/MySQL_Thread.cpp b/lib/MySQL_Thread.cpp index 5d6e470beb..9ca69501e2 100644 --- a/lib/MySQL_Thread.cpp +++ b/lib/MySQL_Thread.cpp @@ -2280,7 +2280,7 @@ char ** MySQL_Threads_Handler::get_variables_list() { VariablesPointers_int["max_allowed_packet"] = make_tuple(&variables.max_allowed_packet, 8192, 1024*1024*1024, false); VariablesPointers_int["max_connections"] = make_tuple(&variables.max_connections, 1, 1000*1000, false); VariablesPointers_int["max_stmts_per_connection"] = make_tuple(&variables.max_stmts_per_connection, 1, 1024, false); - VariablesPointers_int["max_stmts_cache"] = make_tuple(&variables.max_stmts_cache, 1024, 1024*1024, false); + VariablesPointers_int["max_stmts_cache"] = make_tuple(&variables.max_stmts_cache, 128, 1024*1024, false); VariablesPointers_int["max_transaction_idle_time"] = make_tuple(&variables.max_transaction_idle_time, 1000, 20*24*3600*1000, false); VariablesPointers_int["max_transaction_time"] = make_tuple(&variables.max_transaction_time, 1000, 20*24*3600*1000, false); VariablesPointers_int["query_cache_size_mb"] = make_tuple(&variables.query_cache_size_MB, 0, 1024*10240, false); diff --git a/lib/ProxySQL_Admin.cpp b/lib/ProxySQL_Admin.cpp index a420b3e1e5..efae6b6043 100644 --- a/lib/ProxySQL_Admin.cpp +++ b/lib/ProxySQL_Admin.cpp @@ -910,6 +910,18 @@ admin_metrics_map = std::make_tuple( "This is the number of global prepared statements for which proxysql has metadata.", metric_tags {} ), + std::make_tuple( + p_admin_gauge::prepare_stmt_metadata_memory_bytes, + "prepare_stmt_metadata_memory_bytes", + "Memory used to store meta data related to prepare statements.", + metric_tags{} + ), + std::make_tuple( + p_admin_gauge::prepare_stmt_backend_memory_bytes, + "prepare_stmt_backend_memory_bytes", + "Memory used by backend server related to prepare statements.", + metric_tags{} + ), std::make_tuple ( p_admin_gauge::fds_in_use, "proxysql_fds_in_use", @@ -8979,6 +8991,13 @@ void ProxySQL_Admin::p_stats___memory_metrics() { __sync_fetch_and_add(&GloVars.statuses.stack_memory_cluster_threads, 0); this->metrics.p_gauge_array[p_admin_gauge::stack_memory_cluster_threads]->Set(stack_memory_cluster_threads); + // proxysql_prepare_statement_memory metric + uint64_t prepare_stmt_metadata_mem_used; + uint64_t prepare_stmt_backend_mem_used; + GloMyStmt->get_memory_usage(prepare_stmt_metadata_mem_used, prepare_stmt_backend_mem_used); + this->metrics.p_gauge_array[p_admin_gauge::prepare_stmt_metadata_memory_bytes]->Set(prepare_stmt_metadata_mem_used); + this->metrics.p_gauge_array[p_admin_gauge::prepare_stmt_backend_memory_bytes]->Set(prepare_stmt_backend_mem_used); + // Update opened file descriptors int32_t cur_fds = get_open_fds(); if (cur_fds != -1) { @@ -9093,6 +9112,23 @@ void ProxySQL_Admin::stats___memory_metrics() { statsdb->execute(query); free(query); } + if (GloMyStmt) { + uint64_t prep_stmt_metadata_mem_usage; + uint64_t prep_stmt_backend_mem_usage; + GloMyStmt->get_memory_usage(prep_stmt_metadata_mem_usage, prep_stmt_backend_mem_usage); + vn = (char*)"prepare_statement_metadata_memory"; + sprintf(bu, "%llu", prep_stmt_metadata_mem_usage); + query=(char*)malloc(strlen(a)+strlen(vn)+strlen(bu)+16); + sprintf(query, a, vn, bu); + statsdb->execute(query); + free(query); + vn = (char*)"prepare_statement_backend_memory"; + sprintf(bu, "%llu", prep_stmt_backend_mem_usage); + query=(char*)malloc(strlen(a)+strlen(vn)+strlen(bu)+16); + sprintf(query, a, vn, bu); + statsdb->execute(query); + free(query); + } if (GloQPro) { unsigned long long mu = 0; mu = GloQPro->get_mysql_firewall_memory_users_table(); diff --git a/test/tap/tests/test_prepare_statement_memory_usage-t.cpp b/test/tap/tests/test_prepare_statement_memory_usage-t.cpp new file mode 100644 index 0000000000..9b2d48dbee --- /dev/null +++ b/test/tap/tests/test_prepare_statement_memory_usage-t.cpp @@ -0,0 +1,151 @@ +/** + * @file test_prepare_statement_memory_usage-t.cpp + * @brief Examines the memory consumption of the prepared statement cache.. + * @details This test assesses the memory utilization of prepared statement metadata/backend cache memory. + */ + +#include +#include "mysql.h" +#include "tap.h" +#include "command_line.h" +#include "proxysql_utils.h" +#include "utils.h" + +enum ComparisonOperator { + kEqual = 0x00000001, + kGreaterThan = 0x00000002, + kLessThan = 0x00000004 +}; + +int get_prepare_stmt_mem_usage(MYSQL* admin, uint64_t& prep_stmt_metadata_mem, uint64_t& prep_stmt_backend_mem) { + prep_stmt_metadata_mem = prep_stmt_backend_mem = 0; + MYSQL_QUERY_T(admin, "SELECT variable_name, variable_value FROM stats_memory_metrics WHERE \ + variable_name IN ('prepare_statement_metadata_memory', 'prepare_statement_backend_memory')"); + MYSQL_RES* myres = mysql_store_result(admin); + while (MYSQL_ROW myrow = mysql_fetch_row(myres)) { + if (strncmp(myrow[0], "prepare_statement_metadata_memory", sizeof("prepare_statement_metadata_memory") - 1) == 0) { + prep_stmt_metadata_mem = std::stoull(myrow[1], nullptr, 10); + } else if (strncmp(myrow[0], "prepare_statement_backend_memory", sizeof("prepare_statement_backend_memory") - 1) == 0) { + prep_stmt_backend_mem = std::stoull(myrow[1], nullptr, 10); + } else { + fprintf(stderr, "File %s, line %d, Error: %s\n", __FILE__, __LINE__, "Invalid resultset"); + mysql_free_result(myres); + return EXIT_FAILURE; + } + } + mysql_free_result(myres); + return EXIT_SUCCESS; +} + +int check_prepare_statement_mem_usage(MYSQL* proxysql_admin, MYSQL* proxysql, const char* query, int prep_stmt_metadata_mem_comp, + int prep_stmt_backend_mem_comp) { + uint64_t old_prep_stmt_metadata_mem, old_prep_stmt_backend_mem; + if (get_prepare_stmt_mem_usage(proxysql_admin, old_prep_stmt_metadata_mem, old_prep_stmt_backend_mem) == EXIT_FAILURE) { + return EXIT_FAILURE; + } + MYSQL_STMT* stmt = mysql_stmt_init(proxysql); + if (!stmt) { + diag("mysql_stmt_init(), out of memory\n"); + return EXIT_FAILURE; + } + if (mysql_stmt_prepare(stmt, query, strlen(query))) { + diag("query: %s", query); + diag("mysql_stmt_prepare at line %d failed: %s", __LINE__, mysql_error(proxysql)); + mysql_stmt_close(stmt); + return EXIT_FAILURE; + } else { + ok(true, "Prepare succeeded: %s", query); + } + uint64_t new_prep_stmt_metadata_mem, new_prep_stmt_backend_mem; + if (get_prepare_stmt_mem_usage(proxysql_admin, new_prep_stmt_metadata_mem, new_prep_stmt_backend_mem) == EXIT_FAILURE) { + mysql_stmt_close(stmt); + return EXIT_FAILURE; + } + auto fnCompare = [](const uint64_t& val1, const uint64_t& val2, int co) -> bool { + bool res = false; + if ((co & kLessThan) == kLessThan) { + if ((co & kEqual) == kEqual) { + res = (val1 >= val2); + } else { + res = (val1 > val2); + } + } else if ((co & kGreaterThan) == kGreaterThan) { + if ((co & kEqual) == kEqual) { + res = (val1 <= val2); + } else { + res = (val1 < val2); + } + } else { + res = (val1 == val2); + } + return res; + }; + + ok(fnCompare(old_prep_stmt_metadata_mem, new_prep_stmt_metadata_mem, prep_stmt_metadata_mem_comp), + "Memory usage check [%d]. 'prepare_statement_metadata_memory':[%lu] [%lu]", prep_stmt_metadata_mem_comp, + old_prep_stmt_metadata_mem, new_prep_stmt_metadata_mem); + + ok(fnCompare(old_prep_stmt_backend_mem, new_prep_stmt_backend_mem, prep_stmt_backend_mem_comp), + "Memory usage check [%d]. 'prepare_statement_backend_memory':[%lu] [%lu]", prep_stmt_backend_mem_comp, + old_prep_stmt_backend_mem, new_prep_stmt_backend_mem); + + mysql_stmt_close(stmt); + return EXIT_SUCCESS; +} + +int main(int argc, char** argv) { + + CommandLine cl; + + if (cl.getEnv()) { + diag("Failed to get the required environmental variables."); + return -1; + } + + plan(4 * // query + 3 // checks + ); + + // Initialize Admin connection + MYSQL* proxysql_admin = mysql_init(NULL); + if (!proxysql_admin) { + fprintf(stderr, "File %s, line %d, Error: %s\n", __FILE__, __LINE__, mysql_error(proxysql_admin)); + return -1; + } + // Connnect to ProxySQL Admin + if (!mysql_real_connect(proxysql_admin, cl.host, cl.admin_username, cl.admin_password, NULL, cl.admin_port, NULL, 0)) { + fprintf(stderr, "File %s, line %d, Error: %s\n", __FILE__, __LINE__, mysql_error(proxysql_admin)); + return exit_status(); + } + + // Initialize ProxySQL connection + MYSQL* proxysql = mysql_init(NULL); + if (!proxysql) { + fprintf(stderr, "File %s, line %d, Error: %s\n", __FILE__, __LINE__, mysql_error(proxysql)); + return exit_status(); + } + + // Connect to ProxySQL + if (!mysql_real_connect(proxysql, cl.host, cl.username, cl.password, NULL, cl.port, NULL, 0)) { + fprintf(stderr, "File %s, line %d, Error: %s\n", __FILE__, __LINE__, mysql_error(proxysql)); + return exit_status(); + } + + if (check_prepare_statement_mem_usage(proxysql_admin, proxysql, "SELECT 1", kGreaterThan, (kGreaterThan | kEqual)) == EXIT_FAILURE) + goto __cleanup; + + if (check_prepare_statement_mem_usage(proxysql_admin, proxysql, "SELECT 2", kGreaterThan, (kGreaterThan | kEqual)) == EXIT_FAILURE) + goto __cleanup; + + if (check_prepare_statement_mem_usage(proxysql_admin, proxysql, "SELECT 1", kGreaterThan, kEqual) == EXIT_FAILURE) + goto __cleanup; + + if (check_prepare_statement_mem_usage(proxysql_admin, proxysql, "SELECT 2", kGreaterThan, kEqual) == EXIT_FAILURE) + goto __cleanup; + +__cleanup: + mysql_close(proxysql); + mysql_close(proxysql_admin); + + return exit_status(); +}