From 9b141032363eecc6f54a8353227d61f23b565a3d Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Ren=C3=A9=20Canna=C3=B2?= Date: Sat, 19 Jan 2019 17:50:30 +1100 Subject: [PATCH] Collect memory statistics about query rules --- .gitignore | 3 ++ include/query_processor.h | 2 ++ include/sqlite3db.h | 25 +++++++++++++++++ lib/ProxySQL_Admin.cpp | 12 ++++++++ lib/Query_Processor.cpp | 58 +++++++++++++++++++++++++++++++++++++++ 5 files changed, 100 insertions(+) diff --git a/.gitignore b/.gitignore index ee65c0856d..78eb4275e3 100644 --- a/.gitignore +++ b/.gitignore @@ -50,6 +50,9 @@ binaries/*deb binaries/*rpm tools/eventslog_reader_sample +proxysql-2.0.0/ +docker/images/proxysql/rhel-compliant/rpmmacros + #core core diff --git a/include/query_processor.h b/include/query_processor.h index 4c16a7edf6..f6d16cd0a4 100644 --- a/include/query_processor.h +++ b/include/query_processor.h @@ -200,6 +200,7 @@ class Query_Processor { std::unordered_map rules_fast_routing; Command_Counter * commands_counters[MYSQL_COM_QUERY___NONE]; volatile unsigned int version; + unsigned long long rules_mem_used; public: Query_Processor(); ~Query_Processor(); @@ -239,6 +240,7 @@ class Query_Processor { SQLite3_result * get_query_digests_reset(); unsigned long get_query_digests_total_size(); + unsigned long long get_rules_mem_used(); // fast routing diff --git a/include/sqlite3db.h b/include/sqlite3db.h index fbe3288a74..adbe92d2d3 100644 --- a/include/sqlite3db.h +++ b/include/sqlite3db.h @@ -7,6 +7,7 @@ class SQLite3_row { public: int cnt; + int ds; int *sizes; char **fields; char *data; @@ -16,6 +17,14 @@ class SQLite3_row { memset(fields,0,sizeof(char *)*c); cnt=c; data=NULL; + ds=0; + }; + unsigned long long get_size() { + unsigned long long s = sizeof(SQLite3_row); + s += cnt * sizeof(int); + s += cnt * sizeof(char *); + s += ds; + return s; }; ~SQLite3_row() { free(fields); @@ -57,6 +66,7 @@ class SQLite3_row { data_ptr++; // leading 0 } } + ds=data_size; }; void add_fields(char **_fields) { int i; @@ -85,6 +95,7 @@ class SQLite3_row { fields[i]=NULL; } } + ds=data_size; }; }; @@ -117,6 +128,20 @@ class SQLite3_result { SQLite3_result() { columns=0; }; + unsigned long long get_size() { + unsigned long long s = sizeof(SQLite3_result); + s += column_definition.size() * sizeof(SQLite3_column *); + s += rows.size() * sizeof(SQLite3_row *); + for (std::vector::iterator it = column_definition.begin() ; it != column_definition.end(); ++it) { + SQLite3_column *r=*it; + s+= sizeof(SQLite3_column) + strlen(r->name); + } + for (std::vector::iterator it = rows.begin() ; it != rows.end(); ++it) { + SQLite3_row *r=*it; + s += r->get_size(); + } + return s; + }; void add_column_definition(int a, const char *b) { SQLite3_column *cf=new SQLite3_column(a,b); column_definition.push_back(cf); diff --git a/lib/ProxySQL_Admin.cpp b/lib/ProxySQL_Admin.cpp index af3d1f14b0..9e3f4eb92e 100644 --- a/lib/ProxySQL_Admin.cpp +++ b/lib/ProxySQL_Admin.cpp @@ -5520,6 +5520,18 @@ void ProxySQL_Admin::stats___memory_metrics() { statsdb->execute(query); free(query); } + if (GloQPro) { + unsigned long long mu = GloQPro->get_rules_mem_used(); + if (GloMTH) { + mu += mu * GloMTH->num_threads; + } + vn=(char *)"mysql_query_rules_memory"; + sprintf(bu,"%llu",mu); + query=(char *)malloc(strlen(a)+strlen(vn)+strlen(bu)+16); + sprintf(query,a,vn,bu); + statsdb->execute(query); + free(query); + } } { unsigned long mu; diff --git a/lib/Query_Processor.cpp b/lib/Query_Processor.cpp index 85529ab732..e053800f16 100644 --- a/lib/Query_Processor.cpp +++ b/lib/Query_Processor.cpp @@ -245,6 +245,39 @@ typedef struct __RE2_objects_t re2_t; static bool rules_sort_comp_function (QP_rule_t * a, QP_rule_t * b) { return (a->rule_id < b->rule_id); } + +static unsigned long long mem_used_rule(QP_rule_t *qr) { + unsigned long long s = sizeof(QP_rule_t); + if (qr->username) + s+=strlen(qr->username); + if (qr->schemaname) + s+=strlen(qr->schemaname); + if (qr->client_addr) + s+=strlen(qr->client_addr); + if (qr->proxy_addr) + s+=strlen(qr->proxy_addr); + if (qr->match_digest) + s+=strlen(qr->match_digest)*10; // not sure how much is used for regex + if (qr->match_pattern) + s+=strlen(qr->match_pattern)*10; // not sure how much is used for regex + if (qr->replace_pattern) + s+=strlen(qr->replace_pattern)*10; // not sure how much is used for regex + if (qr->error_msg) + s+=strlen(qr->error_msg); + if (qr->OK_msg) + s+=strlen(qr->OK_msg); + if (qr->comment) + s+=strlen(qr->comment); + if (qr->match_digest || qr->match_pattern || qr->replace_pattern) { + s+= sizeof(__RE2_objects_t *)+sizeof(__RE2_objects_t); + s+= sizeof(pcrecpp::RE_Options *) + sizeof(pcrecpp::RE_Options); + s+= sizeof(pcrecpp::RE *) + sizeof(pcrecpp::RE); + s+= sizeof(re2::RE2::Options *) + sizeof(re2::RE2::Options); + s+= sizeof(RE2 *) + sizeof(RE2); + } + return s; +} + static re2_t * compile_query_rule(QP_rule_t *qr, int i) { re2_t *r=(re2_t *)malloc(sizeof(re2_t)); r->opt1=NULL; @@ -346,6 +379,7 @@ Query_Processor::Query_Processor() { spinlock_rwlock_init(&digest_rwlock); #endif version=0; + rules_mem_used=0; for (int i=0; irule_id=rule_id; @@ -577,6 +619,7 @@ void Query_Processor::reset_all(bool lock) { #else spin_wrunlock(&rwlock); #endif + rules_mem_used=0; }; bool Query_Processor::insert(QP_rule_t *qr, bool lock) { @@ -588,6 +631,7 @@ bool Query_Processor::insert(QP_rule_t *qr, bool lock) { spin_wrlock(&rwlock); #endif rules.push_back(qr); + rules_mem_used += mem_used_rule(qr); if (lock) #ifdef PROXYSQL_QPRO_PTHREAD_MUTEX pthread_rwlock_unlock(&rwlock); @@ -1935,7 +1979,21 @@ void Query_Processor::load_fast_routing(SQLite3_result *resultset) { s.append(r->fields[2]); int destination_hostgroup = atoi(r->fields[3]); rules_fast_routing[s] = destination_hostgroup; + rules_mem_used += s.length() + sizeof(int); + } + { + size_t count = 0; + for (unsigned i = 0; i < rules_fast_routing.bucket_count(); ++i) { + size_t bucket_size = rules_fast_routing.bucket_size(i); + if (bucket_size == 0) { + count++; + } else { + count += bucket_size; + } + } + rules_mem_used += count; } delete fast_routing_resultset; fast_routing_resultset = resultset; // save it + rules_mem_used += fast_routing_resultset->get_size(); };