From a67db17709b53ba331f1ad732283b8ac9d2b0547 Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Javier=20Jaramago=20Fern=C3=A1ndez?= Date: Wed, 10 Aug 2022 21:55:32 +0200 Subject: [PATCH 1/7] Fix current 'auto_increment_delay_multiplex_timeout_ms' behavior #3923 --- include/MySQL_Session.h | 17 ++++++++++++- include/mysql_connection.h | 6 ++++- lib/MySQL_Session.cpp | 52 ++++++++++++++++++++++++++++++++++++++ lib/MySQL_Thread.cpp | 24 ++++++++++++++---- lib/mysql_connection.cpp | 31 +++++++++++++++++++++-- 5 files changed, 121 insertions(+), 9 deletions(-) diff --git a/include/MySQL_Session.h b/include/MySQL_Session.h index 15c826fc93..5f65a38cea 100644 --- a/include/MySQL_Session.h +++ b/include/MySQL_Session.h @@ -1,5 +1,9 @@ #ifndef __CLASS_MYSQL_SESSION_H #define __CLASS_MYSQL_SESSION_H + +#include +#include + #include "proxysql.h" #include "cpp.h" #include "MySQL_Variables.h" @@ -155,7 +159,11 @@ class MySQL_Session void init(); void reset(); void add_ldap_comment_to_pkt(PtrSize_t *); - + /** + * @brief Performs the required housekeeping operations over the session and its connections before + * performing any processing on received client packets. + */ + void housekeeping_before_pkts(); int get_pkts_from_client(bool&, PtrSize_t&); void handler___status_WAITING_CLIENT_DATA___STATE_SLEEP___MYSQL_COM_STMT_RESET(PtrSize_t&); void handler___status_WAITING_CLIENT_DATA___STATE_SLEEP___MYSQL_COM_STMT_CLOSE(PtrSize_t&); @@ -215,6 +223,12 @@ class MySQL_Session PtrArray *mybes; MySQL_Data_Stream *client_myds; MySQL_Data_Stream *server_myds; + /* + * @brief Store the hostgroups that hold connections that have been flagged as 'expired' by the + * maintenance thread. This values will be used to release the retained connection in the specific + * hostgroup in housekeeping operations, before client packet processing. Currently 'housekeeping_before_pkts'. + */ + std::vector hgs_expired_conns {}; char * default_schema; char * user_attributes; @@ -319,6 +333,7 @@ class MySQL_Session void Memory_Stats(); void create_new_session_and_reset_connection(MySQL_Data_Stream *_myds); bool handle_command_query_kill(PtrSize_t *); + void update_expired_conns(const std::vector>&); /** * @brief Performs the final operations after current query has finished to be executed. It updates the session * 'transaction_persistent_hostgroup', and updates the 'MySQL_Data_Stream' and 'MySQL_Connection' before diff --git a/include/mysql_connection.h b/include/mysql_connection.h index 6283e149f8..3b07fc7416 100644 --- a/include/mysql_connection.h +++ b/include/mysql_connection.h @@ -95,6 +95,8 @@ class MySQL_Connection { char scramble_buff[40]; unsigned long long creation_time; unsigned long long last_time_used; + /* @brief Time at which the last 'event' was processed by 'handler' */ + unsigned long long last_event_time; unsigned long long timeout; int auto_increment_delay_token; int fd; @@ -217,11 +219,13 @@ class MySQL_Connection { bool IsServerOffline(); bool IsAutoCommit(); bool AutocommitFalse_AndSavepoint(); - bool MultiplexDisabled(); + bool MultiplexDisabled(bool check_delay_token = true); bool IsKeepMultiplexEnabledVariables(char *query_digest_text); void ProcessQueryAndSetStatusFlags(char *query_digest_text); void optimize(); void close_mysql(); + uint64_t idle_time(uint64_t curtime); + bool expire_auto_increment_delay(uint64_t curtime, uint64_t timeout); void set_is_client(); // used for local_stmts diff --git a/lib/MySQL_Session.cpp b/lib/MySQL_Session.cpp index e4c99625e9..fd0b1de22c 100644 --- a/lib/MySQL_Session.cpp +++ b/lib/MySQL_Session.cpp @@ -51,6 +51,8 @@ #define EXPMARIA +using std::function; +using std::vector; static inline char is_digit(char c) { if(c >= '0' && c <= '9') @@ -686,6 +688,28 @@ MySQL_Backend * MySQL_Session::find_backend(int hostgroup_id) { return NULL; // NULL = backend not found }; +void MySQL_Session::update_expired_conns(const vector>& checks) { + for (uint32_t i = 0; i < mybes->len; i++) { + MySQL_Backend* mybe = static_cast(mybes->index(i)); + MySQL_Data_Stream* myds = mybe != nullptr ? mybe->server_myds : nullptr; + MySQL_Connection* myconn = myds != nullptr ? myds->myconn : nullptr; + + if (myconn != nullptr) { + const bool is_active_transaction = myconn->IsActiveTransaction(); + const bool multiplex_disabled = myconn->MultiplexDisabled(false); + + // Make sure the connection is reusable before performing any check + if (myconn->reusable==true && is_active_transaction==false && multiplex_disabled==false) { + for (const function& check : checks) { + if (check(myconn)) { + this->hgs_expired_conns.push_back(mybe->hostgroup_id); + break; + } + } + } + } + } +} MySQL_Backend * MySQL_Session::create_backend(int hostgroup_id, MySQL_Data_Stream *_myds) { MySQL_Backend *_mybe=new MySQL_Backend(); @@ -4374,6 +4398,33 @@ void MySQL_Session::handler___status_WAITING_CLIENT_DATA() { } } +void MySQL_Session::housekeeping_before_pkts() { + if (mysql_thread___multiplexing) { + for (const int hg_id : hgs_expired_conns) { + MySQL_Backend* mybe = find_backend(hg_id); + + if (mybe != nullptr) { + MySQL_Data_Stream* myds = mybe->server_myds; + + if (mysql_thread___autocommit_false_not_reusable && myds->myconn->IsAutoCommit()==false) { + if (mysql_thread___reset_connection_algorithm == 2) { + create_new_session_and_reset_connection(myds); + } else { + myds->destroy_MySQL_Connection_From_Pool(true); + } + } else { + myds->return_MySQL_Connection_To_Pool(); + } + } + } + // We are required to perform a cleanup after consuming the elements, thus preventing any subsequent + // 'handler' call to perform recomputing of the already processed elements. + if (hgs_expired_conns.empty() == false) { + hgs_expired_conns.clear(); + } + } +} + // this function was inline void MySQL_Session::handler_rc0_Process_GTID(MySQL_Connection *myconn) { if (myconn->get_gtid(mybe->gtid_uuid,&mybe->gtid_trxid)) { @@ -4426,6 +4477,7 @@ int MySQL_Session::handler() { } } + housekeeping_before_pkts(); handler_ret = get_pkts_from_client(wrong_pass, pkt); if (handler_ret != 0) { return handler_ret; diff --git a/lib/MySQL_Thread.cpp b/lib/MySQL_Thread.cpp index c95d756413..d38013c227 100644 --- a/lib/MySQL_Thread.cpp +++ b/lib/MySQL_Thread.cpp @@ -1,4 +1,8 @@ //#define __CLASS_STANDARD_MYSQL_THREAD_H + +#include +#include + #include "MySQL_HostGroups_Manager.h" #include "prometheus_helpers.h" #define MYSQL_THREAD_IMPLEMENTATION @@ -17,6 +21,9 @@ #include "MySQL_PreparedStatement.h" #include "MySQL_Logger.hpp" +using std::vector; +using std::function; + #ifdef DEBUG MySQL_Session *sess_stopat; #endif @@ -3733,12 +3740,19 @@ void MySQL_Thread::ProcessAllSessions_MaintenanceLoop(MySQL_Session *sess, unsig } } - if (sess->mybe && sess->mybe->server_myds && sess->mybe->server_myds->myconn) { - MySQL_Connection* myconn = sess->mybe->server_myds->myconn; + // Perform the maintenance of expired 'auto_increment_delay_multiplex' for connections on the session + if (mysql_thread___multiplexing) { + const auto auto_incr_delay_multiplex_check = [curtime=this->curtime] (MySQL_Connection* myconn) -> bool { + const uint64_t multiplex_timeout_us = static_cast(mysql_thread___auto_increment_delay_multiplex_timeout_ms) * 1000; + const bool timeout_expired = multiplex_timeout_us != 0 && myconn->expire_auto_increment_delay(curtime, multiplex_timeout_us); + return timeout_expired; + }; - if (mysql_thread___auto_increment_delay_multiplex_timeout_ms != 0 && (sess_time/1000 > (unsigned long long)mysql_thread___auto_increment_delay_multiplex_timeout_ms)) { - myconn->auto_increment_delay_token = 0; - } + const vector> expire_conn_checks { + auto_incr_delay_multiplex_check + }; + + sess->update_expired_conns(expire_conn_checks); } } diff --git a/lib/mysql_connection.cpp b/lib/mysql_connection.cpp index c5b25b2e40..ff505ba729 100644 --- a/lib/mysql_connection.cpp +++ b/lib/mysql_connection.cpp @@ -560,6 +560,32 @@ bool MySQL_Connection::get_status(uint32_t status_flag) { return this->status_flags & status_flag; } +uint64_t MySQL_Connection::idle_time(uint64_t curtime) { + // ASYNC_QUERY_END not required due to being transient state + const bool is_idle = this->async_state_machine == ASYNC_IDLE; + + if (is_idle) { + return curtime - this->last_event_time; + } else { + return 0; + } +} + +bool MySQL_Connection::expire_auto_increment_delay(uint64_t curtime, uint64_t timeout) { + if (timeout == 0) { + return false; + } + + uint64_t idle_time = this->idle_time(curtime); + + if (idle_time > timeout) { + this->auto_increment_delay_token = 0; + return true; + } else { + return false; + } +} + void MySQL_Connection::set_status_sql_log_bin0(bool v) { if (v) { status_flags |= STATUS_MYSQL_CONNECTION_SQL_LOG_BIN0; @@ -1013,6 +1039,7 @@ void MySQL_Connection::set_is_client() { #define NEXT_IMMEDIATE(new_st) do { async_state_machine = new_st; goto handler_again; } while (0) MDB_ASYNC_ST MySQL_Connection::handler(short event) { + this->last_event_time = myds->sess->thread->curtime; unsigned long long processed_bytes=0; // issue #527 : this variable will store the amount of bytes processed during this event if (mysql==NULL) { // it is the first time handler() is being called @@ -2321,14 +2348,14 @@ bool MySQL_Connection::IsAutoCommit() { return ret; } -bool MySQL_Connection::MultiplexDisabled() { +bool MySQL_Connection::MultiplexDisabled(bool check_delay_token) { // status_flags stores information about the status of the connection // can be used to determine if multiplexing can be enabled or not bool ret=false; if (status_flags & (STATUS_MYSQL_CONNECTION_TRANSACTION|STATUS_MYSQL_CONNECTION_USER_VARIABLE|STATUS_MYSQL_CONNECTION_PREPARED_STATEMENT|STATUS_MYSQL_CONNECTION_LOCK_TABLES|STATUS_MYSQL_CONNECTION_TEMPORARY_TABLE|STATUS_MYSQL_CONNECTION_GET_LOCK|STATUS_MYSQL_CONNECTION_NO_MULTIPLEX|STATUS_MYSQL_CONNECTION_SQL_LOG_BIN0|STATUS_MYSQL_CONNECTION_FOUND_ROWS|STATUS_MYSQL_CONNECTION_HAS_SAVEPOINT) ) { ret=true; } - if (auto_increment_delay_token) return true; + if (check_delay_token && auto_increment_delay_token) return true; return ret; } From e12f0c71734d91cfd67c1947fa96289a1d4d86cb Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Javier=20Jaramago=20Fern=C3=A1ndez?= Date: Wed, 10 Aug 2022 21:57:24 +0200 Subject: [PATCH 2/7] Fix test to match new 'auto_increment_delay_multiplex_timeout_ms' behavior #3923 --- .../test_auto_increment_delay_multiplex-t.cpp | 167 +++++++++++++++++- 1 file changed, 163 insertions(+), 4 deletions(-) diff --git a/test/tap/tests/test_auto_increment_delay_multiplex-t.cpp b/test/tap/tests/test_auto_increment_delay_multiplex-t.cpp index b995ee6067..8f3c93e21a 100644 --- a/test/tap/tests/test_auto_increment_delay_multiplex-t.cpp +++ b/test/tap/tests/test_auto_increment_delay_multiplex-t.cpp @@ -7,7 +7,10 @@ * are present. * 2. 'auto_increment_delay_multiplex' behaves properly for different values. * 3. 'auto_increment_delay_multiplex_timeout_ms' behaves properly for different values. - * 4. 'auto_increment_delay_multiplex_timeout_ms' behaves properly for value '0' (disabled). + * 4. 'auto_increment_delay_multiplex_timeout_ms' should be delayed by queries in the same hostgroup. + * 5. 'auto_increment_delay_multiplex_timeout_ms' should not take effect on transactions. + * 6. 'auto_increment_delay_multiplex_timeout_ms' should not take effect on multiplex dissabling scenarios. + * Eg: SET statements that create session variables. */ #include @@ -112,7 +115,10 @@ int main(int argc, char** argv) { 1 + // Check variables are present ((VAL_RANGE / STEP) + 1) * 2 + // Tests for different 'auto_increment_delay_multiplex' values (VAL_RANGE / STEP) * 3 + // Tests for different 'auto_increment_delay_multiplex_timeout_ms' values - 3 // Tests for 'auto_increment_delay_multiplex_timeout_ms' zero value + 3 + // Tests for 'auto_increment_delay_multiplex_timeout_ms' zero value + 2 + // Tests for 'auto_increment_delay_multiplex_timeout_ms' keep alive queries + 2 + // Tests for 'auto_increment_delay_multiplex_timeout_ms' transaction behavior + 1 // Tests for 'auto_increment_delay_multiplex_timeout_ms' multiplex disabled by SET statement ); MYSQL* proxy_mysql = mysql_init(NULL); @@ -253,8 +259,8 @@ int main(int argc, char** argv) { ); } else { ok( - 0 == cur_auto_inc_delay_mult, - "'auto_increment_delay_token' val should be '0' after timeout:" + -1 == cur_auto_inc_delay_mult, + "'auto_increment_delay_token' val should be '-1' after timeout:" " { Exp: %d, Act: %d, Timeout: %d, Waited: %d }", 0, cur_auto_inc_delay_mult, auto_inc_delay_to, waited ); @@ -283,6 +289,8 @@ int main(int argc, char** argv) { return EXIT_SUCCESS; }; + const char* insert_query { "INSERT INTO test.auto_inc_multiplex (c2, c3) VALUES ('foo','bar')" }; + // 3. Change and check 'auto_increment_delay_multiplex_timeout_ms' behavior { // Set the default 'mysql-auto_increment_delay_multiplex' since it's no longer relevant @@ -316,6 +324,157 @@ int main(int argc, char** argv) { // Check that value '0' for 'auto_increment_delay_multiplex_timeout_ms' disables the feature int c_res = check_auto_increment_to(proxy_admin, proxy_mysql, f_auto_incr_val, poll_timeout, 0); if (c_res != EXIT_SUCCESS) { return EXIT_FAILURE; } + + // Check that using the connection reset the internal timer, keeping the connection attached + const uint32_t timeout_ms = 2000; + // Impose a big delay so we are sure only 'timeout' is being relevant + const uint32_t delay = 100; + const string timeout_query { + "SET mysql-auto_increment_delay_multiplex_timeout_ms=" + std::to_string(timeout_ms) + }; + const string delay_query { "SET mysql-auto_increment_delay_multiplex=" + std::to_string(delay) }; + poll_timeout = 0; + const string poll_timeout_query { + "SELECT variable_value FROM global_variables WHERE variable_name='mysql-poll_timeout'" + }; + + g_res = get_query_result(proxy_admin, poll_timeout_query.c_str(), poll_timeout); + if (g_res != EXIT_SUCCESS) { return EXIT_FAILURE; } + + MYSQL_QUERY(proxy_admin, delay_query.c_str()); + diag("%s: Executing query `%s`...", tap_curtime().c_str(), delay_query.c_str()); + MYSQL_QUERY(proxy_admin, timeout_query.c_str()); + diag("%s: Executing query `%s`...", tap_curtime().c_str(), timeout_query.c_str()); + MYSQL_QUERY(proxy_admin, "LOAD MYSQL VARIABLES TO RUNTIME"); + + { + // Insert disabling multiplexing for the connection + diag("%s: Executing query `%s`...", tap_curtime().c_str(), insert_query); + MYSQL_QUERY(proxy_mysql, insert_query); + + // Perform queries in the same connection + diag("Execute queries beyond imposed timeout"); + uint32_t waited = 0; + while (waited < timeout_ms * 3) { + sleep(1); + + diag("%s: Executing query `%s`...", tap_curtime().c_str(), "/* hostgroup=0 */ DO 1"); + MYSQL_QUERY(proxy_mysql, "/* hostgroup=0 */ DO 1"); + waited += 1000; + } + + int cur_delay = 0; + int g_res = get_conn_auto_inc_delay_token(proxy_mysql, cur_delay); + if (g_res != EXIT_SUCCESS) { + return EXIT_FAILURE; + } + + uint32_t exp_delay = delay - (waited/1000); + ok( + exp_delay == cur_delay, + "Connection was kept after timeout due to queries issues in the same hostgroup:" + " 'auto_increment_delay_multiplex' - Exp: '%d', Act: '%d'", + exp_delay, cur_delay + ); + + waited = 0; + // Perform queries in other connections + while (waited < timeout_ms * 3) { + sleep(1); + + diag("%s: Executing query `%s`...", tap_curtime().c_str(), "SELECT 1"); + MYSQL_QUERY(proxy_mysql, "SELECT 1"); + mysql_free_result(mysql_store_result(proxy_mysql)); + + waited += 1000; + // After this time the connection should have timeout already + if (waited > timeout_ms + poll_timeout + 500) { + break; + } + } + + cur_delay = 0; + g_res = get_conn_auto_inc_delay_token(proxy_mysql, cur_delay); + if (g_res != EXIT_SUCCESS) { + return EXIT_FAILURE; + } + + ok( + -1 == cur_delay, + "Connection returned to connpool when queries are issued in different hostgroup - Exp: '%d', Act: '%d'", + -1, cur_delay + ); + } + + // Transactions connections should be preserved by 'auto_increment_delay_multiplex_timeout_ms' + { + diag("%s: Executing query `%s`...", tap_curtime().c_str(), "BEGIN"); + MYSQL_QUERY(proxy_mysql, "BEGIN"); + diag("%s: Executing query `%s`...", tap_curtime().c_str(), insert_query); + MYSQL_QUERY(proxy_mysql, insert_query); + + // Wait for the timeout and check the value + diag("%s: Waiting for timeout to expire...", tap_curtime().c_str()); + usleep(timeout_ms * 1000 + poll_timeout * 1000 + 500 * 1000 * 2); + + diag("%s: Extracting current auto inc delay...", tap_curtime().c_str()); + int cur_delay = 0; + int g_res = get_conn_auto_inc_delay_token(proxy_mysql, cur_delay); + if (g_res != EXIT_SUCCESS) { + return EXIT_FAILURE; + } + + ok( + delay == cur_delay, + "Connection should not be returned to conn_pool due to transaction - Exp: '%d', Act: '%d'", + delay, cur_delay + ); + + diag("%s: Executing query `%s`...", tap_curtime().c_str(), "COMMIT"); + MYSQL_QUERY(proxy_mysql, "COMMIT"); + + diag("%s: Waiting for timeout to expire...", tap_curtime().c_str()); + usleep(timeout_ms * 1000 + poll_timeout * 1000 + 500 * 1000 * 2); + + diag("%s: Extracting current auto inc delay...", tap_curtime().c_str()); + cur_delay = 0; + g_res = get_conn_auto_inc_delay_token(proxy_mysql, cur_delay); + if (g_res != EXIT_SUCCESS) { + return EXIT_FAILURE; + } + + ok( + -1 == cur_delay, + "Connection should be returned to conn_pool after 'COMMIT' and 'timeout' wait - Exp: '%d', Act: '%d'", + -1, cur_delay + ); + } + + // Multiplex disabled by any action should take precedence over 'auto_increment_delay_multiplex_timeout_ms' + { + const char* set_query { "SET @local_var='foo'" }; + diag("%s: Executing query `%s`...", tap_curtime().c_str(), set_query); + MYSQL_QUERY(proxy_mysql, "SET @local_var='foo'"); + diag("%s: Executing query `%s`...", tap_curtime().c_str(), insert_query); + MYSQL_QUERY(proxy_mysql, insert_query); + + // Wait for the timeout and check the value + diag("%s: Waiting for timeout to expire...", tap_curtime().c_str()); + usleep(timeout_ms * 1000 + poll_timeout * 1000 + 500 * 1000 * 2); + + diag("%s: Extracting current auto inc delay...", tap_curtime().c_str()); + int cur_delay = 0; + int g_res = get_conn_auto_inc_delay_token(proxy_mysql, cur_delay); + if (g_res != EXIT_SUCCESS) { + return EXIT_FAILURE; + } + + ok( + delay == cur_delay, + "Connection should not be returned to conn_pool due to SET session var - Exp: '%d', Act: '%d'", + delay, cur_delay + ); + } } cleanup: From dbdfec349475255185ae409320a8e027167cee6e Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Javier=20Jaramago=20Fern=C3=A1ndez?= Date: Sat, 13 Aug 2022 09:02:53 +0200 Subject: [PATCH 3/7] Add 'ASYNC_IDLE' precondition for connections considered for expiring If any scenario is found in which we may want to consider a non 'ASYNC_IDLE' connection for expiring, this precondition shall be removed, and checks should be responsible for ensuring 'ASYNC_IDLE' state in the connection. --- lib/MySQL_Session.cpp | 3 ++- 1 file changed, 2 insertions(+), 1 deletion(-) diff --git a/lib/MySQL_Session.cpp b/lib/MySQL_Session.cpp index fd0b1de22c..a33860ed6b 100644 --- a/lib/MySQL_Session.cpp +++ b/lib/MySQL_Session.cpp @@ -697,9 +697,10 @@ void MySQL_Session::update_expired_conns(const vectorIsActiveTransaction(); const bool multiplex_disabled = myconn->MultiplexDisabled(false); + const bool is_idle = myconn->async_state_machine == ASYNC_IDLE; // Make sure the connection is reusable before performing any check - if (myconn->reusable==true && is_active_transaction==false && multiplex_disabled==false) { + if (myconn->reusable==true && is_active_transaction==false && multiplex_disabled==false && is_idle) { for (const function& check : checks) { if (check(myconn)) { this->hgs_expired_conns.push_back(mybe->hostgroup_id); From 583218e9c56ed2494b7b43cff6c2944800ab737b Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Javier=20Jaramago=20Fern=C3=A1ndez?= Date: Sat, 13 Aug 2022 09:15:21 +0200 Subject: [PATCH 4/7] Fix typo in 'MySQL_Session::hgs_expired_conns' doc --- include/MySQL_Session.h | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/include/MySQL_Session.h b/include/MySQL_Session.h index 5f65a38cea..1cec602020 100644 --- a/include/MySQL_Session.h +++ b/include/MySQL_Session.h @@ -225,8 +225,8 @@ class MySQL_Session MySQL_Data_Stream *server_myds; /* * @brief Store the hostgroups that hold connections that have been flagged as 'expired' by the - * maintenance thread. This values will be used to release the retained connection in the specific - * hostgroup in housekeeping operations, before client packet processing. Currently 'housekeeping_before_pkts'. + * maintenance thread. These values will be used to release the retained connections in the specific + * hostgroups in housekeeping operations, before client packet processing. Currently 'housekeeping_before_pkts'. */ std::vector hgs_expired_conns {}; char * default_schema; From bcc6532d66130130f18d46998dcb835e24423adc Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Javier=20Jaramago=20Fern=C3=A1ndez?= Date: Fri, 19 Aug 2022 23:05:18 +0200 Subject: [PATCH 5/7] Modify previous impl for 'auto_increment_delay_multiplex_timeout_ms' and fix 'connection_delay_multiplex_ms' This commit contains an implementation rework and a fix: - Impl for 'auto_increment_delay_multiplex_timeout_ms' has been reworked in favor of reusing 'wait_until' to share logic with previous 'connection_delay_multiplex_ms' variable. - Fix previous 'connection_delay_multiplex_ms' behavior preventing connection retaining when traffic unrelated to target hostgroup is being received by the session. --- include/mysql_connection.h | 4 ---- lib/MySQL_HostGroups_Manager.cpp | 1 + lib/MySQL_Session.cpp | 31 ++++++++++++++++++++++++++++--- lib/MySQL_Thread.cpp | 16 ++++++++++++---- lib/mysql_connection.cpp | 27 --------------------------- 5 files changed, 41 insertions(+), 38 deletions(-) diff --git a/include/mysql_connection.h b/include/mysql_connection.h index 3b07fc7416..186bc25fc4 100644 --- a/include/mysql_connection.h +++ b/include/mysql_connection.h @@ -95,8 +95,6 @@ class MySQL_Connection { char scramble_buff[40]; unsigned long long creation_time; unsigned long long last_time_used; - /* @brief Time at which the last 'event' was processed by 'handler' */ - unsigned long long last_event_time; unsigned long long timeout; int auto_increment_delay_token; int fd; @@ -224,8 +222,6 @@ class MySQL_Connection { void ProcessQueryAndSetStatusFlags(char *query_digest_text); void optimize(); void close_mysql(); - uint64_t idle_time(uint64_t curtime); - bool expire_auto_increment_delay(uint64_t curtime, uint64_t timeout); void set_is_client(); // used for local_stmts diff --git a/lib/MySQL_HostGroups_Manager.cpp b/lib/MySQL_HostGroups_Manager.cpp index f19489a1cd..a13f2c48c6 100644 --- a/lib/MySQL_HostGroups_Manager.cpp +++ b/lib/MySQL_HostGroups_Manager.cpp @@ -2726,6 +2726,7 @@ void MySQL_HostGroups_Manager::push_MyConn_to_pool(MySQL_Connection *c, bool _lo MySrvC *mysrvc=NULL; if (_lock) wrlock(); + c->auto_increment_delay_token = 0; status.myconnpoll_push++; mysrvc=(MySrvC *)c->parent; proxy_debug(PROXY_DEBUG_MYSQL_CONNPOOL, 7, "Returning MySQL_Connection %p, server %s:%d with status %d\n", c, mysrvc->address, mysrvc->port, mysrvc->status); diff --git a/lib/MySQL_Session.cpp b/lib/MySQL_Session.cpp index a33860ed6b..f5f7b281cf 100644 --- a/lib/MySQL_Session.cpp +++ b/lib/MySQL_Session.cpp @@ -4377,6 +4377,9 @@ int MySQL_Session::RunQuery(MySQL_Data_Stream *myds, MySQL_Connection *myconn) { // this function was inline void MySQL_Session::handler___status_WAITING_CLIENT_DATA() { +// NOTE: Maintenance of 'multiplex_delayed' has been moved to 'housekeeping_before_pkts'. The previous impl +// is left below as an example of how to perform a more passive maintenance over session connections. +/* if (mybes) { MySQL_Backend *_mybe; unsigned int i; @@ -4397,6 +4400,7 @@ void MySQL_Session::handler___status_WAITING_CLIENT_DATA() { } } } +*/ } void MySQL_Session::housekeeping_before_pkts() { @@ -7310,6 +7314,7 @@ void MySQL_Session::create_new_session_and_reset_connection(MySQL_Data_Stream *_ new_myds->myprot.init(&new_myds, new_myds->myconn->userinfo, NULL); new_sess->status = RESETTING_CONNECTION; mc->async_state_machine = ASYNC_IDLE; // may not be true, but is used to correctly perform error handling + mc->auto_increment_delay_token = 0; new_myds->DSS = STATE_MARIADB_QUERY; thread->register_session_connection_handler(new_sess,true); if (new_myds->mypolls==NULL) { @@ -7430,9 +7435,29 @@ void MySQL_Session::finishQuery(MySQL_Data_Stream *myds, MySQL_Connection *mycon myds->myconn->set_status(true, STATUS_MYSQL_CONNECTION_NO_MULTIPLEX); } } - if (mysql_thread___multiplexing && (myds->myconn->reusable==true) && myds->myconn->IsActiveTransaction()==false && myds->myconn->MultiplexDisabled()==false) { - if (mysql_thread___connection_delay_multiplex_ms && mirror==false) { - myds->wait_until=thread->curtime+mysql_thread___connection_delay_multiplex_ms*1000; + + const bool is_active_transaction = myds->myconn->IsActiveTransaction(); + const bool multiplex_disabled_by_status = myds->myconn->MultiplexDisabled(false); + + const bool multiplex_delayed = myds->myconn->auto_increment_delay_token > 0; + const bool multiplex_delayed_with_timeout = + !multiplex_disabled_by_status && multiplex_delayed && mysql_thread___auto_increment_delay_multiplex_timeout_ms > 0; + + const bool multiplex_disabled = !multiplex_disabled_by_status && (!multiplex_delayed || multiplex_delayed_with_timeout); + const bool conn_is_reusable = myds->myconn->reusable == true && !is_active_transaction && multiplex_disabled; + + if (mysql_thread___multiplexing && conn_is_reusable) { + if ((mysql_thread___connection_delay_multiplex_ms || multiplex_delayed_with_timeout) && mirror==false) { + if (multiplex_delayed_with_timeout) { + uint64_t delay_multiplex_us = mysql_thread___connection_delay_multiplex_ms * 1000; + uint64_t auto_increment_delay_us = mysql_thread___auto_increment_delay_multiplex_timeout_ms * 1000; + uint64_t delay_us = delay_multiplex_us > auto_increment_delay_us ? delay_multiplex_us : auto_increment_delay_us; + + myds->wait_until=thread->curtime + delay_us; + } else { + myds->wait_until=thread->curtime+mysql_thread___connection_delay_multiplex_ms*1000; + } + myconn->async_state_machine=ASYNC_IDLE; myconn->multiplex_delayed=true; myds->DSS=STATE_MARIADB_GENERIC; diff --git a/lib/MySQL_Thread.cpp b/lib/MySQL_Thread.cpp index d38013c227..652a684eac 100644 --- a/lib/MySQL_Thread.cpp +++ b/lib/MySQL_Thread.cpp @@ -3740,16 +3740,24 @@ void MySQL_Thread::ProcessAllSessions_MaintenanceLoop(MySQL_Session *sess, unsig } } - // Perform the maintenance of expired 'auto_increment_delay_multiplex' for connections on the session + // Perform the maintenance for expired connections on the session if (mysql_thread___multiplexing) { const auto auto_incr_delay_multiplex_check = [curtime=this->curtime] (MySQL_Connection* myconn) -> bool { - const uint64_t multiplex_timeout_us = static_cast(mysql_thread___auto_increment_delay_multiplex_timeout_ms) * 1000; - const bool timeout_expired = multiplex_timeout_us != 0 && myconn->expire_auto_increment_delay(curtime, multiplex_timeout_us); + const uint64_t multiplex_timeout_ms = mysql_thread___auto_increment_delay_multiplex_timeout_ms; + const bool multiplex_delayed_enabled = multiplex_timeout_ms != 0 && myconn->auto_increment_delay_token > 0; + const bool timeout_expired = multiplex_delayed_enabled && myconn->myds->wait_until != 0 && myconn->myds->wait_until < curtime; + return timeout_expired; + }; + + const auto conn_delay_multiplex = [curtime=this->curtime] (MySQL_Connection* myconn) -> bool { + const bool multiplex_delayed = mysql_thread___connection_delay_multiplex_ms != 0 && myconn->multiplex_delayed == true; + const bool timeout_expired = multiplex_delayed && myconn->myds->wait_until != 0 && myconn->myds->wait_until < curtime; return timeout_expired; }; const vector> expire_conn_checks { - auto_incr_delay_multiplex_check + auto_incr_delay_multiplex_check, + conn_delay_multiplex }; sess->update_expired_conns(expire_conn_checks); diff --git a/lib/mysql_connection.cpp b/lib/mysql_connection.cpp index ff505ba729..81646e7197 100644 --- a/lib/mysql_connection.cpp +++ b/lib/mysql_connection.cpp @@ -560,32 +560,6 @@ bool MySQL_Connection::get_status(uint32_t status_flag) { return this->status_flags & status_flag; } -uint64_t MySQL_Connection::idle_time(uint64_t curtime) { - // ASYNC_QUERY_END not required due to being transient state - const bool is_idle = this->async_state_machine == ASYNC_IDLE; - - if (is_idle) { - return curtime - this->last_event_time; - } else { - return 0; - } -} - -bool MySQL_Connection::expire_auto_increment_delay(uint64_t curtime, uint64_t timeout) { - if (timeout == 0) { - return false; - } - - uint64_t idle_time = this->idle_time(curtime); - - if (idle_time > timeout) { - this->auto_increment_delay_token = 0; - return true; - } else { - return false; - } -} - void MySQL_Connection::set_status_sql_log_bin0(bool v) { if (v) { status_flags |= STATUS_MYSQL_CONNECTION_SQL_LOG_BIN0; @@ -1039,7 +1013,6 @@ void MySQL_Connection::set_is_client() { #define NEXT_IMMEDIATE(new_st) do { async_state_machine = new_st; goto handler_again; } while (0) MDB_ASYNC_ST MySQL_Connection::handler(short event) { - this->last_event_time = myds->sess->thread->curtime; unsigned long long processed_bytes=0; // issue #527 : this variable will store the amount of bytes processed during this event if (mysql==NULL) { // it is the first time handler() is being called From 4a88876ea4c6c905f708c4a73a01616dee859900 Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Javier=20Jaramago=20Fern=C3=A1ndez?= Date: Fri, 19 Aug 2022 23:09:31 +0200 Subject: [PATCH 6/7] Add warning for 'auto_increment_delay_multiplex_timeout_ms' when set to a low value --- lib/MySQL_Thread.cpp | 8 ++++++++ 1 file changed, 8 insertions(+) diff --git a/lib/MySQL_Thread.cpp b/lib/MySQL_Thread.cpp index 652a684eac..cb24acb5f5 100644 --- a/lib/MySQL_Thread.cpp +++ b/lib/MySQL_Thread.cpp @@ -1648,6 +1648,14 @@ bool MySQL_Threads_Handler::set_variable(char *name, const char *value) { // thi // integer variable ? std::unordered_map>::const_iterator it = VariablesPointers_int.find(nameS); if (it != VariablesPointers_int.end()) { + // Log warnings for variables with possibly wrong values + if (nameS == "auto_increment_delay_multiplex_timeout_ms") { + int intv = atoi(value); + if (intv <= 60) { + proxy_warning("'mysql-auto_increment_delay_multiplex_timeout_ms' is set to a low value: %ums. Remember value is in 'ms'\n", intv); + } + } + bool special_variable = std::get<3>(it->second); // if special_variable is true, min and max values are ignored, and more input validation is needed if (special_variable == false) { int intv=atoi(value); From 45d2bf2d6b6da7a6db5bdfcff7e0c98dc629b73a Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Javier=20Jaramago=20Fern=C3=A1ndez?= Date: Fri, 19 Aug 2022 23:11:42 +0200 Subject: [PATCH 7/7] Rework of 'test_auto_increment_delay_multiplex' for new cases and also testing 'connection_delay_multiplex_ms' --- .../test_auto_increment_delay_multiplex-t.cpp | 1084 ++++++++++++----- 1 file changed, 793 insertions(+), 291 deletions(-) diff --git a/test/tap/tests/test_auto_increment_delay_multiplex-t.cpp b/test/tap/tests/test_auto_increment_delay_multiplex-t.cpp index 8f3c93e21a..a6b321bec5 100644 --- a/test/tap/tests/test_auto_increment_delay_multiplex-t.cpp +++ b/test/tap/tests/test_auto_increment_delay_multiplex-t.cpp @@ -11,24 +11,47 @@ * 5. 'auto_increment_delay_multiplex_timeout_ms' should not take effect on transactions. * 6. 'auto_increment_delay_multiplex_timeout_ms' should not take effect on multiplex dissabling scenarios. * Eg: SET statements that create session variables. + * 7. Test 'connection_delay_multiplex_ms' retaining and expiring connections + * 8. Test 'connection_delay_multiplex_ms' integration with multiplexing disabling operations. + * 9. Test 'connection_delay_multiplex_ms' integration with traffic hitting the session. + * 10. Test 'connection_delay_multiplex_ms' interaction with 'auto_increment_delay_multiplex_timeout_ms'. + * + * TODO: This test requires a deep rework in order to make the code more clear and structured. */ #include +#include +#include #include #include #include +#include #include #include #include #include "json.hpp" -#include "tap.h" + #include "command_line.h" +#include "proxysql_utils.h" #include "utils.h" +#include "tap.h" + +using std::function; using std::string; using std::vector; +using nlohmann::json; + +const char* INSERT_QUERY { "INSERT INTO test.auto_inc_multiplex (c2, c3) VALUES ('foo','bar')" }; +const char* CREATE_TABLE_QUERY { + "CREATE TABLE IF NOT EXISTS test.auto_inc_multiplex " + "(c1 INT NOT NULL AUTO_INCREMENT PRIMARY KEY, c2 VARCHAR(100), c3 VARCHAR(100))" +}; + +uint32_t VAL_RANGE = 10; +uint32_t STEP = 5; int get_query_result(MYSQL* mysql, const string& query, uint64_t& out_val) { int rc = mysql_query(mysql, query.c_str()); @@ -100,387 +123,866 @@ int get_conn_auto_inc_delay_token(MYSQL* proxy_mysql, int& out_auto_inc_delay) { return EXIT_SUCCESS; } -uint32_t VAL_RANGE = 10; -uint32_t STEP = 5; +int get_session_backends(MYSQL* proxy_mysql,vector& out_backend_conns) { + MYSQL_QUERY(proxy_mysql, "PROXYSQL INTERNAL SESSION"); + MYSQL_RES* my_res = mysql_store_result(proxy_mysql); + vector int_sess_res = extract_mysql_rows(my_res); + mysql_free_result(my_res); -int main(int argc, char** argv) { - CommandLine cl; + int cur_auto_inc_delay_mult = 0; - if (cl.getEnv()) { - diag("Failed to get the required environmental variables."); + if (int_sess_res.empty()) { + log_err("Empty result received from 'PROXYSQL INTERNAL SESSION'"); return EXIT_FAILURE; } - plan( - 1 + // Check variables are present - ((VAL_RANGE / STEP) + 1) * 2 + // Tests for different 'auto_increment_delay_multiplex' values - (VAL_RANGE / STEP) * 3 + // Tests for different 'auto_increment_delay_multiplex_timeout_ms' values - 3 + // Tests for 'auto_increment_delay_multiplex_timeout_ms' zero value - 2 + // Tests for 'auto_increment_delay_multiplex_timeout_ms' keep alive queries - 2 + // Tests for 'auto_increment_delay_multiplex_timeout_ms' transaction behavior - 1 // Tests for 'auto_increment_delay_multiplex_timeout_ms' multiplex disabled by SET statement - ); + try { + nlohmann::json j_int_sess = nlohmann::json::parse(int_sess_res[0][0]); + nlohmann::json backend_conns = j_int_sess.at("backends"); + vector _out_conns {}; - MYSQL* proxy_mysql = mysql_init(NULL); - MYSQL* proxy_admin = mysql_init(NULL); + for (const auto& j_conn : backend_conns) { + _out_conns.push_back(j_conn); + } - if (!mysql_real_connect(proxy_mysql, cl.host, cl.username, cl.password, NULL, cl.port, NULL, 0)) { - fprintf(stderr, "File %s, line %d, Error: \"%s\"\n", __FILE__, __LINE__, mysql_error(proxy_mysql)); + out_backend_conns = _out_conns; + } catch (const std::exception& ex) { + const string err_msg { + string { "Invalid JSON received from 'PROXYSQL INTERNAL SESSION'. Ex: '" } + ex.what() + "'" + }; + log_err(err_msg.c_str()); return EXIT_FAILURE; } - if (!mysql_real_connect(proxy_admin, cl.host, cl.admin_username, cl.admin_password, NULL, cl.admin_port, NULL, 0)) { - fprintf(stderr, "File %s, line %d, Error: \"%s\"\n", __FILE__, __LINE__, mysql_error(proxy_admin)); + + return EXIT_SUCCESS; +} + +int check_auto_increment_timeout( + MYSQL* proxy_admin, MYSQL* proxy_mysql, uint32_t f_auto_incr_val, uint64_t poll_to, uint32_t auto_inc_delay_to +) { + int cur_auto_inc_delay_mult = 0; + const string set_auto_inc_to_query { + "SET mysql-auto_increment_delay_multiplex_timeout_ms=" + std::to_string(auto_inc_delay_to) + }; + diag("%s: Executing query `%s`...", tap_curtime().c_str(), set_auto_inc_to_query.c_str()); + MYSQL_QUERY(proxy_admin, set_auto_inc_to_query.c_str()); + + MYSQL_QUERY(proxy_admin, "LOAD MYSQL VARIABLES TO RUNTIME"); + diag("%s: Executing query `%s`...", tap_curtime().c_str(), "LOAD MYSQL VARIABLES TO RUNTIME"); + + MYSQL_QUERY(proxy_mysql, INSERT_QUERY); + diag("%s: Executing query `%s`...", tap_curtime().c_str(), INSERT_QUERY); + + // Wait at least '500' milliseconds over the poll period + usleep((poll_to + 500) * 1000); + uint32_t waited = poll_to + 500; + + int g_res = get_conn_auto_inc_delay_token(proxy_mysql, cur_auto_inc_delay_mult); + if (g_res != EXIT_SUCCESS) { return EXIT_FAILURE; } - MYSQL_QUERY(proxy_mysql, "CREATE DATABASE IF NOT EXISTS test"); - MYSQL_QUERY(proxy_mysql, "DROP TABLE IF EXISTS test.auto_inc_multiplex"); - MYSQL_QUERY(proxy_mysql, "CREATE TABLE IF NOT EXISTS test.auto_inc_multiplex (c1 INT NOT NULL AUTO_INCREMENT PRIMARY KEY, c2 VARCHAR(100), c3 VARCHAR(100))"); + ok( + f_auto_incr_val == cur_auto_inc_delay_mult, + "'auto_increment_delay_token' val unchanged before timeout:" + " { Exp: %d, Act: %d, Timeout: %d, Waited: %d }", + f_auto_incr_val, cur_auto_inc_delay_mult, auto_inc_delay_to, waited + ); - // 1. Check that the required variables are present - { - uint64_t auto_increment_delay_multiplex = 0; - MYSQL_QUERY(proxy_admin, "SELECT variable_value FROM global_variables WHERE variable_name='mysql-auto_increment_delay_multiplex'"); - MYSQL_RES* my_res_auto_inc_multiplex = mysql_store_result(proxy_admin); - uint64_t auto_inc_row_num = mysql_num_rows(my_res_auto_inc_multiplex); - mysql_free_result(my_res_auto_inc_multiplex); + uint32_t DEF_TIMEOUT = 5; + uint32_t timeout = auto_inc_delay_to == 0 ? DEF_TIMEOUT : auto_inc_delay_to; - MYSQL_QUERY(proxy_admin, "SELECT variable_value FROM global_variables WHERE variable_name='mysql-auto_increment_delay_multiplex_timeout_ms'"); - MYSQL_RES* my_res_auto_inc_multiplex_to = mysql_store_result(proxy_admin); - uint64_t auto_inc_to_row_num = mysql_num_rows(my_res_auto_inc_multiplex_to); - mysql_free_result(my_res_auto_inc_multiplex_to); + // Wait timeout and check that the connection is detached + usleep((timeout + poll_to + 500) * 1000); + waited = (timeout + poll_to + 500); + + // Check 'auto_increment_delay_token' is '0' after timeout + g_res = get_conn_auto_inc_delay_token(proxy_mysql, cur_auto_inc_delay_mult); + if (g_res != EXIT_SUCCESS) { + return EXIT_FAILURE; + } + if (auto_inc_delay_to == 0) { ok( - auto_inc_row_num == 1 && auto_inc_to_row_num == 1, - "'mysql-auto_increment_delay_multiplex' and 'mysql-auto_increment_delay_multiplex_timeout_ms' variables present" + f_auto_incr_val == cur_auto_inc_delay_mult, + "'auto_increment_delay_token' val should remain unchanged '%d' after default timeout:" + " { Exp: %d, Act: %d, mysql-auto_increment_delay_multiplex_timeout_ms: %d, Timeout: %d, Waited: %d }", + f_auto_incr_val, f_auto_incr_val, cur_auto_inc_delay_mult, auto_inc_delay_to, timeout, waited + ); + } else { + ok( + -1 == cur_auto_inc_delay_mult, + "'auto_increment_delay_token' val should be '-1' after timeout:" + " { Exp: %d, Act: %d, Timeout: %d, Waited: %d }", + -1, cur_auto_inc_delay_mult, auto_inc_delay_to, waited ); } - // 2. Change and check 'auto_increment_delay_multiplex' behavior - { - // Disable the 'timeout' for the this check since it can be fixated now - MYSQL_QUERY(proxy_admin, "SET mysql-auto_increment_delay_multiplex_timeout_ms=0"); - - int cur_auto_inc_delay_mult = 0; - int exp_auto_inc_delay_mult = 0; - - for (uint32_t val = 0; val <= VAL_RANGE; val += STEP) { - MYSQL_QUERY(proxy_admin, string {"SET mysql-auto_increment_delay_multiplex=" + std::to_string(val)}.c_str()); - MYSQL_QUERY(proxy_admin, "LOAD MYSQL VARIABLES TO RUNTIME"); - MYSQL_QUERY(proxy_mysql, "INSERT INTO test.auto_inc_multiplex (c2, c3) VALUES ('foo','bar')"); - - for (uint32_t i = 1; i < val; i++) { - // We target the same hostgroup as before - MYSQL_QUERY(proxy_mysql, "DO 1"); - int g_res = get_conn_auto_inc_delay_token(proxy_mysql, cur_auto_inc_delay_mult); - if (g_res != EXIT_SUCCESS) { - return EXIT_FAILURE; - } - - exp_auto_inc_delay_mult = val - i; - - diag( - "'auto_increment_delay_token' should be reduced by one with each query to the same hostgroup: { Exp: %d, Act: %d }", - exp_auto_inc_delay_mult, cur_auto_inc_delay_mult - ); - if (cur_auto_inc_delay_mult != exp_auto_inc_delay_mult) { - break; - } - } + MYSQL_QUERY(proxy_mysql, "DO 1"); + diag("%s: Executing query `%s`...", tap_curtime().c_str(), "DO 1"); - ok( - exp_auto_inc_delay_mult == cur_auto_inc_delay_mult, - "'auto_increment_delay_token' should be reduced by one with each query to the same hostgroup: { Exp: %d, Act: %d }", - exp_auto_inc_delay_mult, cur_auto_inc_delay_mult - ); + uint32_t old_auto_inc_delay_mult = cur_auto_inc_delay_mult; + g_res = get_conn_auto_inc_delay_token(proxy_mysql, cur_auto_inc_delay_mult); + if (g_res != EXIT_SUCCESS) { + return EXIT_FAILURE; + } - if (cur_auto_inc_delay_mult != exp_auto_inc_delay_mult) { - break; - } + if (auto_inc_delay_to == 0) { + ok( + old_auto_inc_delay_mult == cur_auto_inc_delay_mult + 1, + "'auto_increment_delay_token' should be reduced by one because timeout is meaningless: { Old: %d, New: %d }", + old_auto_inc_delay_mult, cur_auto_inc_delay_mult + ); + } else { + ok( + cur_auto_inc_delay_mult == -1, + "Connection should no longer be attached when 'auto_increment_delay_token' reaches '0': { Exp: %d, Act: %d }", + -1, cur_auto_inc_delay_mult + ); + } + + return EXIT_SUCCESS; +}; + +int check_variables_config(MYSQL* proxy_mysql, MYSQL* proxy_admin) { + uint64_t auto_increment_delay_multiplex = 0; + MYSQL_QUERY(proxy_admin, "SELECT variable_value FROM global_variables WHERE variable_name='mysql-auto_increment_delay_multiplex'"); + MYSQL_RES* my_res_auto_inc_multiplex = mysql_store_result(proxy_admin); + uint64_t auto_inc_row_num = mysql_num_rows(my_res_auto_inc_multiplex); + mysql_free_result(my_res_auto_inc_multiplex); + + MYSQL_QUERY(proxy_admin, "SELECT variable_value FROM global_variables WHERE variable_name='mysql-auto_increment_delay_multiplex_timeout_ms'"); + MYSQL_RES* my_res_auto_inc_multiplex_to = mysql_store_result(proxy_admin); + uint64_t auto_inc_to_row_num = mysql_num_rows(my_res_auto_inc_multiplex_to); + mysql_free_result(my_res_auto_inc_multiplex_to); + + ok( + auto_inc_row_num == 1 && auto_inc_to_row_num == 1, + "'mysql-auto_increment_delay_multiplex' and 'mysql-auto_increment_delay_multiplex_timeout_ms' variables present" + ); + + return EXIT_SUCCESS; +} + +int check_auto_increment_delay_multiplex(MYSQL* proxy_mysql, MYSQL* proxy_admin) { + // Disable the 'timeout' for the this check since it can be fixated now + diag("%s: Executing query `%s`...", tap_curtime().c_str(), "SET mysql-auto_increment_delay_multiplex_timeout_ms=0"); + MYSQL_QUERY(proxy_admin, "SET mysql-auto_increment_delay_multiplex_timeout_ms=0"); + diag("%s: Executing query `%s`...", tap_curtime().c_str(), "SET mysql-connection_delay_multiplex_ms=0"); + MYSQL_QUERY(proxy_admin, "SET mysql-connection_delay_multiplex_ms=0"); - // Check that the connection is no longer attached when `auto_increment_delay_token` reaches `0`. + int cur_auto_inc_delay_mult = 0; + int exp_auto_inc_delay_mult = 0; + + for (uint32_t val = 0; val <= VAL_RANGE; val += STEP) { + diag("Testing 'mysql-auto_increment_delay_multiplex_timeout_ms' for value '%d'", val); + MYSQL_QUERY(proxy_admin, string {"SET mysql-auto_increment_delay_multiplex=" + std::to_string(val)}.c_str()); + MYSQL_QUERY(proxy_admin, "LOAD MYSQL VARIABLES TO RUNTIME"); + MYSQL_QUERY(proxy_mysql, "INSERT INTO test.auto_inc_multiplex (c2, c3) VALUES ('foo','bar')"); + + for (uint32_t i = 1; i < val; i++) { + // We target the same hostgroup as before MYSQL_QUERY(proxy_mysql, "DO 1"); int g_res = get_conn_auto_inc_delay_token(proxy_mysql, cur_auto_inc_delay_mult); if (g_res != EXIT_SUCCESS) { return EXIT_FAILURE; } - ok( - cur_auto_inc_delay_mult == -1, - "Connection should no longer be attached when 'auto_increment_delay_token' reaches '0'" + exp_auto_inc_delay_mult = val - i; + + diag( + "'auto_increment_delay_token' should be reduced by one with each query to the same hostgroup: { Exp: %d, Act: %d }", + exp_auto_inc_delay_mult, cur_auto_inc_delay_mult ); + if (cur_auto_inc_delay_mult != exp_auto_inc_delay_mult) { + break; + } } - } - const auto check_auto_increment_to = [] (MYSQL* proxy_admin, MYSQL* proxy_mysql, uint32_t f_auto_incr_val, uint64_t poll_to, uint32_t auto_inc_delay_to) -> int { - int cur_auto_inc_delay_mult = 0; - const string set_auto_inc_to_query { - "SET mysql-auto_increment_delay_multiplex_timeout_ms=" + std::to_string(auto_inc_delay_to) - }; - MYSQL_QUERY(proxy_admin, set_auto_inc_to_query.c_str()); - MYSQL_QUERY(proxy_admin, "LOAD MYSQL VARIABLES TO RUNTIME"); - MYSQL_QUERY(proxy_mysql, "INSERT INTO test.auto_inc_multiplex (c2, c3) VALUES ('foo','bar')"); + ok( + exp_auto_inc_delay_mult == cur_auto_inc_delay_mult, + "'auto_increment_delay_token' should be reduced by one with each query to the same hostgroup: { Exp: %d, Act: %d }", + exp_auto_inc_delay_mult, cur_auto_inc_delay_mult + ); - // Wait at least '500' milliseconds over the poll period - usleep((poll_to + 500) * 1000); - uint32_t waited = poll_to + 500; + if (cur_auto_inc_delay_mult != exp_auto_inc_delay_mult) { + break; + } + // Check that the connection is no longer attached when `auto_increment_delay_token` reaches `0`. + MYSQL_QUERY(proxy_mysql, "DO 1"); int g_res = get_conn_auto_inc_delay_token(proxy_mysql, cur_auto_inc_delay_mult); if (g_res != EXIT_SUCCESS) { return EXIT_FAILURE; } ok( - f_auto_incr_val == cur_auto_inc_delay_mult, - "'auto_increment_delay_token' val unchanged before timeout:" - " { Exp: %d, Act: %d, Timeout: %d, Waited: %d }", - f_auto_incr_val, cur_auto_inc_delay_mult, auto_inc_delay_to, waited + cur_auto_inc_delay_mult == -1, + "Connection should no longer be attached when 'auto_increment_delay_token' reaches '0'" ); + } - uint32_t DEF_TIMEOUT = 5; - uint32_t timeout = auto_inc_delay_to == 0 ? DEF_TIMEOUT : auto_inc_delay_to; + return EXIT_SUCCESS; +} + +int check_auto_increment_delay_multiplex_timeout(MYSQL* proxy_mysql, MYSQL* proxy_admin) { + // Set the default 'mysql-auto_increment_delay_multiplex' since it's no longer relevant + const int f_auto_incr_val = 5; + const string set_auto_inc_query { "SET mysql-auto_increment_delay_multiplex=" + std::to_string(f_auto_incr_val) }; + uint64_t poll_timeout = 0; + int g_res = 0; + + diag("%s: Executing query `%s`...", tap_curtime().c_str(), set_auto_inc_query.c_str()); + MYSQL_QUERY(proxy_admin, set_auto_inc_query.c_str()); + diag("%s: Executing query `%s`...", tap_curtime().c_str(), "SET mysql-connection_delay_multiplex_ms=0"); + MYSQL_QUERY(proxy_admin, "SET mysql-connection_delay_multiplex_ms=0"); + + const string q_poll_timeout { "SELECT variable_value FROM global_variables WHERE variable_name='mysql-poll_timeout'" }; + diag("%s: Executing query `%s`...", tap_curtime().c_str(), q_poll_timeout.c_str()); + g_res = get_query_result(proxy_admin, q_poll_timeout.c_str(), poll_timeout); + if (g_res != EXIT_SUCCESS) { return EXIT_FAILURE; } + + // Check that different values for 'auto_increment_delay_multiplex_timeout_ms' behave properly + for (uint32_t auto_inc_delay_to = 5; auto_inc_delay_to <= VAL_RANGE; auto_inc_delay_to += STEP) { + uint32_t _auto_inc_delay_to = auto_inc_delay_to * 1000; + + if (_auto_inc_delay_to < (poll_timeout + 500)) { + diag( + "Error: Supplied 'auto_increment_delay_multiplex_timeout_ms' too small: { Act: %d, Min: %ld }", + _auto_inc_delay_to, poll_timeout + 500 + ); + return EXIT_FAILURE; + } + + int c_res = check_auto_increment_timeout(proxy_admin, proxy_mysql, f_auto_incr_val, poll_timeout, _auto_inc_delay_to); + if (c_res != EXIT_SUCCESS) { return EXIT_FAILURE; } + } + + // Check that value '0' for 'auto_increment_delay_multiplex_timeout_ms' disables the feature + int c_res = check_auto_increment_timeout(proxy_admin, proxy_mysql, f_auto_incr_val, poll_timeout, 0); + if (c_res != EXIT_SUCCESS) { return EXIT_FAILURE; } + + // Check that using the connection reset the internal timer, keeping the connection attached + const uint32_t timeout_ms = 2000; + // Impose a big delay so we are sure only 'timeout' is being relevant + const uint32_t delay = 100; + const string timeout_query { + "SET mysql-auto_increment_delay_multiplex_timeout_ms=" + std::to_string(timeout_ms) + }; + const string delay_query { "SET mysql-auto_increment_delay_multiplex=" + std::to_string(delay) }; + poll_timeout = 0; + const string poll_timeout_query { + "SELECT variable_value FROM global_variables WHERE variable_name='mysql-poll_timeout'" + }; + + g_res = get_query_result(proxy_admin, poll_timeout_query.c_str(), poll_timeout); + if (g_res != EXIT_SUCCESS) { return EXIT_FAILURE; } + + MYSQL_QUERY(proxy_admin, delay_query.c_str()); + diag("%s: Executing query `%s`...", tap_curtime().c_str(), delay_query.c_str()); + MYSQL_QUERY(proxy_admin, timeout_query.c_str()); + diag("%s: Executing query `%s`...", tap_curtime().c_str(), timeout_query.c_str()); + MYSQL_QUERY(proxy_admin, "LOAD MYSQL VARIABLES TO RUNTIME"); - // Wait timeout and check that the connection is detached - usleep((timeout + poll_to + 500) * 1000); - waited = (timeout + poll_to + 500); + { + // Insert disabling multiplexing for the connection + diag("%s: Executing query `%s`...", tap_curtime().c_str(), INSERT_QUERY); + MYSQL_QUERY(proxy_mysql, INSERT_QUERY); + + // Perform queries in the same connection + diag("Execute queries beyond imposed timeout"); + uint32_t waited = 0; + while (waited < timeout_ms * 3) { + sleep(1); + + diag("%s: Executing query `%s`...", tap_curtime().c_str(), "/* hostgroup=0 */ DO 1"); + MYSQL_QUERY(proxy_mysql, "/* hostgroup=0 */ DO 1"); + waited += 1000; + } - // Check 'auto_increment_delay_token' is '0' after timeout - g_res = get_conn_auto_inc_delay_token(proxy_mysql, cur_auto_inc_delay_mult); + int cur_delay = 0; + int g_res = get_conn_auto_inc_delay_token(proxy_mysql, cur_delay); if (g_res != EXIT_SUCCESS) { return EXIT_FAILURE; } - if (auto_inc_delay_to == 0) { - ok( - f_auto_incr_val == cur_auto_inc_delay_mult, - "'auto_increment_delay_token' val should remain unchanged '%d' after default timeout:" - " { Exp: %d, Act: %d, mysql-auto_increment_delay_multiplex_timeout_ms: %d, Timeout: %d, Waited: %d }", - f_auto_incr_val, f_auto_incr_val, cur_auto_inc_delay_mult, auto_inc_delay_to, timeout, waited - ); - } else { - ok( - -1 == cur_auto_inc_delay_mult, - "'auto_increment_delay_token' val should be '-1' after timeout:" - " { Exp: %d, Act: %d, Timeout: %d, Waited: %d }", - 0, cur_auto_inc_delay_mult, auto_inc_delay_to, waited - ); + uint32_t exp_delay = delay - (waited/1000); + ok( + exp_delay == cur_delay, + "Connection was kept after timeout due to queries issues in the same hostgroup:" + " 'auto_increment_delay_multiplex' - Exp: '%d', Act: '%d'", + exp_delay, cur_delay + ); + + waited = 0; + // Perform queries in other connections + while (waited < timeout_ms * 3) { + sleep(1); + + diag("%s: Executing query `%s`...", tap_curtime().c_str(), "SELECT 1"); + MYSQL_QUERY(proxy_mysql, "SELECT 1"); + mysql_free_result(mysql_store_result(proxy_mysql)); + + waited += 1000; + // After this time the connection should have timeout already + if (waited > timeout_ms + poll_timeout + 500) { + break; + } } - MYSQL_QUERY(proxy_mysql, "DO 1"); - uint32_t old_auto_inc_delay_mult = cur_auto_inc_delay_mult; - g_res = get_conn_auto_inc_delay_token(proxy_mysql, cur_auto_inc_delay_mult); + cur_delay = 0; + g_res = get_conn_auto_inc_delay_token(proxy_mysql, cur_delay); if (g_res != EXIT_SUCCESS) { return EXIT_FAILURE; } - if (auto_inc_delay_to == 0) { - ok( - old_auto_inc_delay_mult == cur_auto_inc_delay_mult + 1, - "'auto_increment_delay_token' should be reduced by one because timeout is meaningless: { Old: %d, New: %d }", - old_auto_inc_delay_mult, cur_auto_inc_delay_mult - ); - } else { - ok( - cur_auto_inc_delay_mult == -1, - "Connection should no longer be attached when 'auto_increment_delay_token' reaches '0'" - ); + ok( + -1 == cur_delay, + "Connection returned to connpool when queries are issued in different hostgroup - Exp: '%d', Act: '%d'", + -1, cur_delay + ); + } + + // Transactions connections should be preserved by 'auto_increment_delay_multiplex_timeout_ms' + { + diag("%s: Executing query `%s`...", tap_curtime().c_str(), "BEGIN"); + MYSQL_QUERY(proxy_mysql, "BEGIN"); + diag("%s: Executing query `%s`...", tap_curtime().c_str(), INSERT_QUERY); + MYSQL_QUERY(proxy_mysql, INSERT_QUERY); + + // Wait for the timeout and check the value + diag("%s: Waiting for timeout to expire...", tap_curtime().c_str()); + usleep(timeout_ms * 1000 + poll_timeout * 1000 + 500 * 1000 * 2); + + diag("%s: Extracting current auto inc delay...", tap_curtime().c_str()); + int cur_delay = 0; + int g_res = get_conn_auto_inc_delay_token(proxy_mysql, cur_delay); + if (g_res != EXIT_SUCCESS) { + return EXIT_FAILURE; } - return EXIT_SUCCESS; - }; + ok( + delay == cur_delay, + "Connection should not be returned to conn_pool due to transaction - Exp: '%d', Act: '%d'", + delay, cur_delay + ); - const char* insert_query { "INSERT INTO test.auto_inc_multiplex (c2, c3) VALUES ('foo','bar')" }; + diag("%s: Executing query `%s`...", tap_curtime().c_str(), "COMMIT"); + MYSQL_QUERY(proxy_mysql, "COMMIT"); - // 3. Change and check 'auto_increment_delay_multiplex_timeout_ms' behavior + diag("%s: Waiting for timeout to expire...", tap_curtime().c_str()); + usleep(timeout_ms * 1000 + poll_timeout * 1000 + 500 * 1000 * 2); + + diag("%s: Extracting current auto inc delay...", tap_curtime().c_str()); + cur_delay = 0; + g_res = get_conn_auto_inc_delay_token(proxy_mysql, cur_delay); + if (g_res != EXIT_SUCCESS) { + return EXIT_FAILURE; + } + + ok( + -1 == cur_delay, + "Connection should be returned to conn_pool after 'COMMIT' and 'timeout' wait - Exp: '%d', Act: '%d'", + -1, cur_delay + ); + } + + // Multiplex disabled by any action should take precedence over 'auto_increment_delay_multiplex_timeout_ms' { - // Set the default 'mysql-auto_increment_delay_multiplex' since it's no longer relevant - const int f_auto_incr_val = 5; - const string set_auto_inc_query { - "SET mysql-auto_increment_delay_multiplex=" + std::to_string(f_auto_incr_val) - }; - MYSQL_QUERY(proxy_admin, set_auto_inc_query.c_str()); - - uint64_t poll_timeout = 0; - const string q_poll_timeout { "SELECT variable_value FROM global_variables WHERE variable_name='mysql-poll_timeout'" }; - int g_res = get_query_result(proxy_admin, q_poll_timeout.c_str(), poll_timeout); - if (g_res != EXIT_SUCCESS) { return EXIT_FAILURE; } - - // Check that different values for 'auto_increment_delay_multiplex_timeout_ms' behave properly - for (uint32_t auto_inc_delay_to = 5; auto_inc_delay_to <= VAL_RANGE; auto_inc_delay_to += STEP) { - uint32_t _auto_inc_delay_to = auto_inc_delay_to * 1000; - - if (_auto_inc_delay_to < (poll_timeout + 500)) { - diag( - "Error: Supplied 'auto_increment_delay_multiplex_timeout_ms' too small: { Act: %d, Min: %ld }", - _auto_inc_delay_to, poll_timeout + 500 - ); - return EXIT_FAILURE; - } + const char* set_query { "SET @local_var='foo'" }; + diag("%s: Executing query `%s`...", tap_curtime().c_str(), set_query); + MYSQL_QUERY(proxy_mysql, set_query); + diag("%s: Executing query `%s`...", tap_curtime().c_str(), INSERT_QUERY); + MYSQL_QUERY(proxy_mysql, INSERT_QUERY); + + // Wait for the timeout and check the value + diag("%s: Waiting for timeout to expire...", tap_curtime().c_str()); + usleep(timeout_ms * 1000 + poll_timeout * 1000 + 500 * 1000 * 2); + + diag("%s: Extracting current auto inc delay...", tap_curtime().c_str()); + int cur_delay = 0; + int g_res = get_conn_auto_inc_delay_token(proxy_mysql, cur_delay); + if (g_res != EXIT_SUCCESS) { + return EXIT_FAILURE; + } + + ok( + delay == cur_delay, + "Connection should not be returned to conn_pool due to SET session var - Exp: '%d', Act: '%d'", + delay, cur_delay + ); + + // A new connection is required after multiplexing is disabled by local session variables + mysql_close(proxy_mysql); + } + + return EXIT_SUCCESS; +} + +typedef std::chrono::high_resolution_clock hrc; + +void check_connection_retained(MYSQL* proxy_mysql, uint32_t exp_conns) { + vector j_sess_backends {}; + + diag("Extracting info from 'PROXYSQL INTERNAL SESSION'"); + int g_res = get_session_backends(proxy_mysql, j_sess_backends); + if (g_res != EXIT_SUCCESS) { + diag("Failed to optain info from 'PROXYSQL INTERNAL SESSION'"); + } - int c_res = check_auto_increment_to(proxy_admin, proxy_mysql, f_auto_incr_val, poll_timeout, _auto_inc_delay_to); - if (c_res != EXIT_SUCCESS) { return EXIT_FAILURE; } + uint32_t backend_conns = 0; + for (const json& j_sess_backend : j_sess_backends) { + if (j_sess_backend.find("conn") != j_sess_backend.end()) { + backend_conns += 1; } + } - // Check that value '0' for 'auto_increment_delay_multiplex_timeout_ms' disables the feature - int c_res = check_auto_increment_to(proxy_admin, proxy_mysql, f_auto_incr_val, poll_timeout, 0); - if (c_res != EXIT_SUCCESS) { return EXIT_FAILURE; } + if (exp_conns > 0) { + ok( + backend_conns == exp_conns, + "Backend connection should be RETAINED - NumBackendConns: '%d'", + backend_conns + ); + } else { + ok( + backend_conns == exp_conns, + "Backend connection should be RETURNED - NumBackendConns: '%d'", + backend_conns + ); + } +}; - // Check that using the connection reset the internal timer, keeping the connection attached - const uint32_t timeout_ms = 2000; - // Impose a big delay so we are sure only 'timeout' is being relevant - const uint32_t delay = 100; - const string timeout_query { - "SET mysql-auto_increment_delay_multiplex_timeout_ms=" + std::to_string(timeout_ms) - }; - const string delay_query { "SET mysql-auto_increment_delay_multiplex=" + std::to_string(delay) }; - poll_timeout = 0; - const string poll_timeout_query { - "SELECT variable_value FROM global_variables WHERE variable_name='mysql-poll_timeout'" - }; +int check_transactions_and_multiplex_disable( + MYSQL* proxy_mysql, const char* query, const uint32_t timeout, uint64_t poll_timeout=2 +) { + diag("%s: Executing query `%s`...", tap_curtime().c_str(), "BEGIN"); + MYSQL_QUERY(proxy_mysql, "BEGIN"); + diag("%s: Executing query `%s`...", tap_curtime().c_str(), query); + MYSQL_QUERY(proxy_mysql, query); - g_res = get_query_result(proxy_admin, poll_timeout_query.c_str(), poll_timeout); - if (g_res != EXIT_SUCCESS) { return EXIT_FAILURE; } + diag("Checking connection present before timeout..."); + check_connection_retained(proxy_mysql, 1); - MYSQL_QUERY(proxy_admin, delay_query.c_str()); - diag("%s: Executing query `%s`...", tap_curtime().c_str(), delay_query.c_str()); - MYSQL_QUERY(proxy_admin, timeout_query.c_str()); - diag("%s: Executing query `%s`...", tap_curtime().c_str(), timeout_query.c_str()); - MYSQL_QUERY(proxy_admin, "LOAD MYSQL VARIABLES TO RUNTIME"); + diag("Sleeping for '%ld' seconds", timeout + poll_timeout); + sleep(timeout + poll_timeout); - { - // Insert disabling multiplexing for the connection - diag("%s: Executing query `%s`...", tap_curtime().c_str(), insert_query); - MYSQL_QUERY(proxy_mysql, insert_query); + diag("Checking connection is still present after timeout due to transaction..."); + check_connection_retained(proxy_mysql, 1); - // Perform queries in the same connection - diag("Execute queries beyond imposed timeout"); - uint32_t waited = 0; - while (waited < timeout_ms * 3) { - sleep(1); + diag("%s: Executing query `%s`...", tap_curtime().c_str(), "COMMIT"); + MYSQL_QUERY(proxy_mysql, "COMMIT"); - diag("%s: Executing query `%s`...", tap_curtime().c_str(), "/* hostgroup=0 */ DO 1"); - MYSQL_QUERY(proxy_mysql, "/* hostgroup=0 */ DO 1"); - waited += 1000; - } + diag("Sleeping for '%lf' seconds", timeout / 2.0); + sleep(timeout / 2.0); - int cur_delay = 0; - int g_res = get_conn_auto_inc_delay_token(proxy_mysql, cur_delay); - if (g_res != EXIT_SUCCESS) { - return EXIT_FAILURE; - } + diag("Checking connection is present after 'COMMIT' due to timeout..."); + check_connection_retained(proxy_mysql, 1); - uint32_t exp_delay = delay - (waited/1000); - ok( - exp_delay == cur_delay, - "Connection was kept after timeout due to queries issues in the same hostgroup:" - " 'auto_increment_delay_multiplex' - Exp: '%d', Act: '%d'", - exp_delay, cur_delay - ); + diag("Sleeping for '%ld' seconds", timeout + poll_timeout); + sleep(timeout + poll_timeout); - waited = 0; - // Perform queries in other connections - while (waited < timeout_ms * 3) { - sleep(1); + diag("Checking connection is RETURNED after 'COMMIT' and after timeout..."); + check_connection_retained(proxy_mysql, 0); - diag("%s: Executing query `%s`...", tap_curtime().c_str(), "SELECT 1"); - MYSQL_QUERY(proxy_mysql, "SELECT 1"); - mysql_free_result(mysql_store_result(proxy_mysql)); + diag("Checking multiplex disabled by any action take precedence over 'connection_delay_multiplex_ms'..."); - waited += 1000; - // After this time the connection should have timeout already - if (waited > timeout_ms + poll_timeout + 500) { - break; - } - } + const char* set_query { "SET @local_var='foo'" }; + diag("%s: Executing query `%s`...", tap_curtime().c_str(), set_query); + MYSQL_QUERY(proxy_mysql, set_query); - cur_delay = 0; - g_res = get_conn_auto_inc_delay_token(proxy_mysql, cur_delay); - if (g_res != EXIT_SUCCESS) { - return EXIT_FAILURE; - } + diag("Sleeping for '%ld' seconds", timeout + poll_timeout); + sleep(timeout + poll_timeout); - ok( - -1 == cur_delay, - "Connection returned to connpool when queries are issued in different hostgroup - Exp: '%d', Act: '%d'", - -1, cur_delay - ); - } + check_connection_retained(proxy_mysql, 1); - // Transactions connections should be preserved by 'auto_increment_delay_multiplex_timeout_ms' - { - diag("%s: Executing query `%s`...", tap_curtime().c_str(), "BEGIN"); - MYSQL_QUERY(proxy_mysql, "BEGIN"); - diag("%s: Executing query `%s`...", tap_curtime().c_str(), insert_query); - MYSQL_QUERY(proxy_mysql, insert_query); + return EXIT_SUCCESS; +} - // Wait for the timeout and check the value - diag("%s: Waiting for timeout to expire...", tap_curtime().c_str()); - usleep(timeout_ms * 1000 + poll_timeout * 1000 + 500 * 1000 * 2); - diag("%s: Extracting current auto inc delay...", tap_curtime().c_str()); - int cur_delay = 0; - int g_res = get_conn_auto_inc_delay_token(proxy_mysql, cur_delay); - if (g_res != EXIT_SUCCESS) { - return EXIT_FAILURE; - } +int check_connection_delay_multiplex_ms(MYSQL* proxy_mysql, MYSQL* proxy_admin) { + std::chrono::nanoseconds duration; + hrc::time_point start; + hrc::time_point end; - ok( - delay == cur_delay, - "Connection should not be returned to conn_pool due to transaction - Exp: '%d', Act: '%d'", - delay, cur_delay - ); + const uint32_t timeout = 3; + string set_delay_multiplex {}; + string_format("SET mysql-connection_delay_multiplex_ms=%d", set_delay_multiplex, timeout * 1000); + const char* set_auto_inc_delay { "SET mysql-auto_increment_delay_multiplex_timeout_ms=0" }; - diag("%s: Executing query `%s`...", tap_curtime().c_str(), "COMMIT"); - MYSQL_QUERY(proxy_mysql, "COMMIT"); + diag("%s: Executing query `%s`...", tap_curtime().c_str(), set_delay_multiplex.c_str()); + MYSQL_QUERY(proxy_admin, set_delay_multiplex.c_str()); - diag("%s: Waiting for timeout to expire...", tap_curtime().c_str()); - usleep(timeout_ms * 1000 + poll_timeout * 1000 + 500 * 1000 * 2); + diag("%s: Executing query `%s`...", tap_curtime().c_str(), set_auto_inc_delay); + MYSQL_QUERY(proxy_admin, set_auto_inc_delay); - diag("%s: Extracting current auto inc delay...", tap_curtime().c_str()); - cur_delay = 0; - g_res = get_conn_auto_inc_delay_token(proxy_mysql, cur_delay); - if (g_res != EXIT_SUCCESS) { - return EXIT_FAILURE; - } + diag("%s: Executing query `%s`...", tap_curtime().c_str(), "LOAD MYSQL VARIABLES TO RUNTIME"); + MYSQL_QUERY(proxy_admin, "LOAD MYSQL VARIABLES TO RUNTIME"); - ok( - -1 == cur_delay, - "Connection should be returned to conn_pool after 'COMMIT' and 'timeout' wait - Exp: '%d', Act: '%d'", - -1, cur_delay - ); + MYSQL_QUERY(proxy_mysql, "SELECT 1"); + mysql_free_result(mysql_store_result(proxy_mysql)); + + start = hrc::now(); + + check_connection_retained(proxy_mysql, 1); + diag("Sleeping for '%d' seconds", 2); + sleep(2); + + end = hrc::now(); + duration = end - start; + + double waited = duration.count() / pow(10, 9); + if (waited < 3) { + diag("Performing second check after '%lf' waited seconds...", waited); + check_connection_retained(proxy_mysql, 1); + } else { + diag("Second check can't be performed due to timeout already expired."); + } + + diag("Sleeping for '%d' seconds", 2); + sleep(2); + waited += 2; + + diag("Performing third check after '%lf' waited seconds...", waited); + check_connection_retained(proxy_mysql, 0); + + return EXIT_SUCCESS; +} + +int check_multiplex_disabled_connection_delay_multiplex_ms(MYSQL* proxy_mysql, MYSQL* proxy_admin) { + const uint32_t timeout = 2; + string set_delay_multiplex {}; + string_format("SET mysql-connection_delay_multiplex_ms=%d", set_delay_multiplex, timeout * 1000); + const char* set_auto_inc_delay { "SET mysql-auto_increment_delay_multiplex_timeout_ms=0" }; + + diag("%s: Executing query `%s`...", tap_curtime().c_str(), set_delay_multiplex.c_str()); + MYSQL_QUERY(proxy_admin, set_delay_multiplex.c_str()); + + diag("%s: Executing query `%s`...", tap_curtime().c_str(), set_auto_inc_delay); + MYSQL_QUERY(proxy_admin, set_auto_inc_delay); + + diag("%s: Executing query `%s`...", tap_curtime().c_str(), "LOAD MYSQL VARIABLES TO RUNTIME"); + MYSQL_QUERY(proxy_admin, "LOAD MYSQL VARIABLES TO RUNTIME"); + + // Check transactions behavior and multiplex disabling actions + check_transactions_and_multiplex_disable(proxy_mysql, "DO 1", timeout); + + return EXIT_SUCCESS; +} + +int check_traffic_connection_delay_multiplex_ms(MYSQL* proxy_mysql, MYSQL* proxy_admin) { + const uint32_t timeout = 2; + const char* set_delay_multiplex_query { "SET mysql-connection_delay_multiplex_ms=2000" }; + const char* set_auto_inc_timeout_query { "SET mysql-auto_increment_delay_multiplex_timeout_ms=0" }; + + diag("%s: Executing query `%s`...", tap_curtime().c_str(), set_delay_multiplex_query); + MYSQL_QUERY(proxy_admin, set_delay_multiplex_query); + diag("%s: Executing query `%s`...", tap_curtime().c_str(), set_auto_inc_timeout_query); + MYSQL_QUERY(proxy_admin, set_auto_inc_timeout_query); + diag("%s: Executing query `%s`...", tap_curtime().c_str(), "LOAD MYSQL VARIABLES TO RUNTIME"); + MYSQL_QUERY(proxy_admin, "LOAD MYSQL VARIABLES TO RUNTIME"); + + // Retain connection in 'hg=0' + diag("Checking connection not expiring with traffic on same hostgroup..."); + + uint32_t waited = 0; + while (waited < 2*timeout) { + diag("%s: Executing query `%s`...", tap_curtime().c_str(), "DO 1"); + MYSQL_QUERY(proxy_mysql, "DO 1"); + + sleep(1); + waited += 1; + } + check_connection_retained(proxy_mysql, 1); + + diag("Check connection expiring when traffic stops to the hostgroup..."); + diag("Sleeping for '%d' seconds", timeout + 1); + sleep(timeout + 1); + check_connection_retained(proxy_mysql, 0); + + diag("Check connection expiring when traffic issued to different hostgroup..."); + + diag("%s: Executing query `%s`...", tap_curtime().c_str(), "DO 1"); + MYSQL_QUERY(proxy_mysql, "DO 1"); + diag("%s: Executing query `%s`...", tap_curtime().c_str(), "SELECT 1"); + MYSQL_QUERY(proxy_mysql, "SELECT 1"); + mysql_free_result(mysql_store_result(proxy_mysql)); + + diag("* First check that connections from both hostgroups have been retained before timeout"); + + diag("Sleeping '%d' seconds...", 1); + sleep(1); + + { + vector j_sess_backends {}; + int b_conns_res = get_session_backends(proxy_mysql, j_sess_backends); + if (b_conns_res != EXIT_SUCCESS) { + return b_conns_res; } - // Multiplex disabled by any action should take precedence over 'auto_increment_delay_multiplex_timeout_ms' - { - const char* set_query { "SET @local_var='foo'" }; - diag("%s: Executing query `%s`...", tap_curtime().c_str(), set_query); - MYSQL_QUERY(proxy_mysql, "SET @local_var='foo'"); - diag("%s: Executing query `%s`...", tap_curtime().c_str(), insert_query); - MYSQL_QUERY(proxy_mysql, insert_query); - - // Wait for the timeout and check the value - diag("%s: Waiting for timeout to expire...", tap_curtime().c_str()); - usleep(timeout_ms * 1000 + poll_timeout * 1000 + 500 * 1000 * 2); - - diag("%s: Extracting current auto inc delay...", tap_curtime().c_str()); - int cur_delay = 0; - int g_res = get_conn_auto_inc_delay_token(proxy_mysql, cur_delay); - if (g_res != EXIT_SUCCESS) { - return EXIT_FAILURE; + vector hg_ids {}; + + for (const json& j_backend : j_sess_backends) { + if (j_backend.find("hostgroup_id") != j_backend.end() && j_backend.find("conn") != j_backend.end()) { + hg_ids.push_back(j_backend.at("hostgroup_id").get()); } + } - ok( - delay == cur_delay, - "Connection should not be returned to conn_pool due to SET session var - Exp: '%d', Act: '%d'", - delay, cur_delay - ); + bool hgs_found = + std::find(hg_ids.begin(), hg_ids.end(), 0) != hg_ids.end() && + std::find(hg_ids.begin(), hg_ids.end(), 1) != hg_ids.end(); + + ok( + hgs_found && hg_ids.size() == 2, + "Found expected retained connections in target hgs - hostgroups: '%s'", + json { hg_ids }.dump().c_str() + ); + } + + // Check for connections retained in 'hg 0' + waited = 0; + while (waited < timeout * 2) { + diag("%s: Executing query `%s`...", tap_curtime().c_str(), "SELECT 1"); + MYSQL_QUERY(proxy_mysql, "SELECT 1"); + mysql_free_result(mysql_store_result(proxy_mysql)); + + sleep(1); + waited += 1; + } + + diag("* Check that connections from hostgroup '0' (not traffic) is expired after timeout"); + + { + vector j_sess_backends {}; + int b_conns_res = get_session_backends(proxy_mysql, j_sess_backends); + if (b_conns_res != EXIT_SUCCESS) { + return b_conns_res; } + + vector hg_ids {}; + + for (const json& j_backend : j_sess_backends) { + if (j_backend.find("hostgroup_id") != j_backend.end() && j_backend.find("conn") != j_backend.end()) { + hg_ids.push_back(j_backend.at("hostgroup_id").get()); + } + } + + bool hgs_found = std::find(hg_ids.begin(), hg_ids.end(), 1) != hg_ids.end(); + + ok( + hgs_found && hg_ids.size() == 1, + "Found expected retained connections in target hgs - hostgroups: '%s'", + json { hg_ids }.dump().c_str() + ); } -cleanup: + return EXIT_SUCCESS; +} + +int check_auto_inc_delay_and_conn_delay_multiplex(MYSQL* proxy_mysql, MYSQL* proxy_admin) { + uint64_t poll_timeout = 0; + const string poll_timeout_query { "SELECT variable_value FROM global_variables WHERE variable_name='mysql-poll_timeout'" }; + string auto_inc_timeout_query {}; + + int g_res = get_query_result(proxy_admin, poll_timeout_query.c_str(), poll_timeout); + if (g_res != EXIT_SUCCESS) { return EXIT_FAILURE; } + + const uint32_t timeout = 2; + const char* set_delay_multiplex_query { "SET mysql-connection_delay_multiplex_ms=2000" }; + const char* set_auto_inc_timeout_query { "SET mysql-auto_increment_delay_multiplex_timeout_ms=0" }; + + diag("%s: Executing query `%s`...", tap_curtime().c_str(), set_delay_multiplex_query); + MYSQL_QUERY(proxy_admin, set_delay_multiplex_query); + diag("%s: Executing query `%s`...", tap_curtime().c_str(), set_auto_inc_timeout_query); + MYSQL_QUERY(proxy_admin, set_auto_inc_timeout_query); + diag("%s: Executing query `%s`...", tap_curtime().c_str(), "LOAD MYSQL VARIABLES TO RUNTIME"); + MYSQL_QUERY(proxy_admin, "LOAD MYSQL VARIABLES TO RUNTIME"); + + // Retain connection in 'hg=0' + diag("Checking connection not expiring due to 'auto_increment_delay_multiplex'."); + + uint32_t waited = 0; + diag("%s: Executing query `%s`...", tap_curtime().c_str(), INSERT_QUERY); + MYSQL_QUERY(proxy_mysql, INSERT_QUERY); + + diag("* Check connection retained after executing the query"); + check_connection_retained(proxy_mysql, 1); + + diag("* Check connection NOT expiring after imposed timeout '%d'.", timeout); + diag("Sleeping for '%d' seconds", timeout + 1); + sleep(timeout + 1); + + check_connection_retained(proxy_mysql, 1); + + diag( + "Checking interaction with 'auto_increment_delay_multiplex_timeout_ms' -" + " 'auto_increment_delay_multiplex_timeout_ms' is the higher value" + ); + + string_format("SET mysql-auto_increment_delay_multiplex_timeout_ms=%d", auto_inc_timeout_query, timeout*2*1000); + diag("%s: Executing query `%s`...", tap_curtime().c_str(), auto_inc_timeout_query.c_str()); + MYSQL_QUERY(proxy_admin, auto_inc_timeout_query.c_str()); + + diag("%s: Executing query `%s`...", tap_curtime().c_str(), "LOAD MYSQL VARIABLES TO RUNTIME"); + MYSQL_QUERY(proxy_admin, "LOAD MYSQL VARIABLES TO RUNTIME"); + + diag("%s: Executing query `%s`...", tap_curtime().c_str(), INSERT_QUERY); + MYSQL_QUERY(proxy_mysql, INSERT_QUERY); + + diag("Sleeping for '%d' seconds", timeout + 1); + sleep(timeout + 1); + + diag( + "Connection SHOULDN'T be returned due because:" + " auto_increment_delay_multiplex_timeout_ms='%d', connection_delay_multiplex_ms='%d', waited='%d'", + timeout*2, timeout, timeout + 1 + ); + + check_connection_retained(proxy_mysql, 1); + + diag("Sleeping for '%d' seconds", timeout + 1); + sleep(timeout + 1 ); + + diag( + "Connection SHOULD be returned due because:" + " auto_increment_delay_multiplex_timeout_ms='%d', connection_delay_multiplex_ms='%d', waited='%d'", + timeout*2, timeout, (timeout + 1) * 2 + ); + + check_connection_retained(proxy_mysql, 0); + + diag( + "Checking interaction with 'auto_increment_delay_multiplex_timeout_ms' -" + " 'auto_increment_delay_multiplex_timeout_ms' is the smaller value. Higher value should prevail." + ); + + string_format("SET mysql-auto_increment_delay_multiplex_timeout_ms=%d", auto_inc_timeout_query, timeout*1000); + diag("%s: Executing query `%s`...", tap_curtime().c_str(), auto_inc_timeout_query.c_str()); + MYSQL_QUERY(proxy_admin, auto_inc_timeout_query.c_str()); + + const char* set_delay_multiplex_query_2 { "SET mysql-connection_delay_multiplex_ms=4000" }; + diag("%s: Executing query `%s`...", tap_curtime().c_str(), set_delay_multiplex_query_2); + MYSQL_QUERY(proxy_admin, set_delay_multiplex_query_2); + + diag("%s: Executing query `%s`...", tap_curtime().c_str(), "LOAD MYSQL VARIABLES TO RUNTIME"); + MYSQL_QUERY(proxy_admin, "LOAD MYSQL VARIABLES TO RUNTIME"); + + diag("%s: Executing query `%s`...", tap_curtime().c_str(), INSERT_QUERY); + MYSQL_QUERY(proxy_mysql, INSERT_QUERY); + + diag("Sleeping for '%d' seconds", timeout + 1); + sleep(timeout + 1); + + diag( + "Connection SHOULDN'T be returned due because:" + " auto_increment_delay_multiplex_timeout_ms='%d', connection_delay_multiplex_ms='%d', waited='%d'", + timeout*2*1000, timeout*1000, timeout + 1 + ); + + check_connection_retained(proxy_mysql, 1); + + diag("Sleeping for '%d' seconds", timeout + 1); + sleep(timeout + 1); + + diag( + "Connection SHOULD be returned due because:" + " auto_increment_delay_multiplex_timeout_ms='%d', connection_delay_multiplex_ms='%d', waited='%d'", + timeout*2*1000, timeout*1000, (timeout + 1) * 2 + ); + + check_connection_retained(proxy_mysql, 0); + + // Check transactions behavior and multiplex disabling actions with both 'connection_delay_multiplex_ms' + // and 'auto_increment_delay_multiplex_timeout_ms' enabled. + uint64_t higher_timeout = 4; + check_transactions_and_multiplex_disable(proxy_mysql, INSERT_QUERY, higher_timeout); + + return EXIT_SUCCESS; +} + +const vector> auto_increment_delay_multiplex_tests { + // 1. Check that the required variables are present + check_variables_config, + // 2. Change and check 'auto_increment_delay_multiplex' behavior + check_auto_increment_delay_multiplex, + // 3. Change and check 'auto_increment_delay_multiplex_timeout_ms' behavior + check_auto_increment_delay_multiplex_timeout +}; + +const vector> conn_delay_multiplex_tests { + // 4. Test 'connection_delay_multiplex_ms' retaining and expiring connections + check_connection_delay_multiplex_ms, + // 5. Test 'connection_delay_multiplex_ms' integration with multiplexing disabling operations. + check_multiplex_disabled_connection_delay_multiplex_ms, + // 5. Test 'connection_delay_multiplex_ms' integration with traffic hitting the session. + check_traffic_connection_delay_multiplex_ms, + // 7. Test 'connection_delay_multiplex_ms' interaction with 'auto_increment_delay_multiplex_timeout_ms' + check_auto_inc_delay_and_conn_delay_multiplex +}; + +int main(int argc, char** argv) { + CommandLine cl; + + if (cl.getEnv()) { + diag("Failed to get the required environmental variables."); + return EXIT_FAILURE; + } + + plan( + 1 + // Check variables are present + ((VAL_RANGE / STEP) + 1) * 2 + // Tests for different 'auto_increment_delay_multiplex' values + (VAL_RANGE / STEP) * 3 + // Tests for different 'auto_increment_delay_multiplex_timeout_ms' values + 3 + // Tests for 'auto_increment_delay_multiplex_timeout_ms' zero value + 2 + // Tests for 'auto_increment_delay_multiplex_timeout_ms' keep alive queries + 2 + // Tests for 'auto_increment_delay_multiplex_timeout_ms' transaction behavior + 1 + // Tests for 'auto_increment_delay_multiplex_timeout_ms' multiplex disabled by SET statement + 23 // Tests for 'connection_delay_multiplex_ms' and also tests for + // integration with 'auto_increment_delay_multiplex_timeout_ms'. + ); + + MYSQL* proxy_mysql = mysql_init(NULL); + MYSQL* proxy_admin = mysql_init(NULL); + + if (!mysql_real_connect(proxy_mysql, cl.host, cl.username, cl.password, NULL, cl.port, NULL, 0)) { + fprintf(stderr, "File %s, line %d, Error: \"%s\"\n", __FILE__, __LINE__, mysql_error(proxy_mysql)); + return EXIT_FAILURE; + } + if (!mysql_real_connect(proxy_admin, cl.host, cl.admin_username, cl.admin_password, NULL, cl.admin_port, NULL, 0)) { + fprintf(stderr, "File %s, line %d, Error: \"%s\"\n", __FILE__, __LINE__, mysql_error(proxy_admin)); + return EXIT_FAILURE; + } + + MYSQL_QUERY(proxy_mysql, "CREATE DATABASE IF NOT EXISTS test"); + MYSQL_QUERY(proxy_mysql, "DROP TABLE IF EXISTS test.auto_inc_multiplex"); + MYSQL_QUERY(proxy_mysql, CREATE_TABLE_QUERY); + + for (const function& test : auto_increment_delay_multiplex_tests) { + if (test(proxy_mysql, proxy_admin) != EXIT_SUCCESS) { + break; + } + } mysql_close(proxy_admin); - mysql_close(proxy_mysql); + + for (const function& test : conn_delay_multiplex_tests) { + proxy_mysql = mysql_init(NULL); + proxy_admin = mysql_init(NULL); + + if (!mysql_real_connect(proxy_mysql, cl.host, cl.username, cl.password, NULL, cl.port, NULL, 0)) { + fprintf(stderr, "File %s, line %d, Error: \"%s\"\n", __FILE__, __LINE__, mysql_error(proxy_mysql)); + return EXIT_FAILURE; + } + if (!mysql_real_connect(proxy_admin, cl.host, cl.admin_username, cl.admin_password, NULL, cl.admin_port, NULL, 0)) { + fprintf(stderr, "File %s, line %d, Error: \"%s\"\n", __FILE__, __LINE__, mysql_error(proxy_admin)); + return EXIT_FAILURE; + } + + test(proxy_mysql, proxy_admin); + + mysql_close(proxy_admin); + mysql_close(proxy_mysql); + } + +cleanup: return exit_status(); }