Merge pull request #3611 from sysown/v2.2.2-3591

RESTAPI interface is unresponsive when ProxySQL has ~1000+ sockets open

lib/Makefile

@@ -101,9 +101,17 @@ ifeq ($(UNAME_S),Darwin)
 	IDIRS+= -I/usr/local/opt/openssl/include
 endif
 
+# 'libhttpserver': Add 'ENABLE_EPOLL' by default for all platforms except
+# for 'Darwin'. This is required when compiling 'libhttpserver' for avoiding
+# internal use of 'SELECT' in favor of 'EPOLL'. See #3591.
+ifeq ($(UNAME_S),Darwin)
+ENABLE_EPOLL=
+else
+ENABLE_EPOLL=-DENABLE_EPOLL
+endif
 
-MYCFLAGS=$(IDIRS) $(OPTZ) $(DEBUG) -Wall -DGITVERSION=\"$(GIT_VERSION)\" $(NOJEM) $(WASAN)
-MYCXXFLAGS=-std=c++11 $(MYCFLAGS) $(PSQLCH)
+MYCFLAGS=$(IDIRS) $(OPTZ) $(DEBUG) -Wall -DGITVERSION=\"$(GIT_VERSION)\" $(NOJEM) $(WGCOV) $(WASAN)
+MYCXXFLAGS=-std=c++11 $(MYCFLAGS) $(PSQLCH) $(ENABLE_EPOLL)
 
 default: libproxysql.a
 .PHONY: default

test/tap/tap/utils.cpp

@@ -473,3 +473,85 @@ int exec(const std::string& cmd, std::string& result) {
 	}
 	return err;
 }
+
+std::vector<mysql_res_row> extract_mysql_rows(MYSQL_RES* my_res) {
+	if (my_res == nullptr) { return {}; }
+
+	std::vector<mysql_res_row> result {};
+	MYSQL_ROW row = nullptr;
+	uint32_t num_fields = mysql_num_fields(my_res);
+
+	while ((row = mysql_fetch_row(my_res))) {
+		mysql_res_row row_values {};
+		uint64_t *lengths = mysql_fetch_lengths(my_res);
+
+		for (uint32_t i = 0; i < num_fields; i++) {
+			std::string field_val(row[i], lengths[i]);
+			row_values.push_back(field_val);
+		}
+
+		result.push_back(row_values);
+	}
+
+	return result;
+};
+
+size_t my_dummy_write(char*, size_t size, size_t nmemb, void*) {
+	return size * nmemb;
+}
+
+CURLcode perform_simple_post(
+	const std::string& endpoint, const std::string& post_params,
+	uint64_t& curl_res_code, std::string& curl_out_err
+) {
+	CURL *curl;
+	CURLcode res;
+
+	curl_global_init(CURL_GLOBAL_ALL);
+
+	curl = curl_easy_init();
+	if(curl) {
+		curl_easy_setopt(curl, CURLOPT_URL, endpoint.c_str());
+		curl_easy_setopt(curl, CURLOPT_POSTFIELDS, post_params.c_str());
+		curl_easy_setopt(curl, CURLOPT_WRITEFUNCTION, &my_dummy_write);
+
+		res = curl_easy_perform(curl);
+
+		if(res != CURLE_OK) {
+			curl_out_err = std::string { curl_easy_strerror(res) };
+		} else {
+			curl_easy_getinfo(curl, CURLINFO_RESPONSE_CODE, &curl_res_code);
+		}
+
+		curl_easy_cleanup(curl);
+	}
+
+	return res;
+}
+
+int wait_until_enpoint_ready(
+	std::string endpoint, std::string post_params, uint32_t timeout, uint32_t delay
+) {
+	double waited = 0;
+	int res = -1;
+
+	while (waited < timeout) {
+		std::string curl_str_err {};
+		uint64_t curl_res_code = 0;
+		int curl_err = perform_simple_post(endpoint, post_params, curl_res_code, curl_str_err);
+
+		if (curl_err != CURLE_OK) {
+			diag(
+				"'curl_err_code': %d, 'curl_err': '%s', waiting for '%d'ms...",
+				curl_err, curl_str_err.c_str(), delay
+			);
+			waited += static_cast<double>(delay) / 1000;
+			usleep(delay * 1000);
+		} else {
+			res = 0;
+			break;
+		}
+	}
+
+	return res;
+}

test/tap/tap/utils.h

@@ -7,6 +7,8 @@
 #include <sstream>
 #include <vector>
 
+#include <curl/curl.h>
+
 #define MYSQL_QUERY(mysql, query) \
 	do { \
 		if (mysql_query(mysql, query)) { \
@@ -72,4 +74,60 @@ int exec(const std::string& cmd, std::string& result);
 int create_table_test_sbtest1(int num_rows, MYSQL *mysql);
 int add_more_rows_test_sbtest1(int num_rows, MYSQL *mysql);
 
+using mysql_res_row = std::vector<std::string>;
+
+/**
+ * @brief Function that extracts the provided 'MYSQL_RES' into a vector of vector of
+ *   strings.
+ * @param my_res The 'MYSQL_RES' for which to extract the values. In case of
+ *   being NULL an empty vector is returned.
+ * @return The extracted values of all the rows present in the resultset.
+ */
+std::vector<mysql_res_row> extract_mysql_rows(MYSQL_RES* my_res);
+
+/**
+ * @brief Dummy write function to avoid CURL to write received output to stdout.
+ * @return Returns the size presented.
+ */
+size_t my_dummy_write(char*, size_t size, size_t nmemb, void*);
+
+/**
+ * @brief Waits until the provided endpoint is ready to be used or the
+ *   timeout period expired. For this checks the return code of
+ *   'perform_simple_post' which only fails in case the 'CURL' request couldn't
+ *   be performed, which is interpreted as endpoint not being yet ready.
+ *
+ * @param endpoint The endpoint to be queried.
+ * @param post_params The required params to be supplied for the 'POST' endpoint
+ *   call.
+ * @param timeout The max time to wait before declaring a timeout, and
+ *   returning '-1'.
+ * @param delay The delay specified in 'ms' to be waited between retries.
+ *
+ * @return '0' in case the endpoint became available before the timeout, or
+ *   '-1' in case the timeout expired.
+ */
+int wait_until_enpoint_ready(
+	std::string endpoint, std::string post_params, uint32_t timeout, uint32_t delay=100
+);
+
+/**
+ * @brief Perform a simple POST query to the specified endpoint using the supplied
+ *  'post_params'.
+ *
+ * @param endpoint The endpoint to be exercised by the POST.
+ * @param post_params The post parameters to be supplied to the script.
+ * @param curl_out_err A uint64_t reference returning the result code of the
+ *   query in case it has been performed. In case the query couldn't be
+ *   performed, this value is never initialized.
+ * @param curl_out_err A string reference to collect the error as a string reported
+ *   by 'libcurl' in case of failure.
+ *
+ * @return The response code of the query in case of the query.
+ */
+CURLcode perform_simple_post(
+	const std::string& endpoint, const std::string& post_params,
+	uint64_t& curl_res_code, std::string& curl_out_err
+);
+
 #endif // #define UTILS_H

test/tap/tests/reg_test_3591-restapi_num_fds-t.cpp

@@ -0,0 +1,92 @@
+/**
+ * @file reg_test_3591-restapi_num_fds-t.cpp
+ * @brief This is a regression test for issue #3591. The test checks that
+ *   ProxySQL metrics endpoint can be enabled and it's functional when
+ *   '2047' connections are oppened against it.
+ *
+ * @details The tests creates a higher number of connections than the default
+ *   maximum number of file descriptors determined by `FD_SETSIZE` (1024).
+ *   After doing this, it tries to enable the 'RESTAPI' and checks that the
+ *   endpoint is functional.
+ */
+
+#include <cstring>
+#include <vector>
+#include <string>
+#include <stdio.h>
+#include <unistd.h>
+
+#include <mysql.h>
+
+#include "tap.h"
+#include "command_line.h"
+#include "utils.h"
+#include "json.hpp"
+
+#include <sys/time.h>
+#include <sys/resource.h>
+
+using nlohmann::json;
+using std::string;
+
+const int NUM_CONNECTIONS = 2047;
+
+int main(int argc, char** argv) {
+	CommandLine cl;
+
+	if (cl.getEnv()) {
+		diag("Failed to get the required environmental variables.");
+		return -1;
+	}
+	struct rlimit limits { 0, 0 };
+	getrlimit(RLIMIT_NOFILE, &limits);
+	diag("Old process limits: { %ld, %ld }", limits.rlim_cur, limits.rlim_max);
+	limits.rlim_cur = NUM_CONNECTIONS * 2;
+	setrlimit(RLIMIT_NOFILE, &limits);
+	diag("New process limits: { %ld, %ld }", limits.rlim_cur, limits.rlim_max);
+
+	MYSQL* proxysql_admin = mysql_init(NULL);
+
+	// Initialize connections
+	if (!proxysql_admin) {
+		fprintf(stderr, "File %s, line %d, Error: %s\n", __FILE__, __LINE__, mysql_error(proxysql_admin));
+		return -1;
+	}
+
+	if (!mysql_real_connect(proxysql_admin, cl.host, cl.admin_username, cl.admin_password, NULL, cl.admin_port, NULL, 0)) {
+		fprintf(stderr, "File %s, line %d, Error: %s\n", __FILE__, __LINE__, mysql_error(proxysql_admin));
+		return -1;
+	}
+
+	// Enable 'RESTAPI'
+	MYSQL_QUERY(proxysql_admin, "SET admin-restapi_enabled='true'");
+	MYSQL_QUERY(proxysql_admin, "SET admin-restapi_port=6070");
+	MYSQL_QUERY(proxysql_admin, "LOAD ADMIN VARIABLES TO RUNTIME");
+
+	std::vector<MYSQL*> mysql_connections {};
+
+	for (int i = 0; i < NUM_CONNECTIONS; i++) {
+		MYSQL* proxysql_mysql = mysql_init(NULL);
+		if (
+			!mysql_real_connect(
+				proxysql_mysql, cl.host, cl.username, cl.password, NULL, cl.port, NULL, 0
+			)
+		) {
+			fprintf(
+				stderr, "File %s, line %d, Error: %s\n", __FILE__, __LINE__,
+				mysql_error(proxysql_mysql)
+			);
+			return EXIT_FAILURE;
+		}
+		mysql_connections.push_back(proxysql_mysql);
+	}
+
+	int endpoint_timeout = wait_until_enpoint_ready("http://localhost:6070/metrics/", "{}", 10, 500);
+	ok(endpoint_timeout == 0, "The endpoint should be available instead of timing out.");
+
+	for (int i = 0; i < NUM_CONNECTIONS; i++) {
+		mysql_close(mysql_connections[i]);
+	}
+
+	return exit_status();
+}