diff --git a/src/.clang-format b/.clang-format similarity index 100% rename from src/.clang-format rename to .clang-format diff --git a/create-new-actor.sh b/create-new-actor.sh index e73ecc0abe..4bd63a98d5 100755 --- a/create-new-actor.sh +++ b/create-new-actor.sh @@ -12,13 +12,13 @@ usage() { } create_header_text() { - local uuid + local uuid_tag local actor_name - uuid="$1" + uuid_tag="$1" actor_name="$2" - echo "#ifndef HEADER_$uuid" - echo "#define HEADER_$uuid" + echo "#ifndef $uuid_tag" + echo "#define $uuid_tag" echo "" echo "#include " echo "#include " @@ -46,13 +46,13 @@ create_header_text() { echo "" echo "} // namespace genny::actor" echo "" - echo "#endif // HEADER_$uuid" + echo "#endif // $uuid_tag" } create_impl_text() { - local uuid + local uuid_tag local actor_name - uuid="$1" + uuid_tag="$1" actor_name="$2" echo "#include " @@ -84,28 +84,28 @@ create_impl_text() { } create_header() { - local uuid + local uuid_tag local actor_name - uuid="$1" + uuid_tag="$1" actor_name="$2" create_header_text "$@" > "$(dirname "$0")/src/cast_core/include/cast_core/actors/${actor_name}.hpp" } create_impl() { - local uuid + local uuid_tag local actor_name - uuid="$1" + uuid_tag="$1" actor_name="$2" create_impl_text "$@" > "$(dirname "$0")/src/cast_core/src/actors/${actor_name}.cpp" } recreate_cast_core_cmake_file() { - local uuid + local uuid_tag local actor_name local cmake_file - uuid="$1" + uuid_tag="$1" actor_name="$2" cmake_file="$(dirname "$0")/src/cast_core/CMakeLists.txt" @@ -131,8 +131,8 @@ if [ -z "$actor_name" ]; then exit 2 fi -uuid="$(uuidgen | sed s/-/_/g)" +uuid_tag="$("$(dirname "$0")/generate-uuid-tag.sh")" -create_header "$uuid" "$actor_name" -create_impl "$uuid" "$actor_name" -recreate_cast_core_cmake_file "$uuid" "$actor_name" +create_header "$uuid_tag" "$actor_name" +create_impl "$uuid_tag" "$actor_name" +recreate_cast_core_cmake_file "$uuid_tag" "$actor_name" diff --git a/generate-uuid-tag.sh b/generate-uuid-tag.sh new file mode 100755 index 0000000000..0fbd8e7ded --- /dev/null +++ b/generate-uuid-tag.sh @@ -0,0 +1,4 @@ +#/bin/bash + +UUID=$(uuidgen | sed s/-/_/g | tr a-z A-Z) +echo "HEADER_${UUID}" diff --git a/src/gennylib/CMakeLists.txt b/src/gennylib/CMakeLists.txt index 7249ad2829..a4fa311604 100644 --- a/src/gennylib/CMakeLists.txt +++ b/src/gennylib/CMakeLists.txt @@ -21,6 +21,7 @@ add_library(gennylib SHARED src/ActorProducer.cpp src/Cast.cpp src/Orchestrator.cpp + src/PoolFactory.cpp src/value_generators/parser.cpp src/value_generators/value_generators.cpp @@ -71,6 +72,7 @@ add_executable(test_gennylib test/Cast_test.cpp test/context_test.cpp test/conventions_test.cpp + test/PoolFactory_test.cpp test/PhaseLoop_test.cpp test/version_test.cpp test/metrics_test.cpp diff --git a/src/gennylib/include/gennylib/PoolFactory.hpp b/src/gennylib/include/gennylib/PoolFactory.hpp new file mode 100644 index 0000000000..e6ce838df5 --- /dev/null +++ b/src/gennylib/include/gennylib/PoolFactory.hpp @@ -0,0 +1,53 @@ +#ifndef HEADER_3BB17688_900D_4AFB_B736_C9EC8DA9E33B +#define HEADER_3BB17688_900D_4AFB_B736_C9EC8DA9E33B + +#include +#include +#include +#include +#include +#include + +#include + +namespace genny { + +/** + * A pool factory takes in a MongoURI, modifies its components, and makes a pool from it + * + * This class allows for programically modifying all non-host components of the MongoURI. + * Any query parameter can be set via `setStringOption()`, `setIntOption()`, or `setFlag()`. + * It also allows for setting the protocol, username, password, and database via the options + * "Protocol", "Username", "Password", and "Database" in the same manner as query parameters would + * be set. Lastly, it allows for programatically setting up the ssl options for the connection pool + * via `configureSsl()`. + */ +class PoolFactory { +public: + PoolFactory(std::string_view uri); + ~PoolFactory(); + + std::string makeUri() const; + std::unique_ptr makePool() const; + + /** + * Options of note: + * minPoolSize + * maxPoolSize + * connectTimeoutMS + * socketTimeoutMS + */ + void setStringOption(const std::string & option, std::string value); + void setIntOption(const std::string & option, int32_t value); + void setFlag(const std::string & option, bool value = true); + + void configureSsl(mongocxx::options::ssl options, bool enableSsl = true); + +private: + struct Config; + std::unique_ptr _config; +}; + +} // namespace genny + +#endif // HEADER_3BB17688_900D_4AFB_B736_C9EC8DA9E33B diff --git a/src/gennylib/src/PoolFactory.cpp b/src/gennylib/src/PoolFactory.cpp new file mode 100644 index 0000000000..e480d4a66a --- /dev/null +++ b/src/gennylib/src/PoolFactory.cpp @@ -0,0 +1,150 @@ +#include + +#include +#include +#include +#include +#include + +#include +#include + +#include + +namespace genny { + +struct PoolFactory::Config { + Config(std::string_view uri) { + const auto protocolRegex = std::regex("^(mongodb://|mongodb+srv://)?(([^:@]*):([^@]*)@)?"); + const auto hostRegex = std::regex("^,?([^:,/]+(:[0-9]+)?)"); + const auto dbRegex = std::regex("^/([^?]*)\\??"); + const auto queryRegex = std::regex("^&?([^=&]*)=([^&]*)"); + + std::cmatch matches; + auto i = 0; + + // Extract the protocol, and optionally the username and the password + std::regex_search(uri.substr(i).data(), matches, protocolRegex); + if (matches.length(1)) { + accessOptions["Protocol"] = matches[1]; + } else { + accessOptions["Protocol"] = "mongodb://"; + } + if (matches.length(3)) + accessOptions["Username"] = matches[3]; + if (matches.length(4)) + accessOptions["Password"] = matches[4]; + i += matches.length(); + + // Extract each host specified in the uri + while (std::regex_search(uri.substr(i).data(), matches, hostRegex)) { + hosts.insert(matches[1]); + i += matches.length(); + } + + // Extract the db name and optionally the query string prefix + std::regex_search(uri.substr(i).data(), matches, dbRegex); + accessOptions["Database"] = matches[1]; + i += matches.length(); + + // Extract each query parameter + // Note that the official syntax of query strings is poorly defined, keys without values may + // be valid but not supported here. + while (std::regex_search(uri.substr(i).data(), matches, queryRegex)) { + auto key = matches[1]; + auto value = matches[2]; + queryOptions[key] = value; + i += matches.length(); + } + } + + auto makeUri() const { + std::ostringstream ss; + size_t i; + + ss << accessOptions.at("Protocol"); + if (!accessOptions.at("Username").empty()) { + ss << accessOptions.at("Username") << ':' << accessOptions.at("Password") << '@'; + } + + i = 0; + for (auto& host : hosts) { + if (i++ > 0) { + ss << ','; + } + ss << host; + } + + auto dbName = accessOptions.at("Database"); + if (!dbName.empty() || !queryOptions.empty()) { + ss << '/' << accessOptions.at("Database"); + } + + if (!queryOptions.empty()) { + ss << '?'; + } + + i = 0; + for (auto && [ key, value ] : queryOptions) { + if (i++ > 0) { + ss << '&'; + } + ss << key << '=' << value; + } + + return ss.str(); + } + + std::set hosts; + std::map queryOptions; + std::map accessOptions = { + {"Protocol", ""}, {"Username", ""}, {"Password", ""}, {"Database", ""}, + }; + mongocxx::options::pool poolOptions; +}; + +PoolFactory::PoolFactory(std::string_view rawUri) : _config(std::make_unique(rawUri)) {} +PoolFactory::~PoolFactory() {} + +std::string PoolFactory::makeUri() const { + return _config->makeUri(); +} + +std::unique_ptr PoolFactory::makePool() const { + auto uriStr = makeUri(); + BOOST_LOG_TRIVIAL(info) << "Constructing pool with MongoURI '" << uriStr << "'"; + + auto uri = mongocxx::uri{uriStr}; + return std::make_unique(uri, _config->poolOptions); +} + +void PoolFactory::setStringOption(const std::string& option, std::string value) { + // If the value is in the accessOptions set, set it + auto it = _config->accessOptions.find(option); + if (it != _config->accessOptions.end()) { + it->second = value; + return; + } + + // Treat the value as a normal query parameter + _config->queryOptions[option] = value; +} + +void PoolFactory::setIntOption(const std::string& option, int32_t value) { + auto valueStr = std::to_string(value); + setStringOption(option, valueStr); +} + +void PoolFactory::setFlag(const std::string& option, bool value) { + auto valueStr = value ? "true" : "false"; + setStringOption(option, valueStr); +} + +void PoolFactory::configureSsl(mongocxx::options::ssl options, bool enableSsl) { + setFlag("ssl", enableSsl); + + auto clientOpts = _config->poolOptions.client_opts(); + _config->poolOptions = clientOpts.ssl_opts(options); +} + +} // namespace genny diff --git a/src/gennylib/src/context.cpp b/src/gennylib/src/context.cpp index 107851477b..821d3e9218 100644 --- a/src/gennylib/src/context.cpp +++ b/src/gennylib/src/context.cpp @@ -8,6 +8,7 @@ #include #include +#include namespace genny { @@ -27,7 +28,8 @@ WorkloadContext::WorkloadContext(YAML::Node node, mongocxx::instance::current(); // TODO: make this optional and default to mongodb://localhost:27017 - _clientPool = std::make_unique(mongocxx::uri{mongoUri}); + auto poolFactory = PoolFactory(mongoUri); + _clientPool = poolFactory.makePool(); // Make a bunch of actor contexts for (const auto& actor : get_static(node, "Actors")) { diff --git a/src/gennylib/test/PoolFactory_test.cpp b/src/gennylib/test/PoolFactory_test.cpp new file mode 100644 index 0000000000..f24b9880e9 --- /dev/null +++ b/src/gennylib/test/PoolFactory_test.cpp @@ -0,0 +1,196 @@ +#include "test.h" + +#include + +#include +#include + +namespace Catchers = Catch::Matchers; + +TEST_CASE("PoolFactory behavior") { + mongocxx::instance::current(); + + // Testing out core features of the PoolFactory + SECTION("Make a few trivial localhost pools") { + constexpr auto kSourceUri = "mongodb://127.0.0.1:27017"; + + auto factory = genny::PoolFactory(kSourceUri); + + auto factoryUri = factory.makeUri(); + REQUIRE(factoryUri == kSourceUri); + + auto pool = factory.makePool(); + REQUIRE(pool); + + // We should be able to get more from the same factory + auto extraPool = factory.makePool(); + REQUIRE(extraPool); + } + + SECTION("Make a pool with the bare minimum uri") { + constexpr auto kSourceUri = "127.0.0.1"; + + auto factory = genny::PoolFactory(kSourceUri); + + auto factoryUri = factory.makeUri(); + auto expectedUri = [&]() { return std::string{"mongodb://"} + kSourceUri; }; + REQUIRE(factoryUri == expectedUri()); + + auto pool = factory.makePool(); + REQUIRE(pool); + } + + SECTION("Replace the replset and the db") { + constexpr auto kSourceUri = "mongodb://127.0.0.1/bigdata?replicaSet=badChoices"; + + const std::string kBaseString = "mongodb://127.0.0.1/"; + + auto factory = genny::PoolFactory(kSourceUri); + + SECTION("Validate the original URI") { + auto factoryUri = factory.makeUri(); + auto expectedUri = [&]() { return kBaseString + "bigdata?replicaSet=badChoices"; }; + REQUIRE(factoryUri == expectedUri()); + + auto pool = factory.makePool(); + REQUIRE(pool); + } + + SECTION("Modify the URI and check that it works") { + auto expectedUri = [&]() { return kBaseString + "webscale?replicaSet=threeNode"; }; + factory.setStringOption("replicaSet", "threeNode"); + factory.setStringOption("Database", "webscale"); + + auto factoryUri = factory.makeUri(); + REQUIRE(factoryUri == expectedUri()); + + auto pool = factory.makePool(); + REQUIRE(pool); + } + } + + SECTION("Try various set() commands with odd cases") { + const std::string kBaseString = "mongodb://127.0.0.1/"; + constexpr auto kOriginalDatabase = "admin"; + + SECTION("Use the wrong case for 'Database' option") { + auto sourceUri = [&]() { return kBaseString + kOriginalDatabase; }; + auto factory = genny::PoolFactory(sourceUri()); + + auto expectedUri = [&]() { return sourceUri() + "?database=test"; }; + factory.setStringOption("database", "test"); + + auto factoryUri = factory.makeUri(); + REQUIRE(factoryUri == expectedUri()); + } + + // Funny enough, going through MongoURI means we convert to strings. + // So we can set access options like 'Database' through functions we would + // not normally consider for traditional string flags + SECTION("Set the 'Database' option in odd ways") { + auto sourceUri = [&]() { return kBaseString + kOriginalDatabase; }; + auto factory = genny::PoolFactory(sourceUri()); + + + SECTION("Use the flag option") { + auto expectedUri = [&]() { return kBaseString + "true"; }; + factory.setFlag("Database"); + + auto factoryUri = factory.makeUri(); + REQUIRE(factoryUri == expectedUri()); + } + + SECTION("Use the string option to reset the Database") { + auto expectedUri = [&]() { return kBaseString + "true"; }; + factory.setStringOption("Database", "true"); + + auto factoryUri = factory.makeUri(); + REQUIRE(factoryUri == expectedUri()); + } + + SECTION("Use the flag option to flip the Database") { + auto expectedUri = [&]() { return kBaseString + "false"; }; + factory.setFlag("Database", false); + + auto factoryUri = factory.makeUri(); + REQUIRE(factoryUri == expectedUri()); + } + } + + SECTION("Overwrite the replSet option in a variety of ways") { + auto sourceUri = [&]() { return kBaseString + "?replSet=red"; }; + auto factory = genny::PoolFactory(sourceUri()); + + SECTION("Overwrite with a normal string") { + auto expectedUri = [&]() { return kBaseString + "?replSet=blue"; }; + factory.setStringOption("replSet", "blue"); + + auto factoryUri = factory.makeUri(); + REQUIRE(factoryUri == expectedUri()); + } + + SECTION("Overwrite with an empty string") { + // An empty string is still a valid option, even if not a valid replset + auto expectedUri = [&]() { return kBaseString + "?replSet="; }; + factory.setStringOption("replSet", ""); + + auto factoryUri = factory.makeUri(); + REQUIRE(factoryUri == expectedUri()); + } + } + } + + // Moving on to actual pool cases + SECTION("Make a pool with a severely limited max size") { + const std::string kSourceUri = "mongodb://127.0.0.1"; + constexpr int32_t kMaxPoolSize = 2; + + auto factory = genny::PoolFactory(kSourceUri); + + auto expectedUri = [&]() { return kSourceUri + "/?maxPoolSize=2"; }; + factory.setIntOption("maxPoolSize", kMaxPoolSize); + + auto factoryUri = factory.makeUri(); + REQUIRE(factoryUri == expectedUri()); + + auto pool = factory.makePool(); + REQUIRE(pool); + + std::vector> clients; + for (int i = 0; i < kMaxPoolSize; ++i) { + auto client = pool->try_acquire(); + auto gotClient = static_cast(client); + REQUIRE(gotClient); + + clients.emplace_back(std::make_unique(std::move(*client))); + } + + // We should be full up now + auto getOneMore = [&]() { return !static_cast(pool->try_acquire()); }; + REQUIRE(getOneMore()); + } + + SECTION("Make a pool with ssl enabled and auth params") { + const std::string kProtocol = "mongodb://"; + const std::string kHost = "127.0.0.1"; + + auto sourceUrl = [&]() { return kProtocol + kHost; }; + auto factory = genny::PoolFactory(sourceUrl()); + + auto expectedUri = [&]() { return kProtocol + "boss:pass@" + kHost + "/admin?ssl=true"; }; + factory.setStringOption("Username", "boss"); + factory.setStringOption("Password", "pass"); + factory.setStringOption("Database", "admin"); + + // This won't actually work for real, but it does test the interface + mongocxx::options::ssl sslOpts; + sslOpts.allow_invalid_certificates(true); + factory.configureSsl(sslOpts); + + auto factoryUri = factory.makeUri(); + REQUIRE(factoryUri == expectedUri()); + + auto pool = factory.makePool(); + REQUIRE(pool); + } +} diff --git a/src/gennylib/test/context_test.cpp b/src/gennylib/test/context_test.cpp index 8ae8987557..7a6f70f407 100644 --- a/src/gennylib/test/context_test.cpp +++ b/src/gennylib/test/context_test.cpp @@ -157,7 +157,7 @@ SchemaVersion: 2018-07-01 } SECTION("Invalid MongoUri") { auto yaml = YAML::Load("SchemaVersion: 2018-07-01\nActors: []"); - auto test = [&]() { WorkloadContext w(yaml, metrics, orchestrator, "notValid", cast); }; + auto test = [&]() { WorkloadContext w(yaml, metrics, orchestrator, "::notValid::", cast); }; REQUIRE_THROWS_WITH(test(), Matches(R"(an invalid MongoDB URI was provided)")); }