From 985fc831ef874c2b81a7f35261ffd747f62c7b58 Mon Sep 17 00:00:00 2001 From: Jason Chan Date: Thu, 3 Jan 2019 10:52:30 -0800 Subject: [PATCH] TIG-1192 Actor CI Tests can use a real mongo instance (#64) * Actor CI Tests can use a real mongo instance * Cleanup, better cmake/Catch2 integration, better reporting on failed tests --- CONTRIBUTING.md | 6 + README.md | 41 ++-- create-new-actor.sh | 60 ++++++ evergreen.yml | 114 ++++++++++- generate-uuid-tag.sh | 2 +- src/CMakeLists.txt | 3 + src/driver/CMakeLists.txt | 11 +- src/gennylib/CMakeLists.txt | 35 ++-- src/gennylib/test/ActorHelper.cpp | 2 +- src/gennylib/test/MongoTestFixture.cpp | 36 ++++ src/gennylib/test/MongoTestFixture.hpp | 22 ++ src/gennylib/test/actors_test.cpp | 42 ++++ src/resmokeconfig/genny_sharded.yml | 32 +++ .../genny_single_node_replset.yml | 22 ++ src/resmokeconfig/genny_standalone.yml | 21 ++ .../genny_three_node_replset.yml | 23 +++ .../catch2/ParseAndAddCatchTests.cmake | 193 ++++++++++++++++++ src/third_party/catch2/README.md | 21 ++ 18 files changed, 638 insertions(+), 48 deletions(-) create mode 100644 src/gennylib/test/MongoTestFixture.cpp create mode 100644 src/gennylib/test/MongoTestFixture.hpp create mode 100644 src/gennylib/test/actors_test.cpp create mode 100644 src/resmokeconfig/genny_sharded.yml create mode 100644 src/resmokeconfig/genny_single_node_replset.yml create mode 100644 src/resmokeconfig/genny_standalone.yml create mode 100644 src/resmokeconfig/genny_three_node_replset.yml create mode 100644 src/third_party/catch2/ParseAndAddCatchTests.cmake create mode 100644 src/third_party/catch2/README.md diff --git a/CONTRIBUTING.md b/CONTRIBUTING.md index 32e3b12b2b..ab1065ba9e 100644 --- a/CONTRIBUTING.md +++ b/CONTRIBUTING.md @@ -179,3 +179,9 @@ private: - Use scoped enums (`enum class`) wherever possible - Use `k` prefixes for enum values e.g. `enum class Color { kRed, kBlue };` + +## Catch2 +Genny uses [Catch2](https://github.com/catchorg/Catch2) as its test framework. +If you'd like to make changes to the vendored copy of Catch2 (e.g. to upgrade +to a newer copy) Please make sure you read the README in the Catch2 directory +before making changes. diff --git a/README.md b/README.md index a33a2997e6..92928a1f19 100644 --- a/README.md +++ b/README.md @@ -61,7 +61,7 @@ and restarting your shell. TODO: TIG-1263 This is kind of a hack; using built-in package-location mechanisms would avoid having to have OS-specific hacks like this. -### Linux Distributions +#### Linux Distributions Have installations of non-vendored dependent packages in your system, using the package manger. Generally this is: @@ -85,7 +85,7 @@ You only need to run cmake once. Other useful targets include: - test_gennylib test_driver (builds tests) - test (run's tests if they're built) -### Other Operating Systems +#### Other Operating Systems If not using OS X, ensure you have a recent C++ compiler and boost installation. You will also need packages installed corresponding to the @@ -105,7 +105,7 @@ apt-get install -y \ # https://mongodb.github.io/mongo-cxx-driver/mongocxx-v3/installation/ ``` -### IDEs and Whatnot +#### IDEs and Whatnot We follow CMake and C++17 best-practices so anything that doesn't work via "normal means" is probably a bug. @@ -115,8 +115,7 @@ emacs, vim, etc.). Before doing anything cute (see [CONTRIBUTING.md](./CONTRIBUTING.md)), please do due-diligence to ensure it's not going to make common editing environments go wonky. -Running Genny Self-Tests ------------------------- +### Running Genny Self-Tests Genny has self-tests using Catch2. You can run them with the following command: @@ -124,7 +123,7 @@ Genny has self-tests using Catch2. You can run them with the following command: make -C "build" test_gennylib test_driver test ``` -### Perf Tests +#### Perf Tests The above `make test` line also runs so-called "perf" tests. They can take a while to run and may fail occasionally on local developer @@ -143,8 +142,28 @@ Read more about specifying what tests to run [here][s]. [s]: https://github.com/catchorg/Catch2/blob/master/docs/command-line.md#specifying-which-tests-to-run -Running Genny Workloads ------------------------ +#### Actor Integration Tests + +The actor tests use resmoke to set up a real MongoDB cluster and execute +the test binary. The resmoke yaml config files that define the different +cluster configurations are defined in `src/resmokeconfig`. + +resmoke.py can be run locally as follows: +```sh +# Set up virtualenv and install resmoke requirements if needed. +# From genny's top-level directory. +python /path/to/resmoke.py --suite src/resmokeconfig/genny_standalone.yml +``` + +Each yaml configuration file will only run tests that are associated +with their specific tags. (Eg. `genny_standalone.yml` will only run +tests that have been tagged with the "[standalone]" tag.) + +When creating a new actor, `create-new-actor.sh` will generate a new test case +template to ensure the new actor can run against different MongoDB topologies, +please update the template as needed so it uses the newly created actor. + +### Running Genny Workloads First install mongodb and start a mongod process: @@ -173,16 +192,14 @@ in the above example). Post-processing of metrics data is done by Python scripts in the `src/python` directory. See [the README there](./src/python/README.md). -Code Style and Limitations ---------------------------- +### Code Style and Limitations > Don't get cute. Please see [CONTRIBUTING.md](./CONTRIBUTING.md) for code-style etc. Note that we're pretty vanilla. -Sanitizers ----------- +### Sanitizers Genny is periodically manually tested to be free of unknown sanitizer errors. These are not currently run in a CI job. If you are adding diff --git a/create-new-actor.sh b/create-new-actor.sh index eaeaf97d22..be972ad497 100755 --- a/create-new-actor.sh +++ b/create-new-actor.sh @@ -103,6 +103,49 @@ create_impl() { create_impl_text "$@" > "$(dirname "$0")/src/cast_core/src/actors/${actor_name}.cpp" } +create_test() { + local actor_name + actor_name="$1" + + cat << EOF >> "$(dirname "$0")/src/gennylib/test/${actor_name}_test.cpp" +#include "test.h" + +#include +#include + +#include + +namespace { +using namespace genny::testing; +namespace bson_stream = bsoncxx::builder::stream; + +TEST_CASE_METHOD(MongoTestFixture, "${actor_name} successfully connects to a MongoDB instance.", + "[standalone][single_node_replset][three_node_replset][sharded]") { + + dropAllDatabases(); + auto db = client.database("test"); + + SECTION("Insert a document into the database.") { + auto builder = bson_stream::document{}; + bsoncxx::document::value doc_value = builder + << "name" << "MongoDB" + << "type" << "database" + << "count" << 1 + << "info" << bson_stream::open_document + << "x" << 203 + << "y" << 102 + << bson_stream::close_document + << bson_stream::finalize; + bsoncxx::document::view view = doc_value.view(); + db.collection("test").insert_one(view); + // Fail on purpose to encourage contributors to extend automated testing for each new actor. + REQUIRE(db.collection("test").count(view) == 0); + } +} +} // namespace +EOF +} + recreate_cast_core_cmake_file() { local uuid_tag local actor_name @@ -118,6 +161,21 @@ recreate_cast_core_cmake_file() { mv "$$.cmake.txt" "$cmake_file" } +recreate_gennylib_cmake_file() { + local uuid_tag + local actor_name + local cmake_file + uuid="$1" + actor_name="$2" + cmake_file="$(dirname "$0")/src/gennylib/CMakeLists.txt" + + < "$cmake_file" \ + perl -pe "s|((\\s+)# ActorsTestEnd)|\$2test/${actor_name}_test.cpp\\n\$1|" \ + > "$$.cmake.txt" + + mv "$$.cmake.txt" "$cmake_file" +} + if [[ "$#" != 1 ]]; then usage exit 1 @@ -138,3 +196,5 @@ uuid_tag="$("$(dirname "$0")/generate-uuid-tag.sh")" create_header "$uuid_tag" "$actor_name" create_impl "$uuid_tag" "$actor_name" recreate_cast_core_cmake_file "$uuid_tag" "$actor_name" +create_test "$actor_name" +recreate_gennylib_cmake_file "$uuid_tag" "$actor_name" diff --git a/evergreen.yml b/evergreen.yml index e3f6f412ec..d40b2648d3 100644 --- a/evergreen.yml +++ b/evergreen.yml @@ -79,13 +79,40 @@ tasks: - func: f_fetch_source - func: f_build_grpc - func: f_compile + - func: f_checkout_mongodb + - name: t_python_test commands: - func: f_python_test + - name: t_cmake_test commands: - func: f_cmake_test +- name: t_integration_test_standalone + commands: + - func: f_resmoke_test + vars: + resmoke_suite: genny_standalone.yml + +- name: t_integration_test_single_node_replset + commands: + - func: f_resmoke_test + vars: + resmoke_suite: genny_single_node_replset.yml + +- name: t_integration_test_three_node_replset + commands: + - func: f_resmoke_test + vars: + resmoke_suite: genny_three_node_replset.yml + +- name: t_integration_test_sharded + commands: + - func: f_resmoke_test + vars: + resmoke_suite: genny_sharded.yml + ## ⚡️ Task Groups ⚡️ @@ -108,6 +135,10 @@ task_groups: - t_compile - t_python_test - t_cmake_test + - t_integration_test_standalone + - t_integration_test_single_node_replset + - t_integration_test_three_node_replset + - t_integration_test_sharded ## ⚡️ Functions ⚡️ @@ -119,6 +150,26 @@ functions: params: directory: src + ## + # Download the mongodb binary and then clone and checkout the appropriate mongodb repository branch + # that contains the intended gennylib test case (SERVER-38646). Also installs resmoke dependencies. + ## + f_checkout_mongodb: + - command: shell.exec + params: + working_dir: src/build + script: | + yes | pacman -S mongodb + git clone git@github.com:mongodb/mongo.git mongo + + pushd mongo + git checkout 6734c12d17dd4c0e2738a47feb7114221d6ba66d + popd + + virtualenv -p python2 venv + source venv/bin/activate + python -m pip install -r mongo/etc/pip/evgtest-requirements.txt + ## # Compile gRPC. This script should be synchronized with DSI's compilation sript for Genny: # https://github.com/10gen/dsi/blob/master/configurations/workload_setup/workload_setup.common.yml @@ -147,7 +198,7 @@ functions: mkdir cmake-build pushd cmake-build cmake .. - make + make -j$(nproc) ## # Compile the project in src/build. @@ -170,10 +221,10 @@ functions: export CPLUS_INCLUDE_PATH="$cwd/grpc/include:$cwd/grpc/third_party/protobuf/src" cmake -DCMAKE_CXX_FLAGS="${cmake_cxx_flags}" .. - make + make -j$(nproc) ## - # Runs tests via `make test`. + # Runs tests via ctest. # Requires f_compile to have been run first. ## f_cmake_test: @@ -186,7 +237,53 @@ functions: set -o pipefail set -o nounset - make test + ctest --label-exclude "(standalone|sharded|single_node_replset|three_node_replset)" + + ## + # Runs tests via resmoke. + # Requires f_compile to have been run first. + ## + f_resmoke_test: + - command: shell.exec + params: + continue_on_err: true + working_dir: src + script: | + set -o errexit + set -o pipefail + # set -o nounset # the "activate" script has an unbound variable + + source build/venv/bin/activate + + # We rely on catch2 to report test failures, but it doesn't always do so. + # See https://github.com/catchorg/Catch2/issues/1210 + # As a workaround, we generate a dummy report with a failed test that is deleted if resmoke + # succeeds. + + cat << EOF >> "build/sentinel.junit.xml" + + + + + + + + + + + + EOF + + # The tests themselves do the reporting instead of using resmoke. + python build/mongo/buildscripts/resmoke.py \ + --suite src/resmokeconfig/${resmoke_suite} \ + --mongod mongod \ + --mongo mongo \ + --mongos mongos + + # Remove the sentinel report if resmoke succeeds. This line won't be executed if + # resmoke fails because we've set errexit on this shell. + rm build/sentinel.junit.xml ## # Runs python nosetests. @@ -213,6 +310,10 @@ functions: params: optional: true file: src/build/src/*/*.junit.xml + - command: attach.xunit_results + params: + optional: true + file: src/build/*.junit.xml - command: attach.xunit_results params: optional: true @@ -229,4 +330,7 @@ functions: set -o pipefail set -o nounset - rm -f src/build/src/*/*.junit.xml src/src/python/nosetests.xml + rm -f \ + src/build/*.junit.xml \ + src/build/src/*/*.junit.xml \ + src/src/python/nosetests.xml diff --git a/generate-uuid-tag.sh b/generate-uuid-tag.sh index 0fbd8e7ded..6ae4d8ad17 100755 --- a/generate-uuid-tag.sh +++ b/generate-uuid-tag.sh @@ -1,4 +1,4 @@ #/bin/bash UUID=$(uuidgen | sed s/-/_/g | tr a-z A-Z) -echo "HEADER_${UUID}" +echo "HEADER_${UUID}_INCLUDED" diff --git a/src/CMakeLists.txt b/src/CMakeLists.txt index e10fb97ce2..ff5084470b 100644 --- a/src/CMakeLists.txt +++ b/src/CMakeLists.txt @@ -1,3 +1,6 @@ +include(${CMAKE_CURRENT_SOURCE_DIR}/third_party/catch2/ParseAndAddCatchTests.cmake) +set(AdditionalCatchParameters --reporter junit --durations yes --rng-seed 12345 --success) + add_subdirectory(third_party) add_subdirectory(gennylib) diff --git a/src/driver/CMakeLists.txt b/src/driver/CMakeLists.txt index 78ba4d1be3..d641db1ec5 100644 --- a/src/driver/CMakeLists.txt +++ b/src/driver/CMakeLists.txt @@ -31,13 +31,4 @@ target_include_directories(test_driver ) target_link_libraries(test_driver driver catch2) -add_test( - NAME test_driver - COMMAND test_driver - -r junit - -d yes - --rng-seed 12345 - --success - --out test_driver.junit.xml -) - +ParseAndAddCatchTests(test_driver) diff --git a/src/gennylib/CMakeLists.txt b/src/gennylib/CMakeLists.txt index 42b389a2ba..ec1e045e37 100644 --- a/src/gennylib/CMakeLists.txt +++ b/src/gennylib/CMakeLists.txt @@ -93,7 +93,7 @@ install(DIRECTORY include/ # Pull together our utils, header directories, and testing dependencies into one static target add_library(gennylib_testutil STATIC test/ActorHelper.cpp -) + test/MongoTestFixture.cpp) target_include_directories(gennylib_testutil PUBLIC src @@ -120,28 +120,25 @@ target_link_libraries(test_gennylib gennylib_testutil) target_link_libraries(test_gennylib gennylib_testutil) -add_test( - NAME test_gennylib - COMMAND test_gennylib - -r junit - -d yes - --rng-seed 12345 - --success - --out test_gennylib.junit.xml -) - add_executable(benchmark_gennylib test/PhaseLoop_benchmark.cpp test/orchestrator_benchmark.cpp ) + target_link_libraries(benchmark_gennylib gennylib_testutil) -add_test( - NAME benchmark_gennylib - COMMAND benchmark_gennylib - -r junit - -d yes - --rng-seed 12345 - --success - --out benchmark_gennylib.junit.xml + +add_executable(test_gennylib_with_server + # Don't modify the following comments; they're used as anchors in scripts that create new actors. + # ActorsTestStart + test/actors_test.cpp + # ActorsTestEnd ) + +target_link_libraries(test_gennylib_with_server gennylib_testutil) + +ParseAndAddCatchTests(test_gennylib) + +ParseAndAddCatchTests(benchmark_gennylib) + +ParseAndAddCatchTests(test_gennylib_with_server) diff --git a/src/gennylib/test/ActorHelper.cpp b/src/gennylib/test/ActorHelper.cpp index 3f566ce31a..9fdc73c325 100644 --- a/src/gennylib/test/ActorHelper.cpp +++ b/src/gennylib/test/ActorHelper.cpp @@ -1,4 +1,4 @@ -#include +#include "ActorHelper.hpp" #include diff --git a/src/gennylib/test/MongoTestFixture.cpp b/src/gennylib/test/MongoTestFixture.cpp new file mode 100644 index 0000000000..ac75363f49 --- /dev/null +++ b/src/gennylib/test/MongoTestFixture.cpp @@ -0,0 +1,36 @@ +#include "MongoTestFixture.hpp" + +#include +#include + +#include + +#include + +namespace genny::testing { + +const mongocxx::uri MongoTestFixture::kConnectionString = []() { + const char* connChar = getenv("MONGO_CONNECTION_STRING"); + std::string connStr; + + if (!connChar) { + connStr = mongocxx::uri::k_default_uri; + BOOST_LOG_TRIVIAL(info) << "MONGO_CONNECTION_STRING not set, using default value: " + << connStr; + } else { + connStr = connChar; + } + + return mongocxx::uri(connStr); +}(); + +void MongoTestFixture::dropAllDatabases() { + for (auto&& dbDoc : client.list_databases()) { + const auto dbName = dbDoc["name"].get_utf8().value; + const auto dbNameString = std::string(dbName); + if (dbNameString != "admin" && dbNameString != "config" && dbNameString != "local") { + client.database(dbName).drop(); + } + } +} +} // namespace genny::testing diff --git a/src/gennylib/test/MongoTestFixture.hpp b/src/gennylib/test/MongoTestFixture.hpp new file mode 100644 index 0000000000..2de75dc979 --- /dev/null +++ b/src/gennylib/test/MongoTestFixture.hpp @@ -0,0 +1,22 @@ +#ifndef HEADER_D9091084_CB09_4108_A553_5D0EC18C132F_INCLUDED +#define HEADER_D9091084_CB09_4108_A553_5D0EC18C132F_INCLUDED + +#include + +namespace genny::testing { + +class MongoTestFixture { +public: + MongoTestFixture() : client(kConnectionString) {} + +protected: + void dropAllDatabases(); + + mongocxx::client client; + +private: + static const mongocxx::uri kConnectionString; +}; +} // namespace genny::testing + +#endif // HEADER_D9091084_CB09_4108_A553_5D0EC18C132F_INCLUDED diff --git a/src/gennylib/test/actors_test.cpp b/src/gennylib/test/actors_test.cpp new file mode 100644 index 0000000000..aeb56e0248 --- /dev/null +++ b/src/gennylib/test/actors_test.cpp @@ -0,0 +1,42 @@ +#include "test.h" + +#include +#include +#include + +#include +#include + +#include "MongoTestFixture.hpp" + +namespace { + +using namespace genny::testing; + +namespace bson_stream = bsoncxx::builder::stream; + +TEST_CASE_METHOD(MongoTestFixture, + "Successfully connects to a MongoDB instance.", + "[standalone][single_node_replset][three_node_replset][sharded]") { + + dropAllDatabases(); + auto db = client.database("test"); + + auto builder = bson_stream::document{}; + bsoncxx::document::value doc_value = builder + << "name" + << "MongoDB" + << "type" + << "database" + << "count" << 1 << "info" << bson_stream::open_document << "x" << 203 << "y" << 102 + << bson_stream::close_document << bson_stream::finalize; + + SECTION("Insert a document into the database.") { + + bsoncxx::document::view view = doc_value.view(); + db.collection("test").insert_one(view); + + REQUIRE(db.collection("test").count(view) == 1); + } +} +} // namespace diff --git a/src/resmokeconfig/genny_sharded.yml b/src/resmokeconfig/genny_sharded.yml new file mode 100644 index 0000000000..2e77cf5fe0 --- /dev/null +++ b/src/resmokeconfig/genny_sharded.yml @@ -0,0 +1,32 @@ +test_kind: gennylib_test + +executor: + config: + program_executable: build/src/gennylib/test_gennylib_with_server + verbatim_arguments: + - "--reporter" + - "junit" + - "--durations" + - "yes" + - "--rng-seed" + - "12345" + - "--success" + - "--out" + - "build/test_gennylib_with_server.junit.xml" + - "[sharded]" + fixture: + class: ShardedClusterFixture + mongos_options: + set_parameters: + enableTestCommands: 1 + mongod_options: + set_parameters: + enableTestCommands: 1 + num_rs_nodes_per_shard: 3 + num_shards: 3 + num_mongos: 3 + configsvr_options: + num_nodes: 3 + all_nodes_electable: true + shard_options: + all_nodes_electable: true diff --git a/src/resmokeconfig/genny_single_node_replset.yml b/src/resmokeconfig/genny_single_node_replset.yml new file mode 100644 index 0000000000..65cc9ce356 --- /dev/null +++ b/src/resmokeconfig/genny_single_node_replset.yml @@ -0,0 +1,22 @@ +test_kind: gennylib_test + +executor: + config: + program_executable: build/src/gennylib/test_gennylib_with_server + verbatim_arguments: + - "--reporter" + - "junit" + - "--durations" + - "yes" + - "--rng-seed" + - "12345" + - "--success" + - "--out" + - "build/test_gennylib_with_server.junit.xml" + - "[single_node_replset]" + fixture: + class: ReplicaSetFixture + mongod_options: + set_parameters: + enableTestCommands: 1 + num_nodes: 1 diff --git a/src/resmokeconfig/genny_standalone.yml b/src/resmokeconfig/genny_standalone.yml new file mode 100644 index 0000000000..8acd451a3a --- /dev/null +++ b/src/resmokeconfig/genny_standalone.yml @@ -0,0 +1,21 @@ +test_kind: gennylib_test + +executor: + config: + program_executable: build/src/gennylib/test_gennylib_with_server + verbatim_arguments: + - "--reporter" + - "junit" + - "--durations" + - "yes" + - "--rng-seed" + - "12345" + - "--success" + - "--out" + - "build/test_gennylib_with_server.junit.xml" + - "[standalone]" + fixture: + class: MongoDFixture + mongod_options: + set_parameters: + enableTestCommands: 1 diff --git a/src/resmokeconfig/genny_three_node_replset.yml b/src/resmokeconfig/genny_three_node_replset.yml new file mode 100644 index 0000000000..1d8ab2a07c --- /dev/null +++ b/src/resmokeconfig/genny_three_node_replset.yml @@ -0,0 +1,23 @@ +test_kind: gennylib_test + +executor: + config: + program_executable: build/src/gennylib/test_gennylib_with_server + verbatim_arguments: + - "--reporter" + - "junit" + - "--durations" + - "yes" + - "--rng-seed" + - "12345" + - "--success" + - "--out" + - "build/test_gennylib_with_server.junit.xml" + - "[three_node_replset]" + fixture: + class: ReplicaSetFixture + mongod_options: + set_parameters: + enableTestCommands: 1 + num_nodes: 3 + all_nodes_electable: true diff --git a/src/third_party/catch2/ParseAndAddCatchTests.cmake b/src/third_party/catch2/ParseAndAddCatchTests.cmake new file mode 100644 index 0000000000..64dc5f3ffa --- /dev/null +++ b/src/third_party/catch2/ParseAndAddCatchTests.cmake @@ -0,0 +1,193 @@ +#==================================================================================================# +# supported macros # +# - TEST_CASE, # +# - SCENARIO, # +# - TEST_CASE_METHOD, # +# - CATCH_TEST_CASE, # +# - CATCH_SCENARIO, # +# - CATCH_TEST_CASE_METHOD. # +# # +# Usage # +# 1. make sure this module is in the path or add this otherwise: # +# set(CMAKE_MODULE_PATH ${CMAKE_MODULE_PATH} "${CMAKE_SOURCE_DIR}/cmake.modules/") # +# 2. make sure that you've enabled testing option for the project by the call: # +# enable_testing() # +# 3. add the lines to the script for testing target (sample CMakeLists.txt): # +# project(testing_target) # +# set(CMAKE_MODULE_PATH ${CMAKE_MODULE_PATH} "${CMAKE_SOURCE_DIR}/cmake.modules/") # +# enable_testing() # +# # +# find_path(CATCH_INCLUDE_DIR "catch.hpp") # +# include_directories(${INCLUDE_DIRECTORIES} ${CATCH_INCLUDE_DIR}) # +# # +# file(GLOB SOURCE_FILES "*.cpp") # +# add_executable(${PROJECT_NAME} ${SOURCE_FILES}) # +# # +# include(ParseAndAddCatchTests) # +# ParseAndAddCatchTests(${PROJECT_NAME}) # +# # +# The following variables affect the behavior of the script: # +# # +# PARSE_CATCH_TESTS_VERBOSE (Default OFF) # +# -- enables debug messages # +# PARSE_CATCH_TESTS_NO_HIDDEN_TESTS (Default OFF) # +# -- excludes tests marked with [!hide], [.] or [.foo] tags # +# PARSE_CATCH_TESTS_ADD_FIXTURE_IN_TEST_NAME (Default ON) # +# -- adds fixture class name to the test name # +# PARSE_CATCH_TESTS_ADD_TARGET_IN_TEST_NAME (Default ON) # +# -- adds cmake target name to the test name # +# PARSE_CATCH_TESTS_ADD_TO_CONFIGURE_DEPENDS (Default OFF) # +# -- causes CMake to rerun when file with tests changes so that new tests will be discovered # +# # +# One can also set (locally) the optional variable OptionalCatchTestLauncher to precise the way # +# a test should be run. For instance to use test MPI, one can write # +# set(OptionalCatchTestLauncher ${MPIEXEC} ${MPIEXEC_NUMPROC_FLAG} ${NUMPROC}) # +# just before calling this ParseAndAddCatchTests function # +# # +#==================================================================================================# + +cmake_minimum_required(VERSION 2.8.8) + +option(PARSE_CATCH_TESTS_VERBOSE "Print Catch to CTest parser debug messages" OFF) +option(PARSE_CATCH_TESTS_NO_HIDDEN_TESTS "Exclude tests with [!hide], [.] or [.foo] tags" OFF) +option(PARSE_CATCH_TESTS_ADD_FIXTURE_IN_TEST_NAME "Add fixture class name to the test name" ON) +option(PARSE_CATCH_TESTS_ADD_TARGET_IN_TEST_NAME "Add target name to the test name" ON) +option(PARSE_CATCH_TESTS_ADD_TO_CONFIGURE_DEPENDS "Add test file to CMAKE_CONFIGURE_DEPENDS property" OFF) + +function(PrintDebugMessage) + if(PARSE_CATCH_TESTS_VERBOSE) + message(STATUS "ParseAndAddCatchTests: ${ARGV}") + endif() +endfunction() + +# This removes the contents between +# - block comments (i.e. /* ... */) +# - full line comments (i.e. // ... ) +# contents have been read into '${CppCode}'. +# !keep partial line comments +function(RemoveComments CppCode) + string(ASCII 2 CMakeBeginBlockComment) + string(ASCII 3 CMakeEndBlockComment) + string(REGEX REPLACE "/\\*" "${CMakeBeginBlockComment}" ${CppCode} "${${CppCode}}") + string(REGEX REPLACE "\\*/" "${CMakeEndBlockComment}" ${CppCode} "${${CppCode}}") + string(REGEX REPLACE "${CMakeBeginBlockComment}[^${CMakeEndBlockComment}]*${CMakeEndBlockComment}" "" ${CppCode} "${${CppCode}}") + string(REGEX REPLACE "\n[ \t]*//+[^\n]+" "\n" ${CppCode} "${${CppCode}}") + + set(${CppCode} "${${CppCode}}" PARENT_SCOPE) +endfunction() + +# Worker function +function(ParseFile SourceFile TestTarget) + # According to CMake docs EXISTS behavior is well-defined only for full paths. + get_filename_component(SourceFile ${SourceFile} ABSOLUTE) + if(NOT EXISTS ${SourceFile}) + message(WARNING "Cannot find source file: ${SourceFile}") + return() + endif() + PrintDebugMessage("parsing ${SourceFile}") + file(STRINGS ${SourceFile} Contents NEWLINE_CONSUME) + + # Remove block and fullline comments + RemoveComments(Contents) + + # Find definition of test names + string(REGEX MATCHALL "[ \t]*(CATCH_)?(TEST_CASE_METHOD|SCENARIO|TEST_CASE)[ \t]*\\([^\)]+\\)+[ \t\n]*{+[ \t]*(//[^\n]*[Tt][Ii][Mm][Ee][Oo][Uu][Tt][ \t]*[0-9]+)*" Tests "${Contents}") + + if(PARSE_CATCH_TESTS_ADD_TO_CONFIGURE_DEPENDS AND Tests) + PrintDebugMessage("Adding ${SourceFile} to CMAKE_CONFIGURE_DEPENDS property") + set_property( + DIRECTORY + APPEND + PROPERTY CMAKE_CONFIGURE_DEPENDS ${SourceFile} + ) + endif() + + foreach(TestName ${Tests}) + # Strip newlines + string(REGEX REPLACE "\\\\\n|\n" "" TestName "${TestName}") + + # Get test type and fixture if applicable + string(REGEX MATCH "(CATCH_)?(TEST_CASE_METHOD|SCENARIO|TEST_CASE)[ \t]*\\([^,^\"]*" TestTypeAndFixture "${TestName}") + string(REGEX MATCH "(CATCH_)?(TEST_CASE_METHOD|SCENARIO|TEST_CASE)" TestType "${TestTypeAndFixture}") + string(REPLACE "${TestType}(" "" TestFixture "${TestTypeAndFixture}") + + # Get string parts of test definition + string(REGEX MATCHALL "\"+([^\\^\"]|\\\\\")+\"+" TestStrings "${TestName}") + + # Strip wrapping quotation marks + string(REGEX REPLACE "^\"(.*)\"$" "\\1" TestStrings "${TestStrings}") + string(REPLACE "\";\"" ";" TestStrings "${TestStrings}") + + # Validate that a test name and tags have been provided + list(LENGTH TestStrings TestStringsLength) + if(TestStringsLength GREATER 2 OR TestStringsLength LESS 1) + message(FATAL_ERROR "You must provide a valid test name and tags for all tests in ${SourceFile}") + endif() + + # Assign name and tags + list(GET TestStrings 0 Name) + if("${TestType}" STREQUAL "SCENARIO") + set(Name "Scenario: ${Name}") + endif() + if(PARSE_CATCH_TESTS_ADD_FIXTURE_IN_TEST_NAME AND TestFixture) + set(CTestName "${TestFixture}:${Name}") + else() + set(CTestName "${Name}") + endif() + if(PARSE_CATCH_TESTS_ADD_TARGET_IN_TEST_NAME) + set(CTestName "${TestTarget}:${CTestName}") + endif() + # add target to labels to enable running all tests added from this target + set(Labels ${TestTarget}) + if(TestStringsLength EQUAL 2) + list(GET TestStrings 1 Tags) + string(TOLOWER "${Tags}" Tags) + # remove target from labels if the test is hidden + if("${Tags}" MATCHES ".*\\[!?(hide|\\.)\\].*") + list(REMOVE_ITEM Labels ${TestTarget}) + endif() + string(REPLACE "]" ";" Tags "${Tags}") + string(REPLACE "[" "" Tags "${Tags}") + else() + # unset tags variable from previous loop + unset(Tags) + endif() + + list(APPEND Labels ${Tags}) + + list(FIND Labels "!hide" IndexOfHideLabel) + set(HiddenTagFound OFF) + foreach(label ${Labels}) + string(REGEX MATCH "^!hide|^\\." result ${label}) + if(result) + set(HiddenTagFound ON) + break() + endif(result) + endforeach(label) + if(PARSE_CATCH_TESTS_NO_HIDDEN_TESTS AND ${HiddenTagFound}) + PrintDebugMessage("Skipping test \"${CTestName}\" as it has [!hide], [.] or [.foo] label") + else() + PrintDebugMessage("Adding test \"${CTestName}\"") + if(Labels) + PrintDebugMessage("Setting labels to ${Labels}") + endif() + + # Add the test and set its properties + add_test(NAME "\"${CTestName}\"" COMMAND ${OptionalCatchTestLauncher} ${TestTarget} ${Name} ${AdditionalCatchParameters} --out ${CTestName}.junit.xml) + set_tests_properties("\"${CTestName}\"" PROPERTIES FAIL_REGULAR_EXPRESSION "No tests ran" + LABELS "${Labels}") + endif() + + endforeach() +endfunction() + +# entry point +function(ParseAndAddCatchTests TestTarget) + PrintDebugMessage("Started parsing ${TestTarget}") + get_target_property(SourceFiles ${TestTarget} SOURCES) + PrintDebugMessage("Found the following sources: ${SourceFiles}") + foreach(SourceFile ${SourceFiles}) + ParseFile(${SourceFile} ${TestTarget}) + endforeach() + PrintDebugMessage("Finished parsing ${TestTarget}") +endfunction() \ No newline at end of file diff --git a/src/third_party/catch2/README.md b/src/third_party/catch2/README.md new file mode 100644 index 0000000000..62558e27d6 --- /dev/null +++ b/src/third_party/catch2/README.md @@ -0,0 +1,21 @@ +## Some Notes on Genny's vendored version of Catch2 + +#### Source Code Versioning +Genny's vendors the single header version of [Catch2](https://github.com/catchorg/Catch2) +for its testing needs. Catch2's version number is listed at the top of `include/catch.hpp` in a +comment. + +#### Cmake Integration Versioning +Genny also vendors Catch2's cmake integration to allow cmake to natively pick up Catch2's +test cases. The integration code, `ParseAndAddCatchTests.cmake`, is not coupled to the version +of Catch2; this allows bug fixes to be picked up without re-vendoring a newer version of Catch2. + +When upgrading either Catch2 or its cmake integration, manually inspect the test results to +ensure everything still works. + +#### Local Changes to Cmake Integration +Additionally, as of December 2018, there's a local change to `ParseAndAddCatchTests.cmake`. On the +line that `add_test` is called, there's an additional parameter `--out ${CTestName}.junit.xml` +appended to the `add_test` invocation. This allows uniquely named reports to be created for +each test, whereas the default behavior is to have the same report file created, causing +earlier ones to be get clobbered. See [Catch2#1316](https://github.com/catchorg/Catch2/issues/1316) for more detail.