From e82cfd3a749ab9306b143ca931c3a942489179ea Mon Sep 17 00:00:00 2001 From: Raiden Worley Date: Thu, 3 Feb 2022 10:17:58 -0500 Subject: [PATCH] TIG-3391 Upgrade C++ Driver for Genny (#607) --- CMakeLists.txt | 3 +- cmake/FindMongoCxx.cmake | 3 ++ evergreen.yml | 18 +++++----- src/gennylib/include/gennylib/parallel.hpp | 1 + src/gennylib/test/Node_test.cpp | 2 +- src/lamplib/src/genny/tasks/compile.py | 42 +++++++++++++++------- src/lamplib/src/genny/toolchain.py | 12 +++---- 7 files changed, 51 insertions(+), 30 deletions(-) diff --git a/CMakeLists.txt b/CMakeLists.txt index f27da06860..67264519f6 100644 --- a/CMakeLists.txt +++ b/CMakeLists.txt @@ -14,6 +14,7 @@ cmake_minimum_required(VERSION 3.13 FATAL_ERROR) cmake_policy(VERSION 3.13) +cmake_policy(SET CMP0110 OLD) # NB: this version number is duplicated in # src/CMakeList.txt, src/gennylib/CMakeLists.txt and src/gennylib/src/version.cpp @@ -63,7 +64,7 @@ set(Boost_USE_MULTITHREADED ON) set(Boost_USE_STATIC_RUNTIME OFF) find_package( Boost - 1.68 + 1.75 REQUIRED COMPONENTS filesystem diff --git a/cmake/FindMongoCxx.cmake b/cmake/FindMongoCxx.cmake index 29c9852ec0..25ea2138d2 100644 --- a/cmake/FindMongoCxx.cmake +++ b/cmake/FindMongoCxx.cmake @@ -13,6 +13,9 @@ # limitations under the License. find_package(libmongocxx REQUIRED) +find_package(libbsoncxx REQUIRED) +include_directories(${LIBMONGOCXX_INCLUDE_DIR}) +include_directories(${LIBBSONCXX_INCLUDE_DIR}) # Make a target for bsoncxx add_library(MongoCxx::bsoncxx SHARED IMPORTED GLOBAL) diff --git a/evergreen.yml b/evergreen.yml index b00fe63704..6e40be8c74 100644 --- a/evergreen.yml +++ b/evergreen.yml @@ -130,15 +130,15 @@ buildvariants: tasks: - name: tg_compile -- name: centos6-perf - display_name: CentOS 6 for Performance - modules: [mongo] - expansions: - distro: rhel62 - run_on: - - centos6-perf - tasks: - - name: tg_compile_and_benchmark +# - name: centos6-perf +# display_name: CentOS 6 for Performance +# modules: [mongo] +# expansions: +# distro: rhel62 +# run_on: +# - centos6-perf +# tasks: +# - name: tg_compile_and_benchmark ## ⚡️ Tasks ⚡️ diff --git a/src/gennylib/include/gennylib/parallel.hpp b/src/gennylib/include/gennylib/parallel.hpp index 27eb5c0da4..d9b0608d53 100644 --- a/src/gennylib/include/gennylib/parallel.hpp +++ b/src/gennylib/include/gennylib/parallel.hpp @@ -26,6 +26,7 @@ #include #include #include +#include namespace genny { diff --git a/src/gennylib/test/Node_test.cpp b/src/gennylib/test/Node_test.cpp index 5c3b4b3adc..5386e0feca 100644 --- a/src/gennylib/test/Node_test.cpp +++ b/src/gennylib/test/Node_test.cpp @@ -99,7 +99,7 @@ TEST_CASE("YAML::Node Equivalency") { REQUIRE(bool(yaml[0]) == false); // ...but it does barf when treating a scalar like a sequence REQUIRE_THROWS_WITH([&]() { yaml["foo"][0]; }(), - Catch::Matches("operator\\[\\] call on a scalar")); + Catch::Matches("operator\\[\\] call on a scalar \\(key: \"0\"\\)")); } { diff --git a/src/lamplib/src/genny/tasks/compile.py b/src/lamplib/src/genny/tasks/compile.py index dfb97f45b3..73db0eea46 100644 --- a/src/lamplib/src/genny/tasks/compile.py +++ b/src/lamplib/src/genny/tasks/compile.py @@ -48,9 +48,13 @@ def cmake( # We set both the prefix path and the toolchain file here as a hack to allow cmake # to find both shared and static libraries. vcpkg doesn't natively support a project # using both. - cmake_prefix_path = os.path.join( - toolchain_info.toolchain_dir, f"installed/x64-{toolchain_info.triplet_os}-shared", - ) + cmake_prefix_paths = [ + os.path.join( + toolchain_info.toolchain_dir, f"installed/x64-{toolchain_info.triplet_os}-dynamic", + ), + os.path.join(toolchain_info.toolchain_dir, f"installed/x64-{toolchain_info.triplet_os}",), + ] + cmake_toolchain_file = os.path.join( toolchain_info.toolchain_dir, "scripts/buildsystems/vcpkg.cmake" ) @@ -58,10 +62,10 @@ def cmake( cmake_cmd += [ "-DGENNY_WORKSPACE_ROOT={}".format(workspace_root), # "-DGENNY_REPO_ROOT={}".format(genny_repo_root), # Not needed (yet). - "-DCMAKE_PREFIX_PATH={}".format(cmake_prefix_path), + "-DCMAKE_PREFIX_PATH={}".format(";".join(cmake_prefix_paths)), "-DCMAKE_TOOLCHAIN_FILE={}".format(cmake_toolchain_file), "-DCMAKE_EXPORT_COMPILE_COMMANDS=1", - f"-DVCPKG_TARGET_TRIPLET=x64-{toolchain_info.triplet_os}-static", + f"-DVCPKG_TARGET_TRIPLET=x64-{toolchain_info.triplet_os}", ] cmake_cmd += _sanitizer_flags(sanitizer) @@ -163,14 +167,26 @@ def compile_and_install( sanitizer=sanitizer, cmake_args=cmake_args, ) - compile_all( - genny_repo_root=genny_repo_root, - workspace_root=workspace_root, - build_system=build_system, - os_family=os_family, - linux_distro=linux_distro, - ignore_toolchain_version=ignore_toolchain_version, - ) + + try: + compile_all( + genny_repo_root=genny_repo_root, + workspace_root=workspace_root, + build_system=build_system, + os_family=os_family, + linux_distro=linux_distro, + ignore_toolchain_version=ignore_toolchain_version, + ) + + except subprocess.CalledProcessError as ex: + SLOG.critical( + "Genny compile has failed. This is sometimes caused by having an old mongodbtoolchain. To update: curl -o toolchain_installer.sh http://mongodbtoolchain.build.10gen.cc/installer.sh && bash toolchain_installer.sh" + ) + raise + + except: + raise + install( genny_repo_root=genny_repo_root, workspace_root=workspace_root, diff --git a/src/lamplib/src/genny/toolchain.py b/src/lamplib/src/genny/toolchain.py index afaa394fc8..34d3af4a16 100644 --- a/src/lamplib/src/genny/toolchain.py +++ b/src/lamplib/src/genny/toolchain.py @@ -28,14 +28,14 @@ def _create_compile_environment( # For cmake and ctest cmake_bin_relative_dir = { - "linux": "downloads/tools/cmake-3.13.3-linux/cmake-3.13.3-Linux-x86_64/bin", - "osx": "downloads/tools/cmake-3.13.3-osx/cmake-3.13.3-Darwin-x86_64/CMake.app/Contents/bin", + "linux": "downloads/tools/cmake-3.20.2-linux/cmake-3.20.2-linux-x86_64/bin", + "osx": "downloads/tools/cmake-3.20.2-osx/cmake-3.20.2-Darwin-x86_64/CMake.app/Contents/bin", }[triplet_os] paths.insert(0, os.path.join(toolchain_dir, cmake_bin_relative_dir)) # For ninja ninja_bin_dir = os.path.join( - toolchain_dir, "downloads/tools/ninja-1.8.2-{}:".format(triplet_os) + toolchain_dir, "downloads/tools/ninja-1.10.1-{}:".format(triplet_os) ) paths.insert(0, ninja_bin_dir) @@ -157,12 +157,12 @@ class ToolchainDownloader(Downloader): # These build IDs are from the genny-toolchain Evergreen task. # https://evergreen.mongodb.com/waterfall/genny-toolchain # Find a compile task (for any build variant) and modify the URL: - # genny_toolchain_amazon2_t_compile_cf1c50fab5291cd583a5036dc9ade265bca165a3_21_10_20_20_05_26 - # => cf1c50fab5291cd583a5036dc9ade265bca165a3_21_10_20_20_05_26 + # genny_toolchain_archlinux_t_compile_82eb7c32ad09726f3ef0ddc8d7f24a18b03d9644_21_11_23_16_37_21 + # => 82eb7c32ad09726f3ef0ddc8d7f24a18b03d9644_21_11_23_16_37_21 # If we were 💅 we could do the string logic here in python, but we're not that fancy. # - TOOLCHAIN_BUILD_ID = "cf1c50fab5291cd583a5036dc9ade265bca165a3_21_10_20_20_05_26" + TOOLCHAIN_BUILD_ID = "fc5ec55493f12c0791739e66bd9ffc6db78468e1_22_01_31_16_45_23" TOOLCHAIN_GIT_HASH = TOOLCHAIN_BUILD_ID.split("_")[0] TOOLCHAIN_ROOT = "/data/mci" # TODO BUILD-7624 change this to /opt.