TIG-1135 integrate gRPC into genny (mongodb#40)

.gitignore

@@ -2,6 +2,5 @@
 mwg
 
 # clion
+.idea
 cmake-build-debug
-
-/build

README.md

@@ -18,6 +18,7 @@ brew install cmake
 brew install icu4c
 brew install mongo-cxx-driver
 brew install yaml-cpp
+brew install grpc
 brew install boost --build-from-source \
     --include-test --with-icu4c --without-static
 
@@ -53,7 +54,8 @@ apt-get install -y \
     clang-6.0 \
     make \
     libboost-all-dev \
-    libyaml-cpp-dev
+    libyaml-cpp-dev \
+    libgrpc++-dev
 
 # install mongo C++ driver:
 #   https://mongodb.github.io/mongo-cxx-driver/mongocxx-v3/installation/

build/.gitignore

@@ -0,0 +1,4 @@
+# Ignore everything in this directory
+*
+# Except this file
+!.gitignore

evergreen.yml

@@ -52,7 +52,6 @@ buildvariants:
   run_on:
   - archlinux-test
   expansions:
-    path: ""
     cmake_cxx_flags: -pthread
   tasks:
   - name: t_test
@@ -64,7 +63,6 @@ buildvariants:
 #   run_on:
 #   - macos-1012
 #   expansions:
-#     path: "/Applications/CMake.app/Contents/bin"
 #     cmake_cxx_flags: ""
 #   tasks:
 #   - name: t_test
@@ -77,6 +75,7 @@ tasks:
 - name: t_test
   commands:
     - func: f_fetch_source
+    - func: f_build_grpc
     - func: f_compile
     - func: f_cmake_test
     - func: f_python_test
@@ -110,51 +109,72 @@ functions:
     params:
       directory: src
 
+  ##
+  # Compile gRPC. This script should be synchronized with DSI's compilation sript for Genny:
+  # https://github.com/10gen/dsi/blob/master/configurations/workload_setup/workload_setup.common.yml
+  # (URL valid as of 11/2018)
+  ##
+  f_build_grpc:
+  - command: shell.exec
+    params:
+      working_dir: src/build
+      script: |
+        git clone --recurse-submodules https://github.com/grpc/grpc.git
+        git checkout b79462f186cc22550bc8d53a00ae751f50d194f5
+
+        pushd grpc
+
+        # Work around https://github.com/grpc/grpc/issues/16675 by checking out a newer commit.
+        pushd third_party/boringssl
+        git checkout 9edbc7ff9f56dff801a49f5ecb15ee06718b995c
+        popd
+
+        mkdir cmake-build
+        pushd cmake-build
+        cmake ..
+        make
+
   ##
   # Compile the project in src/build.
   #
   # Expansions:
-  #  path: prepended to $PATH
   #  cmake_cxx_flags: sent to cmake as `-DCMAKE_CXX_FLAGS`
   ###
   f_compile:
   - command: shell.exec
     params:
+      working_dir: src/build
       script: |
-        export PATH="${path}:$PATH"
-        cd src/build
+
+        # Add paths to gRPC and protobuf.
+        cwd=$(pwd)
+        export LIBRARY_PATH="$cwd/grpc/cmake-build/:$cwd/grpc/cmake-build/third_party/protobuf:$LIBRARY_PATH"
+        export CPLUS_INCLUDE_PATH="$cwd/grpc/include:$cwd/grpc/third_party/protobuf/src:$CPLUS_INCLUDE_PATH"
+
         cmake -DCMAKE_CXX_FLAGS="${cmake_cxx_flags}" ..
         make
 
   ##
   # Runs tests via `make test`.
   # Requires f_compile to have been run first.
-  #
-  # Expansions:
-  #  path: prepended to $PATH
   ##
   f_cmake_test:
   - command: shell.exec
     params:
       continue_on_err: true
+      working_dir: src/build
       script: |
-        export PATH="${path}:$PATH"
-        cd src/build
         make test
 
   ##
   # Runs python nosetests.
-  #
-  # Expansions:
-  #  path: prepended to $PATH
   ##
   f_python_test:
   - command: shell.exec
     params:
       continue_on_err: true
+      working_dir: src/src/python
       script: |
-        export PATH="${path}:$PATH"
-        cd src/src/python
         python setup.py nosetests
 
   ##

src/CMakeLists.txt

@@ -1,4 +1,2 @@
-
 add_subdirectory(gennylib)
 add_subdirectory(driver)
-

src/gennylib/CMakeLists.txt

@@ -16,7 +16,6 @@ add_library(gennylib_headers SHARED
 # Eventually will move gennylib/actors into its own lib so keep those files together
 add_library(gennylib SHARED
     src/version.cpp
-
     src/context.cpp
     src/Orchestrator.cpp
 
@@ -34,6 +33,8 @@ add_library(gennylib SHARED
     # ActorsEnd
 )
 
+add_subdirectory(jasper)
+
 # Define headers for this library. PUBLIC headers are used for
 # compiling the library, and will be added to consumers' build
 # paths.
@@ -45,6 +46,7 @@ target_include_directories(gennylib
         ${YAML_CPP_INCLUDE_DIR}
         ${LIBMONGOCXX_INCLUDE_DIRS}
     PRIVATE
+        jasper
         src
         src/value_generators
 )
@@ -57,6 +59,8 @@ target_link_libraries(gennylib
         ${Boost_LIBRARIES}
         ${YAML_CPP_LIBRARIES}
         ${LIBMONGOCXX_LIBRARIES}
+    PRIVATE
+        jasperlib
 )
 
 # 'make install' to the correct locations (provided by GNUInstallDirs).

src/gennylib/jasper/CMakeLists.txt

@@ -0,0 +1,13 @@
+add_library(jasperlib STATIC
+    jasper.pb.h
+    jasper.pb.cc
+
+    jasper.grpc.pb.h
+    jasper.grpc.pb.cc
+)
+
+target_link_libraries(jasperlib
+    PRIVATE
+        grpc++
+        protobuf
+)

src/gennylib/jasper/README.md

@@ -0,0 +1,43 @@
+## Jasper gRPC Client for Genny
+
+### Overview
+
+The `/src/third_party/jasper` directory contains files related to the
+[jasper](https://github.com/mongodb/jasper) process management service C++
+gRPC client that Genny uses.
+
+There are three categories of files here:
+
+
+1. `jasper.proto`: The proto file copied verbatim from the jasper repository.
+2. `jasper.pb.*`: The generated protobuf C++ header and source which contain
+   the definition for the protobuf interface structure.
+3. `jasper.grpc.*` The generated gRPC client C++ header and source which
+   contain the RPC functions to jasper.
+
+### Versioning
+`jasper.proto` should correspond to the version used in the `curator` binary
+(which contains jasper, among other things). The curator version used in DSI
+will be updated concurrently with the `jasper.proto` file in the genny
+repository. The best way to find the actual version is to ssh into a DSI
+machine and run `curator --version`, or grep `curator --version` in the DSI
+repository.
+
+The exact version of `curator` and `jasper.proto` will be stored in the DSI
+repo, there is no separate version recorded in the genny repo. This allows
+the source of truth to be kept in a single location.
+
+If you need to update the version of `jasper.proto` in genny manually:
+
+1. replace `jasper.proto` with the newer version.
+2. from this directory, call:
+
+```bash
+protoc --cpp_out=. \
+       --grpc_out=. \
+       --plugin=protoc-gen-grpc=$(which grpc_cpp_plugin) \
+       ./jasper.proto
+```
+
+You will need the `protoc` and `grpc_cpp_plugin` binaries, which are part of
+the `protobuf` and `grpc++` projects respectively.