ci: add benchmarks (#1145)

This commit enables benchmarks for the Node.js agent. For every commit
to master, Jenkins will the benchmarks located in the `test/benchmarks`
directory.

Currently, those benchmarks are:

- `test/benchmarks/001-transaction-and-span-no-stack-trace.js`,
  Measures the overhead of starting and ending a transaction and a span
  without a stack trace

- `test/benchmarks/002-transaction-and-span-overhead-realistic-size.js`,
  Measures the overhead of starting and ending a transaction and a span
  with a realistic size stack trace

- `test/benchmarks/003-transaction-and-span-with-stack-trace.js`,
  Measures the overhead of starting and ending a transaction and a span
  with a simple uniform stack trace

- `test/benchmarks/004-transaction.js`,
  Measures the overhead of starting and ending a transaction only

- `test/benchmarks/005-transaction-reading-file.js`,
  Measures the overhead of starting and ending a transaction only while
  reading a file

The benchmarks are using the benchmark.js benchmarking tool. For each of
the benchmarks, it would be best to get the relative margin of error
down below 1%, but benchmark.js doesn't support running the benchmark
until a given relative margin of error threshold. So for now this commit
just run each of them for 2x60 seconds (60 seconds for the benchmark it
self and 60 seconds for the control), which in most cases gets them
below the magic 1%.

A PR has been opened to add this feature to benchmark.js:

bestiejs/benchmark.js#223

Each of the benchmarks a new document is added to our Elasticsearch
benchmarking cluster when running on Jenkins.

To add a new benchmark, simply add a new file to the `test/benchmarks`
directory.

To get more info about how to run the benchmarks locally, run:

    npm run bench -- --help

Closes #293 
Closes #306

.ci/schedule-daily.groovy

@@ -41,6 +41,7 @@ pipeline {
           job: 'apm-agent-nodejs/apm-agent-nodejs-mbp/master',
           parameters: [
             booleanParam(name: 'Run_As_Master_Branch', value: true),
+            booleanParam(name: 'bench_ci', value: false),
             booleanParam(name: 'doc_ci', value: true),
             booleanParam(name: 'tav_ci', value: true),
             booleanParam(name: 'test_edge_ci', value: true)

.ci/scripts/prepare-benchmarks-env.sh

@@ -0,0 +1,21 @@
+#!/usr/bin/env bash
+set -eo pipefail
+
+# This particular configuration is required to be installed in the baremetal
+curl -o- https://raw.githubusercontent.com/nvm-sh/nvm/v0.34.0/install.sh | bash
+export NVM_DIR="$HOME/.nvm"
+[ -s "$NVM_DIR/nvm.sh" ] && \. "$NVM_DIR/nvm.sh"
+command -v nvm
+
+## If NODE_VERSION env variable exists then use it otherwise use node as default
+if [ -z "${NODE_VERSION}" ] ; then
+  NODE_VERSION="node"
+fi
+nvm install ${NODE_VERSION}
+
+set +x
+npm config list
+npm install
+
+node --version
+npm --version

.ci/scripts/run-benchmarks.sh

@@ -0,0 +1,9 @@
+#!/usr/bin/env bash
+set -ueo pipefail
+
+SCRIPTPATH=$(dirname "$0")
+source ./${SCRIPTPATH}/prepare-benchmarks-env.sh
+
+RESULT_FILE=${1:-apm-agent-benchmark-results.json}
+
+npm run bench:ci ${RESULT_FILE}

.gitignore

@@ -10,3 +10,4 @@ test/types/transpile/index.js
 build
 coverage
 node_modules
+test/benchmarks/.tmp

Jenkinsfile

@@ -28,6 +28,7 @@ pipeline {
   }
   parameters {
     booleanParam(name: 'Run_As_Master_Branch', defaultValue: false, description: 'Allow to run any steps on a PR, some steps normally only run on master branch.')
+    booleanParam(name: 'bench_ci', defaultValue: true, description: 'Enable benchmarks.')
     booleanParam(name: 'tav_ci', defaultValue: true, description: 'Enable TAV tests.')
     booleanParam(name: 'tests_ci', defaultValue: true, description: 'Enable tests.')
     booleanParam(name: 'test_edge_ci', defaultValue: true, description: 'Enable tests for edge versions of nodejs.')
@@ -269,6 +270,52 @@ pipeline {
         githubNotify(context: "${env.GITHUB_CHECK_ITS_NAME}", description: "${env.GITHUB_CHECK_ITS_NAME} ...", status: 'PENDING', targetUrl: "${env.JENKINS_URL}search/?q=${env.ITS_PIPELINE.replaceAll('/','+')}")
       }
     }
+    /**
+      Run the benchmarks and store the results on ES.
+      The result JSON files are also archive into Jenkins.
+    */
+    stage('Benchmarks') {
+      agent { label 'metal' }
+      options { skipDefaultCheckout() }
+      environment {
+        HOME = "${env.WORKSPACE}"
+        RESULT_FILE = 'apm-agent-benchmark-results.json'
+        NODE_VERSION = '12'
+      }
+      when {
+        beforeAgent true
+        allOf {
+          anyOf {
+            branch 'master'
+            tag pattern: 'v\\d+\\.\\d+\\.\\d+.*', comparator: 'REGEXP'
+            expression { return params.Run_As_Master_Branch }
+          }
+          expression { return params.bench_ci }
+        }
+      }
+      steps {
+        withGithubNotify(context: 'Benchmarks', tab: 'artifacts') {
+          dir(env.BUILD_NUMBER) {
+            deleteDir()
+            unstash 'source'
+            dir(BASE_DIR){
+              sh '.ci/scripts/run-benchmarks.sh "${RESULT_FILE}"'
+            }
+          }
+        }
+      }
+      post {
+        always {
+          catchError(message: 'sendBenchmarks failed', buildResult: 'FAILURE') {
+            sendBenchmarks(file: "${BUILD_NUMBER}/${BASE_DIR}/${RESULT_FILE}",
+                           index: 'benchmark-nodejs', archive: true)
+          }
+          catchError(message: 'deleteDir failed', buildResult: 'SUCCESS', stageResult: 'UNSTABLE') {
+            deleteDir()
+          }
+        }
+      }
+    }
   }
   post {
     cleanup {

TESTING.md

@@ -124,3 +124,9 @@ Clean up Docker containers and volumes:
 ```
 npm run docker:clean
 ```
+
+Run the benchmarks:
+
+```
+npm run bench
+```

package.json

@@ -21,6 +21,8 @@
     "test:types": "tsc --project test/types/tsconfig.json && tsc --project test/types/transpile/tsconfig.json && node test/types/transpile/index.js",
     "test:babel": "babel test/babel/src.js --out-file test/babel/out.js && node test/babel/out.js",
     "test:esm": "node --experimental-modules test/esm",
+    "bench": "./test/benchmarks/scripts/run-benchmarks.sh",
+    "bench:ci": "./test/benchmarks/scripts/run-benchmarks-ci.sh",
     "local:start": "./test/script/local-deps-start.sh",
     "local:stop": "./test/script/local-deps-stop.sh",
     "docker:start": "docker-compose -f ./test/docker-compose.yml up -d",
@@ -113,8 +115,10 @@
     "@types/node": "^12.0.8",
     "apollo-server-express": "^2.6.3",
     "aws-sdk": "^2.477.0",
+    "benchmark": "^2.1.4",
     "bluebird": "^3.5.2",
     "cassandra-driver": "^4.0.0",
+    "columnify": "^1.5.4",
     "commitlint-config-squash-pr": "^1.0.0",
     "connect": "^3.7.0",
     "container-info": "^1.0.1",
@@ -148,6 +152,7 @@
     "mysql": "^2.16.0",
     "mysql2": "^1.6.3",
     "ndjson": "^1.5.0",
+    "numeral": "^2.0.6",
     "nyc": "^14.1.1",
     "once": "^1.4.0",
     "p-finally": "^1.0.0",

test/benchmarks/001-transaction-and-span-no-stack-trace.js

@@ -0,0 +1,29 @@
+'use strict'
+
+/* eslint-disable no-unused-vars, no-undef */
+
+const bench = require('./utils/bench')
+
+bench('transaction-and-span-no-stack-trace', {
+  agentConf: {
+    captureSpanStackTraces: false
+  },
+  setup () {
+    var agent = this.benchmark.agent
+  },
+  fn (deferred) {
+    if (agent) agent.startTransaction()
+    setImmediate(() => {
+      const span = agent && agent.startSpan()
+      setImmediate(() => {
+        if (agent) {
+          span.end()
+          agent.endTransaction()
+        }
+        setImmediate(() => {
+          deferred.resolve()
+        })
+      })
+    })
+  }
+})

test/benchmarks/002-transaction-and-span-overhead-realistic-size.js

@@ -0,0 +1,43 @@
+'use strict'
+
+/* eslint-disable no-unused-vars, no-undef */
+
+const bench = require('./utils/bench')
+
+bench('transaction-and-span-overhead-realistic-size', {
+  agentConf: {
+    captureSpanStackTraces: true
+  },
+  setup () {
+    var agent = this.benchmark.agent
+    var callstack = this.benchmark.callstack
+
+    // To avoid randomness, but still generate what appears to be natural random
+    // call stacks, number of spans etc, use a pre-defined set of numbers
+    var numbers = [2, 5, 10, 1, 2, 21, 2, 5, 6, 9, 1, 11, 9, 8, 12]
+    var numbersSpanIndex = 5
+    var numbersStackLevelIndex = 0
+
+    function addSpan (amount, cb) {
+      setImmediate(() => {
+        const span = agent && agent.startSpan()
+        setImmediate(() => {
+          if (agent) span.end()
+          if (--amount === 0) cb()
+          else addSpan(amount, cb)
+        })
+      })
+    }
+  },
+  fn (deferred) {
+    if (agent) agent.startTransaction()
+    const amount = numbers[numbersStackLevelIndex++ % numbers.length]
+    callstack(amount, () => {
+      const amount = numbers[numbersSpanIndex++ % numbers.length]
+      addSpan(amount, () => {
+        if (agent) agent.endTransaction()
+        deferred.resolve()
+      })
+    })
+  }
+})

test/benchmarks/003-transaction-and-span-with-stack-trace.js

@@ -0,0 +1,27 @@
+'use strict'
+
+/* eslint-disable no-unused-vars, no-undef */
+
+const bench = require('./utils/bench')
+
+bench('transaction-and-span-with-stack-trace', {
+  agentConf: {
+    captureSpanStackTraces: false
+  },
+  setup () {
+    var agent = this.benchmark.agent
+  },
+  fn (deferred) {
+    if (agent) agent.startTransaction()
+    setImmediate(() => {
+      const span = agent && agent.startSpan()
+      setImmediate(() => {
+        if (agent) {
+          span.end()
+          agent.endTransaction()
+        }
+        deferred.resolve()
+      })
+    })
+  }
+})

test/benchmarks/004-transaction.js

@@ -0,0 +1,20 @@
+'use strict'
+
+/* eslint-disable no-unused-vars, no-undef */
+
+const bench = require('./utils/bench')
+
+bench('transaction', {
+  setup () {
+    var agent = this.benchmark.agent
+  },
+  fn (deferred) {
+    if (agent) agent.startTransaction()
+    setImmediate(() => {
+      if (agent) agent.endTransaction()
+      setImmediate(() => {
+        deferred.resolve()
+      })
+    })
+  }
+})

test/benchmarks/005-transaction-reading-file.js

@@ -0,0 +1,21 @@
+'use strict'
+
+/* eslint-disable no-unused-vars, no-undef */
+
+const bench = require('./utils/bench')
+
+bench('transaction-reading-file', {
+  setup () {
+    var agent = this.benchmark.agent
+    var fs = this.benchmark.fs
+    var filename = this.benchmark.testFile
+  },
+  fn (deferred) {
+    if (agent) agent.startTransaction()
+    fs.readFile(filename, err => {
+      if (err) throw err
+      if (agent) agent.endTransaction()
+      deferred.resolve()
+    })
+  }
+})

test/benchmarks/scripts/run-benchmarks-ci.sh

@@ -0,0 +1,106 @@
+#!/usr/bin/env bash
+
+set -exo pipefail
+
+SCRIPTPATH=$(dirname "$0")
+RESULT_FILE=${1}
+
+if [ -z "$1" ]
+then
+  echo "Usage:"
+  echo "  run-benchmarks-ci.sh <output-file>"
+  echo
+  echo "Examples:"
+  echo "  run-benchmarks-ci.sh out.ndjson  - Run benchmark + store result in out.ndjson"
+  echo
+  exit
+fi
+
+echo $(pwd)
+
+function setUp() {
+    echo "Setting CPU frequency to base frequency"
+
+    CPU_MODEL=$(lscpu | grep "Model name" | awk '{for(i=3;i<=NF;i++){printf "%s ", $i}; printf "\n"}')
+    if [ "${CPU_MODEL}" == "Intel(R) Xeon(R) CPU E3-1246 v3 @ 3.50GHz " ]
+    then
+        # could also use `nproc`
+        CORE_INDEX=7
+        BASE_FREQ="3.5GHz"
+    elif [ "${CPU_MODEL}" == "Intel(R) Core(TM) i7-6700 CPU @ 3.40GHz " ]
+    then
+        CORE_INDEX=7
+        BASE_FREQ="3.4GHz"
+    elif [ "${CPU_MODEL}" == "Intel(R) Core(TM) i7-7700 CPU @ 3.60GHz " ]
+    then
+        CORE_INDEX=7
+        BASE_FREQ="3.6GHz"
+    elif [ "${CPU_MODEL}" == "Intel(R) Core(TM) i9-8950HK CPU @ 2.90GHz " ]
+    then
+        CORE_INDEX=9
+        BASE_FREQ="2.90GHz"
+    else
+        >&2 echo "Cannot determine base frequency for CPU model [${CPU_MODEL}]. Please adjust the build script."
+        exit 1
+    fi
+    MIN_FREQ=$(cpufreq-info -l -c 0 | awk '{print $1}')
+    # This is the frequency including Turbo Boost. See also http://ark.intel.com/products/80916/Intel-Xeon-Processor-E3-1246-v3-8M-Cache-3_50-GHz
+    MAX_FREQ=$(cpufreq-info -l -c 0 | awk '{print $2}')
+
+    # set all CPUs to the base frequency
+    for (( cpu=0; cpu<=${CORE_INDEX}; cpu++ ))
+    do
+        sudo -n cpufreq-set -c ${cpu} --min ${BASE_FREQ} --max ${BASE_FREQ}
+    done
+
+    # Build cgroups to isolate microbenchmarks and JVM threads
+    echo "Creating groups for OS and microbenchmarks"
+    # Isolate the OS to the first core
+    sudo -n cset set --set=/os --cpu=0-1
+    sudo -n cset proc --move --fromset=/ --toset=/os
+
+    # Isolate the microbenchmarks to all cores except the first two (first physical core)
+    # On a 4 core CPU with hyper threading, this would be 6 cores (3 physical cores)
+    sudo -n cset set --set=/benchmark --cpu=2-${CORE_INDEX}
+}
+
+function escape_quotes() {
+    echo $1 | sed -e 's/"/\\"/g'
+}
+
+# Escapes double quites in environment variables so that they are exported correctly
+function safe_env_export() {
+    echo "export $1=\"$(escape_quotes "${2}")\""
+}
+
+function benchmark() {
+    echo "export GIT_BUILD_CAUSE='${GIT_BUILD_CAUSE}'" > env_vars.sh
+    echo "export GIT_BASE_COMMIT='${GIT_BASE_COMMIT}'" >> env_vars.sh
+    echo "export GIT_COMMIT='${GIT_COMMIT}'" >> env_vars.sh
+    echo "export BRANCH_NAME='${BRANCH_NAME}'" >> env_vars.sh
+    echo "export CHANGE_ID='${CHANGE_ID}'" >> env_vars.sh
+    safe_env_export "CHANGE_TITLE" "${CHANGE_TITLE}" >> env_vars.sh
+    echo "export CHANGE_TARGET='${CHANGE_TARGET}'" >> env_vars.sh
+    echo "export CHANGE_URL='${CHANGE_URL}'" >> env_vars.sh
+    sudo -n cset proc --exec /benchmark -- ./"${SCRIPTPATH}"/run-benchmarks.sh all "${RESULT_FILE}"
+}
+
+function tearDown() {
+    echo "Destroying cgroups"
+    sudo -n cset set --destroy /os
+    sudo -n cset set --destroy /benchmark
+
+    echo "Setting normal frequency range"
+    for (( cpu=0; cpu<=${CORE_INDEX}; cpu++ ))
+    do
+        sudo -n cpufreq-set -c ${cpu} --min ${MIN_FREQ} --max ${MAX_FREQ}
+    done
+
+    echo "Delete env_vars.sh"
+    rm env_vars.sh || true
+}
+
+trap "tearDown" TERM EXIT
+
+setUp
+benchmark