From 59365e93e6ad4052fa08506fd6547f37dcab45ad Mon Sep 17 00:00:00 2001 From: tchapacan Date: Fri, 1 Nov 2024 21:45:32 +0100 Subject: [PATCH] add loadtest workflow & script --- .github/workflows/load-test.yml | 102 ++++++++++++++ .gitignore | 3 + benchmarks/load-test-workflow.js | 222 +++++++++++++++++++++++++++++++ benchmarks/middleware.js | 6 +- 4 files changed, 330 insertions(+), 3 deletions(-) create mode 100644 .github/workflows/load-test.yml create mode 100644 benchmarks/load-test-workflow.js diff --git a/.github/workflows/load-test.yml b/.github/workflows/load-test.yml new file mode 100644 index 0000000000..06a4833592 --- /dev/null +++ b/.github/workflows/load-test.yml @@ -0,0 +1,102 @@ +name: "Express Load Test" + +permissions: + contents: read + pull-requests: write + +on: + pull_request: + types: [ opened, synchronize ] + workflow_dispatch: + inputs: + prev_branch: + description: 'Base branch (branch-branch)' + required: false + default: '' + curr_branch: + description: 'Head branch (branch-branch)' + required: false + default: '' + prev_version: + description: 'Base Version (version-version)' + required: false + default: '' + curr_version: + description: 'Head Version (version-version)' + required: false + default: '' + version: + description: 'Version (version-branch)' + required: false + default: '' + branch: + description: 'Branch (version-branch)' + required: false + default: '' + +jobs: + load_test: + runs-on: ubuntu-latest + steps: + - name: Check Out Repository + uses: actions/checkout@11bd71901bbe5b1630ceea73d27597364c9af683 # v4.2.2 + with: + fetch-depth: 0 + + - name: Fetch All Branches + run: git fetch --all + + - name: Set Up Node.js + uses: actions/setup-node@39370e3970a6d050c480ffad4ff0ed4d3fdee5af #v4.1.0 + with: + node-version: '20' + + - name: Determine Comparison Type + run: | + if [[ "${{ github.event_name }}" == "pull_request" || "${{ github.event_name }}" == "push" ]]; then + # Branch comparison: Default to master for previous branch and use PR branch for current branch + echo "PREV_BRANCH=master" >> $GITHUB_ENV + echo "CURR_BRANCH=${{ github.head_ref || github.ref_name }}" >> $GITHUB_ENV + elif [[ "${{ github.event.inputs.prev_branch }}" && "${{ github.event.inputs.curr_branch }}" ]]; then + # Version comparison + echo "PREV_BRANCH=${{ github.event.inputs.prev_branch }}" >> $GITHUB_ENV + echo "CURR_BRANCH=${{ github.event.inputs.curr_branch }}" >> $GITHUB_ENV + elif [[ "${{ github.event.inputs.prev_version }}" && "${{ github.event.inputs.curr_version }}" ]]; then + # Version comparison + echo "PREV_VERSION=${{ github.event.inputs.prev_version }}" >> $GITHUB_ENV + echo "CURR_VERSION=${{ github.event.inputs.curr_version }}" >> $GITHUB_ENV + elif [[ "${{ github.event.inputs.branch }}" && "${{ github.event.inputs.version }}" ]]; then + # Branch-Version comparison + echo "BRANCH=${{ github.event.inputs.branch }}" >> $GITHUB_ENV + echo "VERSION=${{ github.event.inputs.version }}" >> $GITHUB_ENV + else + echo "Invalid input combination. Provide either two branches, two versions, or one branch and one version." + exit 1 + fi + + - name: Install wrk + run: | + sudo apt-get update + sudo apt-get install -y wrk + + - name: Start Load Test Server + run: node benchmarks/load-test-workflow.js + env: + PREV_BRANCH: ${{ env.PREV_BRANCH }} + CURR_BRANCH: ${{ env.CURR_BRANCH }} + PREV_VERSION: ${{ env.PREV_VERSION }} + CURR_VERSION: ${{ env.CURR_VERSION }} + BRANCH: ${{ env.BRANCH }} + VERSION: ${{ env.VERSION }} + + - name: Output Summary + run: | + cat benchmarks/results*.md >> $GITHUB_STEP_SUMMARY + + - name: Post Summary to PR + if: github.event_name == 'pull_request' + run: | + cat $GITHUB_STEP_SUMMARY + gh pr comment ${{ github.event.pull_request.number }} --body "$(cat benchmarks/results*.md)" + env: + GH_TOKEN: ${{ github.token }} diff --git a/.gitignore b/.gitignore index 1bd5c02b28..33e4303b56 100644 --- a/.gitignore +++ b/.gitignore @@ -12,5 +12,8 @@ coverage # Benchmarking benchmarks/graphs +# Webstorm +.idea + # ignore additional files using core.excludesFile # https://git-scm.com/docs/gitignore diff --git a/benchmarks/load-test-workflow.js b/benchmarks/load-test-workflow.js new file mode 100644 index 0000000000..3f197faf09 --- /dev/null +++ b/benchmarks/load-test-workflow.js @@ -0,0 +1,222 @@ +const {execSync, spawn} = require('child_process') +const fs = require('fs') + +const runCommand = command => execSync(command, {encoding: 'utf8'}).trim() + +const startServer = (middleware, isVersionTest) => { + console.log(`Starting server with ${middleware} middleware layers...`) + const server = spawn('node', ['benchmarks/middleware.js'], { + env: { + ...process.env, + MW: middleware, + NO_LOCAL_EXPRESS: isVersionTest + }, + stdio: 'inherit' + }) + + return new Promise((resolve, reject) => { + setTimeout(() => { + try { + execSync('curl -s http://127.0.0.1:3333') + resolve(server) + } catch (error) { + server.kill() + reject(new Error('Server failed to start.')) + } + }, 3000) + }) +} + +const runLoadTest = (url, connectionsList) => { + return connectionsList.map(connections => { + try { + const output = runCommand(`wrk ${url} -d 3 -c ${connections} -t 8`) + const reqSec = output.match(/Requests\/sec:\s+(\d+.\d+)/)?.[1] + const latency = output.match(/Latency\s+(\d+.\d+)/)?.[1] + return {connections, reqSec, latency} + } catch (error) { + console.error( + `Error running load test for ${connections} connections:`, + error.message + ) + return {connections, reqSec: 'N/A', latency: 'N/A'} + } + }) +} + +const generateMarkdownTable = results => { + const headers = `| Connections | Requests/sec | Latency |\n|-------------|--------------|---------|` + const rows = results + .map( + r => `| ${r.connections} | ${r.reqSec || 'N/A'} | ${r.latency || 'N/A'} |` + ) + .join('\n') + return `${headers}\n${rows}` +} + +const cleanUp = () => { + console.log('Cleaning up...') + runCommand('npm uninstall express') + runCommand('rm -rf package-lock.json node_modules') +} + +const runTests = async ({ + identifier, + connectionsList, + middlewareCounts, + outputFile, + isVersionTest = false +}) => { + if (isVersionTest) { + console.log(`Installing Express v${identifier}...`) + runCommand(`npm install express@${identifier}`) + } else { + console.log(`Checking out branch ${identifier}...`) + runCommand(`git fetch origin ${identifier}`) + runCommand(`git checkout ${identifier}`) + runCommand('npm install') + console.log('Installing deps...') + } + + const resultsMarkdown = [ + `\n\n# Load Test Results for ${isVersionTest ? `Express v${identifier}` : `Branch ${identifier}`}` + ] + + for (const middlewareCount of middlewareCounts) { + try { + const server = await startServer(middlewareCount, isVersionTest) + const results = runLoadTest( + 'http://127.0.0.1:3333/?foo[bar]=baz', + connectionsList + ) + server.kill() + resultsMarkdown.push( + `### Load test for ${middlewareCount} middleware layers\n\n${generateMarkdownTable(results)}` + ) + } catch (error) { + console.error('Error in load test process:', error) + } + } + + fs.writeFileSync(outputFile, resultsMarkdown.join('\n\n')) + cleanUp() +} + +const compareBranches = async ({ + prevBranch, + currBranch, + connectionsList, + middlewareCounts, +}) => { + console.log(`Comparing branches: ${prevBranch} vs ${currBranch}`) + await runTests({ + identifier: prevBranch, + connectionsList, + middlewareCounts, + outputFile: `benchmarks/results_${prevBranch}.md`, + isVersionTest: false + }) + await runTests({ + identifier: currBranch, + connectionsList, + middlewareCounts, + outputFile: `benchmarks/results_${currBranch}.md`, + isVersionTest: false + }) +} + +const compareVersions = async ({ + prevVersion, + currVersion, + connectionsList, + middlewareCounts, +}) => { + console.log( + `Comparing versions: Express v${prevVersion} vs Express v${currVersion}` + ) + await runTests({ + identifier: prevVersion, + connectionsList, + middlewareCounts, + outputFile: `benchmarks/results_${prevVersion}.md`, + isVersionTest: true + }) + await runTests({ + identifier: currVersion, + connectionsList, + middlewareCounts, + outputFile: `benchmarks/results_${currVersion}.md`, + isVersionTest: true + }) +} + +const compareBranchAndVersion = async ({ + branch, + version, + connectionsList, + middlewareCounts, +}) => { + console.log(`Comparing branch ${branch} with Express version ${version}`) + await runTests({ + identifier: branch, + connectionsList, + middlewareCounts, + outputFile: `benchmarks/results_${branch}.md`, + isVersionTest: false + }) + await runTests({ + identifier: version, + connectionsList, + middlewareCounts, + outputFile: `benchmarks/results_${version}.md`, + isVersionTest: true + }) +} + +const main = async () => { + const connectionsList = [50, 100, 250] + const middlewareCounts = [1, 10, 25, 50] + const prevBranch = process.env.PREV_BRANCH + const currBranch = process.env.CURR_BRANCH + const prevVersion = process.env.PREV_VERSION + const currVersion = process.env.CURR_VERSION + const version = process.env.VERSION + const branch = process.env.BRANCH + + if (prevBranch && currBranch) { + await compareBranches({ + prevBranch, + currBranch, + connectionsList, + middlewareCounts, + }) + return + } + + if (prevVersion && currVersion) { + await compareVersions({ + prevVersion, + currVersion, + connectionsList, + middlewareCounts, + }) + return + } + + if (branch && version) { + await compareBranchAndVersion({ + branch, + version, + connectionsList, + middlewareCounts, + }) + return + } + + console.error( + 'Invalid input combination. Provide either two branches, two versions, or one branch and one version.' + ) + process.exit(1) +} + +main() diff --git a/benchmarks/middleware.js b/benchmarks/middleware.js index fed97ba8ce..ed9810eadf 100644 --- a/benchmarks/middleware.js +++ b/benchmarks/middleware.js @@ -1,10 +1,10 @@ -var express = require('..'); -var app = express(); +const express = process.env.NO_LOCAL_EXPRESS === "true" ? require('express') : require('..'); +const app = express(); // number of middleware -var n = parseInt(process.env.MW || '1', 10); +let n = parseInt(process.env.MW || '1', 10); console.log(' %s middleware', n); while (n--) {