Benchmarking (#8)

* chore: Fix benchmarking script * chore: add to ci * fix the hanging
hubot-friends · Jan 7, 2024 · 480db6c · 480db6c
1 parent 0c2fa05
commit 480db6c
Show file tree

Hide file tree

Showing 8 changed files with 524 additions and 57 deletions.
diff --git a/.github/workflows/ci.yml b/.github/workflows/ci.yml
@@ -49,6 +49,27 @@ jobs:
           cache: 'npm'
       - run: npm ci
       - run: npm test
+  benchmark:
+    name: Benchmark
+    runs-on: ubuntu-latest
+    needs: build
+    strategy:
+      matrix:
+        node-version:
+          - 20.x
+    steps:
+      - name: Checkout
+        uses: actions/checkout@v3
+        with:
+          fetch-depth: 0
+      - name: Use Node.js ${{ matrix.node-version }}
+        uses: actions/setup-node@v3
+        with:
+          node-version: ${{ matrix.node-version }}
+          cache: 'npm'
+      - run: npm ci
+      - run: npm run benchmark
+
   release:
     name: Release
     if: github.ref == 'refs/heads/main' && success()

diff --git a/benchmarks/Makefile b/benchmarks/Makefile
diff --git a/benchmarks/README.md b/benchmarks/README.md
@@ -0,0 +1,90 @@
+# Benchmark history
+
+
+# 2024-01-06
+
+Redesigned Express to not create a prototype chain on the req/res objects. The following are the benchmark results between the old version (Express 5 - with prototype chaining) with the new design.
+
+**System Specs**
+
+- Mac mini
+- Apple M1
+- Memory: 16 GB
+- OS Version: 14.2.1 macOS Sonoma
+
+## Old design (with prototype chain)
+
+    autocannon -c 100 -d 5 -p 10 http://localhost:3333/?foo[bar]=baz
+
+    Running 5s test @ http://localhost:3333/?foo[bar]=baz
+    100 connections with 10 pipelining factor
+
+
+    ┌─────────┬───────┬───────┬───────┬───────┬──────────┬──────────┬─────────┐
+    │ Stat    │ 2.5%  │ 50%   │ 97.5% │ 99%   │ Avg      │ Stdev    │ Max     │
+    ├─────────┼───────┼───────┼───────┼───────┼──────────┼──────────┼─────────┤
+    │ Latency │ 16 ms │ 29 ms │ 66 ms │ 70 ms │ 38.63 ms │ 61.67 ms │ 1562 ms │
+    └─────────┴───────┴───────┴───────┴───────┴──────────┴──────────┴─────────┘
+    ┌───────────┬─────────┬─────────┬─────────┬─────────┬──────────┬──────────┬─────────┐
+    │ Stat      │ 1%      │ 2.5%    │ 50%     │ 97.5%   │ Avg      │ Stdev    │ Min     │
+    ├───────────┼─────────┼─────────┼─────────┼─────────┼──────────┼──────────┼─────────┤
+    │ Req/Sec   │ 22,351  │ 22,351  │ 26,143  │ 26,543  │ 25,460.8 │ 1,568.25 │ 22,336  │
+    ├───────────┼─────────┼─────────┼─────────┼─────────┼──────────┼──────────┼─────────┤
+    │ Bytes/Sec │ 5.32 MB │ 5.32 MB │ 6.22 MB │ 6.32 MB │ 6.06 MB  │ 375 kB   │ 5.32 MB │
+    └───────────┴─────────┴─────────┴─────────┴─────────┴──────────┴──────────┴─────────┘
+
+    Req/Bytes counts sampled once per second.
+    # of samples: 5
+
+    128k requests in 5.02s, 30.3 MB read
+
+## New design
+
+    autocannon -c 100 -d 5 -p 10 http://localhost:3333/?foo[bar]=baz
+
+    Running 5s test @ http://localhost:3333/?foo[bar]=baz
+    100 connections with 10 pipelining factor
+
+
+    ┌─────────┬──────┬───────┬───────┬───────┬──────────┬──────────┬────────┐
+    │ Stat    │ 2.5% │ 50%   │ 97.5% │ 99%   │ Avg      │ Stdev    │ Max    │
+    ├─────────┼──────┼───────┼───────┼───────┼──────────┼──────────┼────────┤
+    │ Latency │ 9 ms │ 16 ms │ 22 ms │ 26 ms │ 16.32 ms │ 11.46 ms │ 471 ms │
+    └─────────┴──────┴───────┴───────┴───────┴──────────┴──────────┴────────┘
+    ┌───────────┬─────────┬─────────┬─────────┬─────────┬──────────┬──────────┬─────────┐
+    │ Stat      │ 1%      │ 2.5%    │ 50%     │ 97.5%   │ Avg      │ Stdev    │ Min     │
+    ├───────────┼─────────┼─────────┼─────────┼─────────┼──────────┼──────────┼─────────┤
+    │ Req/Sec   │ 54,943  │ 54,943  │ 60,575  │ 60,703  │ 59,363.2 │ 2,230.44 │ 54,922  │
+    ├───────────┼─────────┼─────────┼─────────┼─────────┼──────────┼──────────┼─────────┤
+    │ Bytes/Sec │ 13.1 MB │ 13.1 MB │ 14.4 MB │ 14.4 MB │ 14.1 MB  │ 531 kB   │ 13.1 MB │
+    └───────────┴─────────┴─────────┴─────────┴─────────┴──────────┴──────────┴─────────┘
+
+    Req/Bytes counts sampled once per second.
+    # of samples: 5
+
+    298k requests in 5.02s, 70.6 MB read
+
+# Results
+
+## Version 3 - 1/6/2024, 6:51:42 PM
+
+| Stat | 2.5% | 50% | 97.5% | 99% | Avg | Stdev | Max |
+|:-----|:----:|:---:|:-----:|:---:|:---:|:-----:|:---:|
+| Latency | 7 ms | 18 ms | 21 ms | 26 ms | 15.23 ms | 17.1 ms | 600 ms |
+
+| Stat | 1% | 2.5% | 50% | 97.5% | Avg | Stdev | Min |
+|:-----|:--:|:----:|:---:|:-----:|:---:|:-----:|:---:|
+| Req/Sec | 56,255 | 56,255 | 64,511 | 66,495 | 63,388.8 | 3,687.84 | 56,241 |
+| Bytes/Sec | 13.4 MB | 13.4 MB | 15.4 MB | 15.8 MB | 15.1 MB | 877 kB | 13.4 MB |
+
+## Version 3 - 1/6/2024, 7:19:01 PM
+
+| Stat | 2.5% | 50% | 97.5% | 99% | Avg | Stdev | Max |
+|:-----|:----:|:---:|:-----:|:---:|:---:|:-----:|:---:|
+| Latency | 8 ms | 18 ms | 20 ms | 22 ms | 15.4 ms | 18.39 ms | 667 ms |
+
+| Stat | 1% | 2.5% | 50% | 97.5% | Avg | Stdev | Min |
+|:-----|:--:|:----:|:---:|:-----:|:---:|:-----:|:---:|
+| Req/Sec | 52,095 | 52,095 | 65,439 | 65,855 | 62,758.4 | 5,344.96 | 52,088 |
+| Bytes/Sec | 12.4 MB | 12.4 MB | 15.6 MB | 15.7 MB | 14.9 MB | 1.27 MB | 12.4 MB |
+
diff --git a/benchmarks/index.mjs b/benchmarks/index.mjs
@@ -0,0 +1,126 @@
+
+import express from '../index.js'
+import autocannon from 'autocannon'
+import prettyBytes from 'pretty-bytes'
+import fs from 'node:fs'
+
+const toMs = (ns) => {
+    return `${Math.floor(ns * 100) / 100} ms`;
+}
+const format = (ns) => {
+    return ns.toLocaleString({minimumFractionDigits: 2, maximumFractionDigits: 2})
+}
+
+const toTable = (data) => {
+    const { latency, requests, throughput } = data
+    const latencyTable = `
+| Stat | 2.5% | 50% | 97.5% | 99% | Avg | Stdev | Max |
+|:-----|:----:|:---:|:-----:|:---:|:---:|:-----:|:---:|
+| Latency | ${toMs(latency.p2_5)} | ${toMs(latency.p50)} | ${toMs(latency.p97_5)} | ${toMs(latency.p99)} | ${toMs(latency.average)} | ${toMs(latency.stddev)} | ${toMs(latency.max)} |
+`
+    const rateTable = `
+| Stat | 1% | 2.5% | 50% | 97.5% | Avg | Stdev | Min |
+|:-----|:--:|:----:|:---:|:-----:|:---:|:-----:|:---:|
+| Req/Sec | ${format(requests.p1)} | ${format(requests.p2_5)} | ${format(requests.p50)} | ${format(requests.p97_5)} | ${format(requests.average)} | ${format(requests.stddev)} | ${format(requests.min)} |
+| Bytes/Sec | ${prettyBytes(throughput.p1)} | ${prettyBytes(throughput.p2_5)} | ${prettyBytes(throughput.p50)} | ${prettyBytes(throughput.p97_5)} | ${prettyBytes(throughput.average)} | ${prettyBytes(throughput.stddev)} | ${prettyBytes(throughput.min)} |
+`
+    return latencyTable + rateTable
+}
+const toHtmlTable = (data) => {
+    const { latency, requests, throughput } = data
+    const latencyTable = `
+<table>
+<thead>
+    <tr>
+        <th>Stat</th>
+        <th>2.5%</th>
+        <th>50%</th>
+        <th>97.5%</th>
+        <th>99%</th>
+        <th>Avg</th>
+        <th>Stdev</th>
+        <th>Max</th>
+    </tr>
+</thead>
+<tbody>
+    <tr>
+        <td>Latency</td>
+        <td>${toMs(latency.p2_5)} ms</td>
+        <td>${toMs(latency.p50)} ms</td>
+        <td>${toMs(latency.p97_5)} ms</td>
+        <td>${toMs(latency.p99)} ms</td>
+        <td>${toMs(latency.average)} ms</td>
+        <td>${toMs(latency.stddev)} ms</td>
+        <td>${toMs(latency.max)} ms</td>
+    </tr>
+</tbody>
+</table>
+`
+    const rateTable = `
+<table>
+<thead>
+    <tr>
+        <th>Stat</th>
+        <th>1%</th>
+        <th>2.5%</th>
+        <th>50%</th>
+        <th>97.5%</th>
+        <th>Avg</th>
+        <th>Stdev</th>
+        <th>Min</th>
+    </tr>
+</thead>
+<tbody>
+    <tr>
+        <td>Req/Sec</td>
+        <td>${requests.p1.toLocaleString()}</td>
+        <td>${requests.p2_5.toLocaleString()}</td>
+        <td>${requests.p50.toLocaleString()}</td>
+        <td>${requests.p97_5.toLocaleString()}</td>
+        <td>${requests.average.toLocaleString()}</td>
+        <td>${requests.stddev.toLocaleString()}</td>
+        <td>${requests.min.toLocaleString()}</td>
+    </tr>
+    <tr>
+        <td>Bytes/Sec</td>
+        <td>${prettyBytes(throughput.p1)}</td>
+        <td>${prettyBytes(throughput.p2_5)}</td>
+        <td>${prettyBytes(throughput.p50)}</td>
+        <td>${prettyBytes(throughput.p97_5)}</td>
+        <td>${prettyBytes(throughput.average)}</td>
+        <td>${prettyBytes(throughput.stddev)}</td>
+        <td>${prettyBytes(throughput.min)}</td>
+    </tr>
+</tbody>
+</table>
+`
+    return latencyTable + rateTable
+}
+const app = express()
+let n = parseInt(process.env.MW || '1', 10)
+console.log('  %s middleware', n)
+while (n--) {
+  app.use(function(req, res, next){
+    next()
+  })
+}
+app.use((req, res) => {
+  res.send('Hello World')
+})
+const server = app.listen()
+const { port } = server.address()
+autocannon({
+  url: `http://localhost:${port}/?foo[bar]=baz`,
+  pipelining: 10, 
+  connections: 100,
+  duration: 5,
+  title: 'Version 3',
+  workers: 4
+}, (err, result) => {
+  console.log(toTable(result))
+  fs.appendFile(`benchmarks/README.md`, `## ${result.title} - ${new Date().toLocaleString()}
+${toTable(result)}\n`, 'utf8', () => {
+      server.close()
+  })
+})
+
diff --git a/benchmarks/middleware.js b/benchmarks/middleware.js
diff --git a/benchmarks/run b/benchmarks/run