chore: Fix benchmarking script

hubot-friends · Jan 7, 2024 · 3e97b1d · 3e97b1d
1 parent 0c2fa05
commit 3e97b1d
Show file tree

Hide file tree

Showing 8 changed files with 495 additions and 58 deletions.
diff --git a/benchmarks/Makefile b/benchmarks/Makefile
diff --git a/benchmarks/README.md b/benchmarks/README.md
@@ -0,0 +1,79 @@
+# Benchmark history
+
+
+# 2024-01-06
+
+Redesigned Express to not create a prototype chain on the req/res objects. The following are the benchmark results between the old version (Express 5 - with prototype chaining) with the new design.
+
+**System Specs**
+
+- Mac mini
+- Apple M1
+- Memory: 16 GB
+- OS Version: 14.2.1 macOS Sonoma
+
+## Old design (with prototype chain)
+
+    autocannon -c 100 -d 5 -p 10 http://localhost:3333/?foo[bar]=baz
+
+    Running 5s test @ http://localhost:3333/?foo[bar]=baz
+    100 connections with 10 pipelining factor
+
+
+    ┌─────────┬───────┬───────┬───────┬───────┬──────────┬──────────┬─────────┐
+    │ Stat    │ 2.5%  │ 50%   │ 97.5% │ 99%   │ Avg      │ Stdev    │ Max     │
+    ├─────────┼───────┼───────┼───────┼───────┼──────────┼──────────┼─────────┤
+    │ Latency │ 16 ms │ 29 ms │ 66 ms │ 70 ms │ 38.63 ms │ 61.67 ms │ 1562 ms │
+    └─────────┴───────┴───────┴───────┴───────┴──────────┴──────────┴─────────┘
+    ┌───────────┬─────────┬─────────┬─────────┬─────────┬──────────┬──────────┬─────────┐
+    │ Stat      │ 1%      │ 2.5%    │ 50%     │ 97.5%   │ Avg      │ Stdev    │ Min     │
+    ├───────────┼─────────┼─────────┼─────────┼─────────┼──────────┼──────────┼─────────┤
+    │ Req/Sec   │ 22,351  │ 22,351  │ 26,143  │ 26,543  │ 25,460.8 │ 1,568.25 │ 22,336  │
+    ├───────────┼─────────┼─────────┼─────────┼─────────┼──────────┼──────────┼─────────┤
+    │ Bytes/Sec │ 5.32 MB │ 5.32 MB │ 6.22 MB │ 6.32 MB │ 6.06 MB  │ 375 kB   │ 5.32 MB │
+    └───────────┴─────────┴─────────┴─────────┴─────────┴──────────┴──────────┴─────────┘
+
+    Req/Bytes counts sampled once per second.
+    # of samples: 5
+
+    128k requests in 5.02s, 30.3 MB read
+
+## New design
+
+    autocannon -c 100 -d 5 -p 10 http://localhost:3333/?foo[bar]=baz
+
+    Running 5s test @ http://localhost:3333/?foo[bar]=baz
+    100 connections with 10 pipelining factor
+
+
+    ┌─────────┬──────┬───────┬───────┬───────┬──────────┬──────────┬────────┐
+    │ Stat    │ 2.5% │ 50%   │ 97.5% │ 99%   │ Avg      │ Stdev    │ Max    │
+    ├─────────┼──────┼───────┼───────┼───────┼──────────┼──────────┼────────┤
+    │ Latency │ 9 ms │ 16 ms │ 22 ms │ 26 ms │ 16.32 ms │ 11.46 ms │ 471 ms │
+    └─────────┴──────┴───────┴───────┴───────┴──────────┴──────────┴────────┘
+    ┌───────────┬─────────┬─────────┬─────────┬─────────┬──────────┬──────────┬─────────┐
+    │ Stat      │ 1%      │ 2.5%    │ 50%     │ 97.5%   │ Avg      │ Stdev    │ Min     │
+    ├───────────┼─────────┼─────────┼─────────┼─────────┼──────────┼──────────┼─────────┤
+    │ Req/Sec   │ 54,943  │ 54,943  │ 60,575  │ 60,703  │ 59,363.2 │ 2,230.44 │ 54,922  │
+    ├───────────┼─────────┼─────────┼─────────┼─────────┼──────────┼──────────┼─────────┤
+    │ Bytes/Sec │ 13.1 MB │ 13.1 MB │ 14.4 MB │ 14.4 MB │ 14.1 MB  │ 531 kB   │ 13.1 MB │
+    └───────────┴─────────┴─────────┴─────────┴─────────┴──────────┴──────────┴─────────┘
+
+    Req/Bytes counts sampled once per second.
+    # of samples: 5
+
+    298k requests in 5.02s, 70.6 MB read
+
+# Results
+
+## Version 3 - 1/6/2024, 6:51:42 PM
+
+| Stat | 2.5% | 50% | 97.5% | 99% | Avg | Stdev | Max |
+|:-----|:----:|:---:|:-----:|:---:|:---:|:-----:|:---:|
+| Latency | 7 ms | 18 ms | 21 ms | 26 ms | 15.23 ms | 17.1 ms | 600 ms |
+
+| Stat | 1% | 2.5% | 50% | 97.5% | Avg | Stdev | Min |
+|:-----|:--:|:----:|:---:|:-----:|:---:|:-----:|:---:|
+| Req/Sec | 56,255 | 56,255 | 64,511 | 66,495 | 63,388.8 | 3,687.84 | 56,241 |
+| Bytes/Sec | 13.4 MB | 13.4 MB | 15.4 MB | 15.8 MB | 15.1 MB | 877 kB | 13.4 MB |
+
diff --git a/benchmarks/index.mjs b/benchmarks/index.mjs
@@ -0,0 +1,128 @@
+
+import express from '../index.js'
+import autocannon from 'autocannon'
+import prettyBytes from 'pretty-bytes'
+import fs from 'node:fs'
+
+const toMs = (ns) => {
+    return `${Math.floor(ns * 100) / 100} ms`;
+}
+const format = (ns) => {
+    return ns.toLocaleString({minimumFractionDigits: 2, maximumFractionDigits: 2})
+}
+
+const toTable = (data) => {
+    const { latency, requests, throughput } = data
+    const latencyTable = `
+| Stat | 2.5% | 50% | 97.5% | 99% | Avg | Stdev | Max |
+|:-----|:----:|:---:|:-----:|:---:|:---:|:-----:|:---:|
+| Latency | ${toMs(latency.p2_5)} | ${toMs(latency.p50)} | ${toMs(latency.p97_5)} | ${toMs(latency.p99)} | ${toMs(latency.average)} | ${toMs(latency.stddev)} | ${toMs(latency.max)} |
+`
+    const rateTable = `
+| Stat | 1% | 2.5% | 50% | 97.5% | Avg | Stdev | Min |
+|:-----|:--:|:----:|:---:|:-----:|:---:|:-----:|:---:|
+| Req/Sec | ${format(requests.p1)} | ${format(requests.p2_5)} | ${format(requests.p50)} | ${format(requests.p97_5)} | ${format(requests.average)} | ${format(requests.stddev)} | ${format(requests.min)} |
+| Bytes/Sec | ${prettyBytes(throughput.p1)} | ${prettyBytes(throughput.p2_5)} | ${prettyBytes(throughput.p50)} | ${prettyBytes(throughput.p97_5)} | ${prettyBytes(throughput.average)} | ${prettyBytes(throughput.stddev)} | ${prettyBytes(throughput.min)} |
+`
+    return latencyTable + rateTable
+}
+const toHtmlTable = (data) => {
+    const { latency, requests, throughput } = data
+    const latencyTable = `
+<table>
+<thead>
+    <tr>
+        <th>Stat</th>
+        <th>2.5%</th>
+        <th>50%</th>
+        <th>97.5%</th>
+        <th>99%</th>
+        <th>Avg</th>
+        <th>Stdev</th>
+        <th>Max</th>
+    </tr>
+</thead>
+<tbody>
+    <tr>
+        <td>Latency</td>
+        <td>${toMs(latency.p2_5)} ms</td>
+        <td>${toMs(latency.p50)} ms</td>
+        <td>${toMs(latency.p97_5)} ms</td>
+        <td>${toMs(latency.p99)} ms</td>
+        <td>${toMs(latency.average)} ms</td>
+        <td>${toMs(latency.stddev)} ms</td>
+        <td>${toMs(latency.max)} ms</td>
+    </tr>
+</tbody>
+</table>
+`
+    const rateTable = `
+<table>
+<thead>
+    <tr>
+        <th>Stat</th>
+        <th>1%</th>
+        <th>2.5%</th>
+        <th>50%</th>
+        <th>97.5%</th>
+        <th>Avg</th>
+        <th>Stdev</th>
+        <th>Min</th>
+    </tr>
+</thead>
+<tbody>
+    <tr>
+        <td>Req/Sec</td>
+        <td>${requests.p1.toLocaleString()}</td>
+        <td>${requests.p2_5.toLocaleString()}</td>
+        <td>${requests.p50.toLocaleString()}</td>
+        <td>${requests.p97_5.toLocaleString()}</td>
+        <td>${requests.average.toLocaleString()}</td>
+        <td>${requests.stddev.toLocaleString()}</td>
+        <td>${requests.min.toLocaleString()}</td>
+    </tr>
+    <tr>
+        <td>Bytes/Sec</td>
+        <td>${prettyBytes(throughput.p1)}</td>
+        <td>${prettyBytes(throughput.p2_5)}</td>
+        <td>${prettyBytes(throughput.p50)}</td>
+        <td>${prettyBytes(throughput.p97_5)}</td>
+        <td>${prettyBytes(throughput.average)}</td>
+        <td>${prettyBytes(throughput.stddev)}</td>
+        <td>${prettyBytes(throughput.min)}</td>
+    </tr>
+</tbody>
+</table>
+`
+    return latencyTable + rateTable
+}
+const app = express()
+let n = parseInt(process.env.MW || '1', 10)
+console.log('  %s middleware', n)
+while (n--) {
+  app.use(function(req, res, next){
+    next()
+  })
+}
+app.use((req, res) => {
+  res.send('Hello World')
+})
+app.on('listening', function () {
+  const { port } = this.address()
+  autocannon({
+    url: `http://localhost:${port}/?foo[bar]=baz`,
+    pipelining: 10, 
+    connections: 100,
+    duration: 5,
+    title: 'Version 3',
+    workers: 4
+  }, (err, result) => {
+    console.log(toTable(result))
+    fs.appendFile(`benchmarks/README.md`, `## ${result.title} - ${new Date().toLocaleString()}
+${toTable(result)}\n`, 'utf8', () => {
+        server.close()
+    })
+  })
+})
+const server = app.listen()
+
diff --git a/benchmarks/middleware.js b/benchmarks/middleware.js
diff --git a/benchmarks/run b/benchmarks/run
diff --git a/lib/application.js b/lib/application.js
@@ -797,7 +797,7 @@ class ExpressApp extends EventEmitter {
 
   listen(...args) {
     this.#server = http.createServer({ IncomingMessage: ExpressRequest, ServerResponse: ExpressResponse }, this.handle.bind(this))
-    return this.#server.listen(...args)
+    return this.#server.listen(...args, () => this.emit('listening'))
   }
   address () {
     return this.#server?.address() ?? null