chore: integration tests setup

.github/actions/test/action.yml

@@ -19,3 +19,6 @@ runs:
     - run: npm run test:miniflare
       name: Miniflare Tests
       shell: bash
+    - run: npm run test:integration
+      name: Integration Tests
+      shell: bash

package.json

@@ -18,8 +18,9 @@
     "build": "esbuild --bundle src/index.js --format=esm --sourcemap --minify --outfile=dist/worker.mjs && npm run build:tsc",
     "build:debug": "esbuild --bundle src/index.js --format=esm --outfile=dist/worker.mjs",
     "build:tsc": "tsc --build",
-    "test:miniflare": "npm run build:debug && mocha --experimental-vm-modules --recursive test/**/*.spec.js",
+    "test:miniflare": "npm run build:debug && mocha --experimental-vm-modules --recursive test/miniflare/**/*.spec.js",
     "test:unit": "npm run build:debug && mocha --experimental-vm-modules --recursive test/unit/**/*.spec.js",
+    "test:integration": "npm run build:debug && mocha --experimental-vm-modules --recursive test/integration/**/*.spec.js --require test/fixtures/worker-fixture.js",
     "lint": "standard",
     "lint:fix": "standard --fix"
   },
@@ -56,6 +57,7 @@
     "multipart-byte-range": "^3.0.1",
     "sinon": "^19.0.2",
     "standard": "^17.1.0",
+    "tree-kill": "^1.2.2",
     "typescript": "^5.3.3",
     "wrangler": "^3.78.8"
   },

src/bindings.d.ts

@@ -12,7 +12,7 @@ export interface Environment {
   ACCOUNTING_SERVICE_URL: string
   RATE_LIMITER: RateLimit
   AUTH_TOKEN_METADATA: KVNamespace
-  FF_RATE_LIMITER_ENABLED: boolean
+  FF_RATE_LIMITER_ENABLED: string
 }
 
 export type RateLimitExceeded = typeof RATE_LIMIT_EXCEEDED[keyof typeof RATE_LIMIT_EXCEEDED]

src/handlers/rate-limiter.js

@@ -19,14 +19,13 @@ import { Accounting } from '../services/accounting.js'
  */
 export function withRateLimit (handler) {
   return async (req, env, ctx) => {
-    if (env.FF_RATE_LIMITER_ENABLED !== true) {
+    if (env.FF_RATE_LIMITER_ENABLED !== 'true') {
       return handler(req, env, ctx)
     }
 
     const { dataCid } = ctx
-    const rateLimitService = createRateLimitService(env, ctx)
+    const rateLimitService = create(env, ctx)
     const isRateLimitExceeded = await rateLimitService.check(dataCid, req)
-
     if (isRateLimitExceeded === RATE_LIMIT_EXCEEDED.YES) {
       throw new HttpError('Too Many Requests', { status: 429 })
     } else {
@@ -43,7 +42,7 @@ export function withRateLimit (handler) {
  * @param {IpfsUrlContext} ctx
  * @returns {RateLimitService}
  */
-function createRateLimitService (env, ctx) {
+function create (env, ctx) {
   return {
     /**
      * @param {import('multiformats/cid').CID} cid

test/fixtures/worker-fixture.js

@@ -0,0 +1,75 @@
+import path, { resolve } from 'node:path'
+import { fileURLToPath } from 'node:url'
+import { runWranglerDev } from '../helpers/run-wrangler.js'
+
+// Get __dirname equivalent in ES module
+const __filename = fileURLToPath(import.meta.url)
+const __dirname = path.dirname(__filename)
+
+/**
+ * Worker information object
+ * @typedef {Object} WorkerInfo
+ * @property {string | undefined} ip - The IP address of the test worker.
+ * @property {number | undefined} port - The port of the test worker.
+ * @property {() => Promise<void> | undefined} stop - Function to stop the test worker.
+ * @property {() => string | undefined} getOutput - Function to get the output of the test worker.
+ * @property {string} wranglerEnv - The wrangler environment to use for the test worker.
+ */
+
+/**
+ * Worker information object
+ * @type {WorkerInfo}
+ */
+const workerInfo = {
+  ip: undefined,
+  port: undefined,
+  stop: undefined,
+  getOutput: undefined,
+  wranglerEnv: process.env.WRANGLER_ENV || 'integration'
+}
+
+/**
+ * Sets up the test worker.
+ * @returns {Promise<void>}
+ */
+export const mochaGlobalSetup = async () => {
+  try {
+    const result = await runWranglerDev(
+      resolve(__dirname, '../../'), // The directory of the worker with the wrangler.toml
+      ['--local'],
+      process.env,
+      workerInfo.wranglerEnv
+    )
+    Object.assign(workerInfo, result)
+    console.log(`Output: ${await workerInfo.getOutput()}`)
+    console.log('WorkerInfo:', workerInfo)
+    console.log('Test worker started!')
+  } catch (error) {
+    console.error('Failed to start test worker:', error)
+    throw error
+  }
+}
+
+/**
+ * Tears down the test worker.
+ * @returns {Promise<void>}
+ */
+export const mochaGlobalTeardown = async () => {
+  try {
+    const { stop } = workerInfo
+    await stop?.()
+    // console.log('getOutput', getOutput()) // uncomment for debugging
+    console.log('Test worker stopped!')
+  } catch (error) {
+    console.error('Failed to stop test worker:', error)
+    throw error
+  }
+}
+
+/**
+ * Gets the worker info.
+ * @returns {WorkerInfo}
+ */
+export function getWorkerInfo () {
+  return workerInfo
+}

test/helpers/run-wrangler.js

@@ -0,0 +1,131 @@
+import assert from 'node:assert'
+import { fork } from 'node:child_process'
+import path from 'node:path'
+import treeKill from 'tree-kill'
+import { fileURLToPath } from 'node:url'
+
+// Get __dirname equivalent in ES module
+const __filename = fileURLToPath(import.meta.url)
+const __dirname = path.dirname(__filename)
+
+export const wranglerEntryPath = path.resolve(
+  __dirname,
+  '../../node_modules/wrangler/bin/wrangler.js'
+)
+
+/**
+ * @typedef {Object} WranglerProcessInfo
+ * @property {string} ip - The IP address of the test worker.
+ * @property {number} port - The port of the test worker.
+ * @property {() => Promise<void>} stop - Function to stop the test worker.
+ * @property {() => string} getOutput - Function to get the output of the test worker.
+ * @property {() => void} clearOutput - Function to clear the output buffer.
+ */
+
+/**
+ * Runs the command `wrangler dev` in a child process.
+ *
+ * Returns an object that gives you access to:
+ *
+ * - `ip` and `port` of the http-server hosting the pages project
+ * - `stop()` function that will close down the server.
+ *
+ * @param {string} cwd - The current working directory.
+ * @param {string[]} options - The options to pass to the wrangler command.
+ * @param {NodeJS.ProcessEnv} [env] - The environment variables.
+ * @param {string} [wranglerEnv] - The wrangler environment to use.
+ * @returns {Promise<WranglerProcessInfo>}
+ */
+export async function runWranglerDev (
+  cwd,
+  options,
+  env,
+  wranglerEnv
+) {
+  return runLongLivedWrangler(
+    ['dev', `--env=${wranglerEnv}`, ...options],
+    cwd,
+    env
+  )
+}
+
+/**
+ * Runs a long-lived wrangler command in a child process.
+ *
+ * @param {string[]} command - The wrangler command to run.
+ * @param {string} cwd - The current working directory.
+ * @param {NodeJS.ProcessEnv} [env] - The environment variables.
+ * @returns {Promise<WranglerProcessInfo>}
+ */
+async function runLongLivedWrangler (
+  command,
+  cwd,
+  env
+) {
+  let settledReadyPromise = false
+  /** @type {(value: { ip: string port: number }) => void} */
+  let resolveReadyPromise
+  /** @type {(reason: unknown) => void} */
+  let rejectReadyPromise
+
+  const ready = new Promise((resolve, reject) => {
+    resolveReadyPromise = resolve
+    rejectReadyPromise = reject
+  })
+
+  const wranglerProcess = fork(wranglerEntryPath, command, {
+    stdio: ['ignore', /* stdout */ 'pipe', /* stderr */ 'pipe', 'ipc'],
+    cwd,
+    env: { ...process.env, ...env, PWD: cwd }
+  }).on('message', (message) => {
+    if (settledReadyPromise) return
+    settledReadyPromise = true
+    clearTimeout(timeoutHandle)
+    resolveReadyPromise(JSON.parse(message.toString()))
+  })
+
+  const chunks = []
+  wranglerProcess.stdout?.on('data', (chunk) => {
+    chunks.push(chunk)
+  })
+  wranglerProcess.stderr?.on('data', (chunk) => {
+    chunks.push(chunk)
+  })
+  const getOutput = () => Buffer.concat(chunks).toString()
+  const clearOutput = () => (chunks.length = 0)
+
+  const timeoutHandle = setTimeout(() => {
+    if (settledReadyPromise) return
+    settledReadyPromise = true
+    const separator = '='.repeat(80)
+    const message = [
+      'Timed out starting long-lived Wrangler:',
+      separator,
+      getOutput(),
+      separator
+    ].join('\n')
+    rejectReadyPromise(new Error(message))
+  }, 50_000)
+
+  async function stop () {
+    return new Promise((resolve) => {
+      assert(
+        wranglerProcess.pid,
+                `Command "${command.join(' ')}" had no process id`
+      )
+      treeKill(wranglerProcess.pid, (e) => {
+        if (e) {
+          console.error(
+            'Failed to kill command: ' + command.join(' '),
+            wranglerProcess.pid,
+            e
+          )
+        }
+        resolve()
+      })
+    })
+  }
+
+  const { ip, port } = await ready
+  return { ip, port, stop, getOutput, clearOutput }
+}

test/integration/rate-limit.spec.js

@@ -0,0 +1,33 @@
+import { expect } from 'chai'
+import { describe, it } from 'mocha'
+import fetch from 'node-fetch'
+import { getWorkerInfo } from '../fixtures/worker-fixture.js'
+
+describe('Rate Limit Handler', () => {
+  /**
+   * This is a test CID that is known to be stored in the staging environment
+   * See https://bafybeibv7vzycdcnydl5n5lbws6lul2omkm6a6b5wmqt77sicrwnhesy7y.ipfs.w3s.link
+   */
+  const cid = 'bafybeibv7vzycdcnydl5n5lbws6lul2omkm6a6b5wmqt77sicrwnhesy7y'
+
+  it('should enforce rate limits', async () => {
+    const { ip, port } = getWorkerInfo()
+    const maxRequests = 130
+    let successCount = 0
+    let rateLimitCount = 0
+
+    const requests = Array.from({ length: maxRequests }, async () => {
+      const response = await fetch(`http://${ip}:${port}/ipfs/${cid}`)
+      if (response.status === 200) {
+        successCount++
+      } else if (response.status === 429) {
+        rateLimitCount++
+      }
+    })
+
+    await Promise.all(requests)
+
+    expect(successCount).to.be.lessThan(maxRequests)
+    expect(rateLimitCount).to.be.greaterThan(0)
+  }).timeout(30_000)
+})

test/index.spec.js → test/miniflare/freeway.spec.js

@@ -9,9 +9,9 @@ import { equals } from 'multiformats/bytes'
 import { Map as LinkMap } from 'lnmap'
 import { CARReaderStream } from 'carstream'
 import { MultipartByteRangeDecoder, decodePartHeader, getBoundary } from 'multipart-byte-range/decoder'
-import { Builder, toBlobKey } from './helpers/builder.js'
-import { generateBlockLocationClaims, mockClaimsService, generateLocationClaim } from './helpers/content-claims.js'
-import { mockBucketService } from './helpers/bucket.js'
+import { Builder, toBlobKey } from '../helpers/builder.js'
+import { generateBlockLocationClaims, mockClaimsService, generateLocationClaim } from '../helpers/content-claims.js'
+import { mockBucketService } from '../helpers/bucket.js'
 
 /**
  * @param {{ arrayBuffer: () => Promise<ArrayBuffer> }} a
@@ -27,9 +27,9 @@ describe('freeway', () => {
   let miniflare
   /** @type {Builder} */
   let builder
-  /** @type {import('./helpers/content-claims.js').MockClaimsService} */
+  /** @type {import('../helpers/content-claims.js').MockClaimsService} */
   let claimsService
-  /** @type {import('./helpers/bucket.js').MockBucketService} */
+  /** @type {import('../helpers/bucket.js').MockBucketService} */
   let bucketService
 
   before(async () => {

test/unit/middlewares/rate-limiter.spec.js

@@ -19,7 +19,7 @@ describe('withRateLimits', () => {
     }
     env = {
       RATE_LIMITER: rateLimiter,
-      FF_RATE_LIMITER_ENABLED: true,
+      FF_RATE_LIMITER_ENABLED: 'true',
       ACCOUNTING_SERVICE_URL: 'http://example.com',
       AUTH_TOKEN_METADATA: {
         get: sandbox.stub(),

wrangler.toml

@@ -110,4 +110,29 @@ simple = { limit = 5, period = 60 }
 
 [[env.fforbeck.kv_namespaces]]
 binding = "AUTH_TOKEN_METADATA"
-id = "07c22b9612b244e797b780e24f74e2cf"
+id = "f848730e45d94f17bcaf3b6d0915da40"
+
+
+### Integration Tests Configuration
+[env.integration]
+name = "freeway-integration-test"
+workers_dev = true
+account_id = "fffa4b4363a7e5250af8357087263b3a"
+r2_buckets = [
+  { binding = "CARPARK", bucket_name = "carpark-integration-0", preview_bucket_name = "carpark-integration-preview-0" }
+]
+
+[env.integration.vars]
+DEBUG = "true"
+FF_RATE_LIMITER_ENABLED = "true"
+CONTENT_CLAIMS_SERVICE_URL = "https://staging.claims.web3.storage"
+
+[[env.integration.unsafe.bindings]]
+name = "RATE_LIMITER"
+type = "ratelimit"
+namespace_id = "0"
+simple = { limit = 100, period = 60 }
+
+[[env.integration.kv_namespaces]]
+binding = "AUTH_TOKEN_METADATA"
+id = "a355501ee4f242b1affa32c1b335db2b"