Refactor To Fastify (#413)

Author: beeequeue

.eslintignore

@@ -0,0 +1 @@
+*.snap

.eslintrc.js

@@ -11,6 +11,8 @@ module.exports = {
     "plugin:@beequeue/prettier",
   ],
   rules: {
+    "no-console": "off",
     "import/no-named-as-default": "off",
+    "@typescript-eslint/require-await": "off",
   },
 }

README.md

@@ -6,7 +6,11 @@
 This app uses data from [`anime-offline-database`](https://github.com/manami-project/anime-offline-database/) - fetching
 and updating itself every 24 hours.
 
-[![Deploy to DigitalOcean](https://mp-assets1.sfo2.digitaloceanspaces.com/deploy-to-do/do-btn-blue.svg)](https://cloud.digitalocean.com/apps/new?repo=https://github.com/BeeeQueue/arm-server/tree/master&refcode=52b251df60e7)
+#### Get notifications on important API changes
+
+Subscribe to new releases in this repo:
+
+![image](https://user-images.githubusercontent.com/472500/121041611-c116fc00-c767-11eb-9aaa-64a894a1598a.png)
 
 ### Missing or duplicate entries
 
@@ -31,12 +35,12 @@ enum Source {
 
 ### Get IDS:
 
-`GET/POST` `/api/ids`
+`POST` `/api/ids`
 
-Either use GET query parameters:
+Either use GET with query parameters:
 `?source={Source}&id={number}`
 
-or send the query as a POST JSON body:
+or use POST with a JSON body:
 
 `{ "anilist": 1337 }`
 
@@ -52,7 +56,9 @@ interface Entry {
   kitsu: number | null
 }
 
+// If JSON body is a single object
 // { "anilist": 1337 } => Entry | null
+// // If JSON body is an array of objects
 // [{ ... }] => Array<Entry | null>
 ```
 

package.json

@@ -4,8 +4,9 @@
   "version": "1.0.0",
   "private": true,
   "license": "AGPL-3.0-only",
+  "homepage": "https://github.com/BeeeQueue/arm-server",
   "engines": {
-    "node": ">=14.17"
+    "node": "^14.17"
   },
   "husky": {
     "hooks": {
@@ -34,37 +35,30 @@
     "@sentry/tracing": "6.3.6",
     "cross-env": "7.0.3",
     "envsafe": "2.0.3",
+    "fastify": "3.17.0",
+    "fastify-cors": "6.0.1",
+    "fastify-helmet": "5.3.1",
     "got": "11.8.2",
-    "joi": "17.4.0",
     "knex": "0.95.6",
-    "koa": "2.13.1",
-    "koa-bodyparser": "4.3.0",
-    "koa-connect": "2.1.0",
-    "koa-pino-logger": "3.0.0",
-    "koa-router": "10.0.0",
-    "pino": "6.11.3",
-    "pino-pretty": "4.8.0",
+    "nanoid": "3.1.23",
     "sqlite3": "5.0.2",
+    "ts-json-validator": "0.7.1",
     "ts-node": "9.1.1",
-    "typescript": "4.2.4"
+    "type-fest": "1.2.0",
+    "typescript": "4.3.2"
   },
   "devDependencies": {
     "@beequeue/eslint-plugin": "0.2.0",
     "@tsconfig/node14": "1.0.0",
     "@types/jest": "26.0.23",
-    "@types/koa": "2.13.1",
-    "@types/koa-bodyparser": "4.3.0",
-    "@types/koa-pino-logger": "3.0.0",
-    "@types/koa-router": "7.4.2",
     "@types/node": "14.17.2",
-    "@types/supertest": "2.0.11",
-    "dotenv": "9.0.2",
+    "dotenv": "10.0.0",
     "eslint": "7.27.0",
     "husky": "4.3.8",
     "jest": "26.6.3",
     "lint-staged": "11.0.0",
+    "pino-pretty": "5.0.2",
     "prettier": "2.3.0",
-    "supertest": "6.1.3",
     "ts-jest": "26.5.6",
     "ts-node-dev": "1.1.6",
     "tsconfig-paths": "3.9.0"

src/app.ts

@@ -1,75 +1,50 @@
-import Koa, { Context } from "koa"
-import BodyParser from "koa-bodyparser"
-import RequestLogger from "koa-pino-logger"
-import Router from "koa-router"
+import Fastify from "fastify"
+import Cors from "fastify-cors"
+import Helmet from "fastify-helmet"
+import { customAlphabet, urlAlphabet } from "nanoid"
 
 import { config } from "@/config"
-import { Logger } from "@/lib/logger"
-import { requestHandler, sendErrorToSentry, tracingMiddleWare } from "@/lib/sentry"
+import { sendErrorToSentry } from "@/lib/sentry"
+import { apiPlugin } from "@/routes/ids"
 
-import { routes } from "./routes"
+import pkgJson from "../package.json"
 
-export const App = new Koa()
-const router = new Router()
+const isProd = config.NODE_ENV === "production"
 
-App.use(
-  RequestLogger({
-    prettyPrint: config.NODE_ENV === "development",
-  }),
-)
+const nanoid = customAlphabet(urlAlphabet, 16)
 
-App.use(requestHandler)
-App.use(tracingMiddleWare)
+export const buildApp = async () => {
+  const App = Fastify({
+    ignoreTrailingSlash: true,
+    onProtoPoisoning: "remove",
+    onConstructorPoisoning: "remove",
+    trustProxy: isProd,
+    genReqId: nanoid,
+    disableRequestLogging: process.env.NODE_ENV === "test",
+    logger: {
+      level: config.LOG_LEVEL,
+      prettyPrint: !isProd,
+    },
+  })
 
-App.on("error", (err, ctx) => {
-  Logger.error(err)
+  await App.register(Cors, {
+    origin: true,
+  })
 
-  sendErrorToSentry(err, ctx)
-})
+  await App.register(Helmet, {
+    hsts: false,
+    contentSecurityPolicy: false,
+  })
 
-App.use(BodyParser())
+  App.addHook("onError", (request, _reply, error, next) => {
+    sendErrorToSentry(error, request as any)
 
-router.get("/", (ctx: Context) => {
-  ctx.body = `
-<pre>
-<b>Get IDs:</b>
-<b>GET/POST /api/ids</b>
+    next()
+  })
 
-enum Source {
-  anilist,
-  anidb,
-  myanimelist,
-  kitsu,
-}
-
-<b>Either use GET query parameters:</b>
-?source={Source}&id={number}
-
-<b>or send the query as a POST JSON body:</b>
+  await App.register(apiPlugin, { prefix: "/api" })
 
-{ "anilist": 1337 }
+  App.get("/", async (_request, reply) => reply.redirect(301, pkgJson.homepage))
 
-[{ "anilist": 1337 }, { "anilist": 69 }, { "anidb": 420 }]
-
-interface Entry {
-  anilist: number | null
-  anidb: number | null
-  myanimelist: number | null
-  kitsu: number | null
+  return App
 }
-
-{ "anilist": 1337 } => Entry | null
-[{ ... }] => Array<Entry | null>
-
-<b>The response code will always be 200 (OK).
-If an entry is not found null is returned instead.</b>
-
-Source code is available on GitHub at <a href="https://github.com/BeeeQueue/arm-server">https://github.com/BeeeQueue/arm-server</a>
-</pre>
-`
-})
-
-router.use(routes)
-
-App.use(router.routes())
-App.use(router.allowedMethods())

src/index.ts

@@ -1,26 +1,25 @@
 import { captureException } from "@sentry/node"
 
-import { Logger } from "@/lib/logger"
+import { config } from "@/config"
 
-import { App } from "./app"
+import { buildApp } from "./app"
 import { updateRelations } from "./update"
 
-const { NODE_ENV, PORT } = process.env
-const port = PORT ?? 3000
+const { NODE_ENV, PORT } = config
 
 const runUpdateScript = () => updateRelations().catch(captureException)
 
-const listen = () => {
+const listen = async () => {
   if (NODE_ENV === "production") {
     void runUpdateScript()
 
     // eslint-disable-next-line @typescript-eslint/no-misused-promises
     setInterval(runUpdateScript, 1000 * 60 * 60 * 24)
   }
 
-  App.listen(port, () => {
-    Logger.info(`Listening on ${port}`)
-  })
+  await (
+    await buildApp()
+  ).listen(PORT, process.env.NODE_ENV === "production" ? "0.0.0.0" : undefined)
 }
 
-listen()
+void listen()

src/lib/logger.ts

@@ -1,82 +1,22 @@
-// eslint-disable-next-line node/no-deprecated-api
-import domain from "domain"
-
-import { Context, Next } from "koa"
+import { FastifyRequest } from "fastify"
 
 import * as Sentry from "@sentry/node"
-import { extractTraceparentData, stripUrlQueryAndFragment } from "@sentry/tracing"
 
 const { NODE_ENV, TRACES_SAMPLERATE, SENTRY_DSN } = process.env
 
 Sentry.init({
   dsn: SENTRY_DSN,
   enabled: NODE_ENV === "production",
-  tracesSampleRate: Number(TRACES_SAMPLERATE ?? 1),
+  tracesSampleRate: Number(TRACES_SAMPLERATE ?? 0.25),
 })
 
-export const requestHandler = (ctx: Context, next: Next) =>
-  new Promise<void>((resolve) => {
-    const local = domain.create()
-
-    local.add(ctx as any) // TODO
-
-    local.on("error", (err) => {
-      ctx.status = err.status || 500
-      ctx.body = err.message
-      ctx.app.emit("error", err, ctx)
-    })
-
-    void local.run(async () => {
-      Sentry.getCurrentHub().configureScope((scope) =>
-        scope.addEventProcessor((event) =>
-          Sentry.Handlers.parseRequest(event, ctx.request as any, {
-            user: false,
-          }),
-        ),
-      )
-
-      await next()
-
-      resolve()
-    })
-  })
-
-export const tracingMiddleWare = async (ctx: Context, next: Next) => {
-  const reqMethod = (ctx.method || "").toUpperCase()
-  const reqUrl = ctx.url && stripUrlQueryAndFragment(ctx.url)
-
-  // connect to trace of upstream app
-  let traceparentData = null
-
-  if (ctx.request.get("sentry-trace")) {
-    traceparentData = extractTraceparentData(ctx.request.get("sentry-trace"))
-  }
-
-  const transaction = Sentry.startTransaction({
-    name: `${reqMethod} ${reqUrl}`,
-    op: "http.server",
-    ...traceparentData,
-  })
-
-  ctx.__sentry_transaction = transaction
-
-  await next()
-
-  // if using koa router, a nicer way to capture transaction using the matched route
-  if (ctx._matchedRoute) {
-    const mountPath = (ctx.mountPath as string) || ""
-    transaction.setName(`${reqMethod} ${mountPath}${ctx._matchedRoute as string}`)
-  }
-
-  transaction.setHttpStatus(ctx.status)
-  transaction.finish()
-}
-
-export const sendErrorToSentry = (err: Error, ctx: Context) => {
+export const sendErrorToSentry = (
+  err: Error,
+  // eslint-disable-next-line @typescript-eslint/naming-convention
+  request: FastifyRequest<{ Querystring: Record<string, unknown> }>,
+) => {
   Sentry.withScope((scope) => {
-    scope.addEventProcessor((event) =>
-      Sentry.Handlers.parseRequest(event, ctx.request as any),
-    )
+    scope.addEventProcessor((event) => Sentry.Handlers.parseRequest(event, request))
 
     Sentry.captureException(err)
   })

src/lib/sentry.ts

@@ -1,5 +1,4 @@
 import { knex, Relation } from "./db"
-import { Logger } from "./lib/logger"
 
 const rules = {
   // Kaguya-sama
@@ -37,7 +36,7 @@ export const updateBasedOnManualRules = async () => {
             knex("relations").update(fromWhere).where(toWhere).transacting(trx),
           ),
       )
-      .catch(Logger.error)
+      .catch(console.error)
   })
 
   await Promise.all(promises)

src/manual-rules.ts

@@ -0,0 +1,25 @@
+// Jest Snapshot v1, https://goo.gl/fbAQLP
+
+exports[`json body array input requires at least one source 1`] = `
+Object {
+  "error": "Bad Request",
+  "message": "body should be object, body[0] should NOT have fewer than 1 properties, body should match exactly one schema in oneOf",
+  "statusCode": 400,
+}
+`;
+
+exports[`json body object input GET fails with json body 1`] = `
+Object {
+  "error": "Bad Request",
+  "message": "querystring should have required property 'source'",
+  "statusCode": 400,
+}
+`;
+
+exports[`json body object input errors correctly on an empty object 1`] = `
+Object {
+  "error": "Bad Request",
+  "message": "body should NOT have fewer than 1 properties, body should be array, body should match exactly one schema in oneOf",
+  "statusCode": 400,
+}
+`;