[L2B-4977] Refactor logger to allow for custom backends and formatters

.github/workflows/ci.yml

@@ -14,7 +14,7 @@ jobs:
       - uses: actions/checkout@v3
       - uses: actions/setup-node@v3
         with:
-          node-version: 16.x
+          node-version: 18.x
           cache: 'yarn'
       - run: yarn --frozen-lockfile
       - run: yarn build
@@ -25,7 +25,7 @@ jobs:
       - uses: actions/checkout@v3
       - uses: actions/setup-node@v3
         with:
-          node-version: 16.x
+          node-version: 18.x
           cache: 'yarn'
       - run: yarn --frozen-lockfile
       - run: yarn build
@@ -36,7 +36,7 @@ jobs:
       - uses: actions/checkout@v3
       - uses: actions/setup-node@v3
         with:
-          node-version: 16.x
+          node-version: 18.x
           cache: 'yarn'
       - run: yarn --frozen-lockfile
       - run: yarn build
@@ -48,7 +48,7 @@ jobs:
       - uses: actions/checkout@v3
       - uses: actions/setup-node@v3
         with:
-          node-version: 16.x
+          node-version: 18.x
           cache: 'yarn'
       - run: yarn --frozen-lockfile
       - run: yarn build

.github/workflows/release-canary.yml

@@ -17,7 +17,7 @@ jobs:
       - uses: actions/checkout@v3
       - uses: actions/setup-node@v3
         with:
-          node-version: 16.x
+          node-version: 18.x
           cache: 'yarn'
 
       - run: yarn --frozen-lockfile

.github/workflows/release.yml

@@ -20,7 +20,7 @@ jobs:
       - uses: actions/checkout@v3
       - uses: actions/setup-node@v3
         with:
-          node-version: 16.x
+          node-version: 18.x
           cache: 'yarn'
 
       - run: yarn --frozen-lockfile

packages/backend-tools/package.json

@@ -30,12 +30,16 @@
     "typecheck": "tsc --noEmit"
   },
   "dependencies": {
+    "@elastic/elasticsearch": "^8.13.1",
     "chalk": "^4.1.2",
     "dotenv": "^16.3.1",
-    "error-stack-parser": "^2.1.4"
+    "error-stack-parser": "^2.1.4",
+    "uuid": "^9.0.1"
   },
   "devDependencies": {
     "@sinonjs/fake-timers": "^11.1.0",
-    "@types/sinonjs__fake-timers": "^8.1.2"
+    "@types/elasticsearch": "^5.0.43",
+    "@types/sinonjs__fake-timers": "^8.1.2",
+    "@types/uuid": "^9.0.8"
   }
 }

packages/backend-tools/src/index.ts

@@ -1,4 +1,9 @@
 export * from './env'
+export * from './logger/ElasticSearchBackend'
+export * from './logger/interfaces'
+export * from './logger/LogFormatterEcs'
+export * from './logger/LogFormatterJson'
+export * from './logger/LogFormatterPretty'
 export * from './logger/Logger'
 export * from './rate-limit/RateLimiter'
 export * from './utils/assert'

packages/backend-tools/src/logger/ElasticSearchBackend.ts

@@ -0,0 +1,100 @@
+import { Client } from '@elastic/elasticsearch'
+import { v4 as uuidv4 } from 'uuid'
+
+import { LoggerBackend } from './interfaces'
+
+export interface ElasticSearchBackendOptions {
+  node: string
+  apiKey: string
+  flushInterval?: number
+  indexPrefix?: string
+}
+
+export class ElasticSearchBackend implements LoggerBackend {
+  private readonly options: Required<ElasticSearchBackendOptions>
+  private readonly buffer: string[]
+  private readonly client: Client
+
+  constructor(options: ElasticSearchBackendOptions) {
+    this.options = {
+      ...options,
+      flushInterval: options.flushInterval ?? 10000,
+      indexPrefix: options.indexPrefix ?? 'logs',
+    }
+
+    this.client = new Client({
+      node: options.node,
+      auth: {
+        apiKey: options.apiKey,
+      },
+    })
+
+    this.buffer = []
+    this.start()
+  }
+
+  public debug(message: string): void {
+    this.buffer.push(message)
+  }
+
+  public log(message: string): void {
+    this.buffer.push(message)
+  }
+
+  public warn(message: string): void {
+    this.buffer.push(message)
+  }
+
+  public error(message: string): void {
+    this.buffer.push(message)
+  }
+
+  private start(): void {
+    // eslint-disable-next-line @typescript-eslint/no-misused-promises
+    setInterval(async () => {
+      await this.flushLogs()
+    }, this.options.flushInterval)
+  }
+
+  private async flushLogs(): Promise<void> {
+    try {
+      const index = await this.createIndex()
+
+      // eslint-disable-next-line @typescript-eslint/no-unsafe-return
+      const documents = this.buffer.map((message) => ({
+        id: uuidv4(),
+        ...JSON.parse(message),
+      }))
+
+      // eslint-disable-next-line @typescript-eslint/no-unsafe-return
+      const operations = documents.flatMap((doc) => [
+        { index: { _index: index } },
+        doc,
+      ])
+
+      // eslint-disable-next-line @typescript-eslint/no-unsafe-return
+      const bulkResponse = await this.client.bulk({ refresh: true, operations })
+
+      if (bulkResponse.errors) {
+        throw new Error('Failed to push liogs to Elastic Search node')
+      }
+    } catch (error) {
+      console.log(error)
+    }
+  }
+
+  private async createIndex(): Promise<string> {
+    const now = new Date()
+    const indexName = `${
+      this.options.indexPrefix
+    }-${now.getFullYear()}.${now.getMonth()}.${now.getDay()}`
+
+    const exist = await this.client.indices.exists({ index: indexName })
+    if (!exist) {
+      await this.client.indices.create({
+        index: indexName,
+      })
+    }
+    return indexName
+  }
+}

packages/backend-tools/src/logger/LogFormatterEcs.ts

@@ -0,0 +1,31 @@
+import { LogEntry, LogFormatter } from './interfaces'
+import { toJSON } from './toJSON'
+
+// https://www.elastic.co/guide/en/ecs/8.11/ecs-reference.html
+export class LogFormatterEcs implements LogFormatter {
+  public format(entry: LogEntry): string {
+    const core = {
+      '@timestamp': entry.time.toISOString(),
+      log: {
+        level: entry.level,
+      },
+      service: {
+        name: entry.service,
+      },
+      message: entry.message,
+      error: entry.resolvedError
+        ? {
+            message: entry.resolvedError.error,
+            type: entry.resolvedError.name,
+            stack_trace: entry.resolvedError.stack,
+          }
+        : undefined,
+    }
+
+    try {
+      return toJSON({ ...core, parameters: entry.parameters })
+    } catch {
+      return toJSON({ ...core })
+    }
+  }
+}

packages/backend-tools/src/logger/LogFormatterJson.ts

@@ -0,0 +1,20 @@
+import { LogEntry, LogFormatter } from './interfaces'
+import { toJSON } from './toJSON'
+
+export class LogFormatterJson implements LogFormatter {
+  public format(entry: LogEntry): string {
+    const core = {
+      time: entry.time.toISOString(),
+      level: entry.level,
+      service: entry.service,
+      message: entry.message,
+      error: entry.resolvedError,
+    }
+
+    try {
+      return toJSON({ ...core, parameters: entry.parameters })
+    } catch {
+      return toJSON({ ...core })
+    }
+  }
+}

packages/backend-tools/src/logger/LogFormatterPretty.ts

@@ -0,0 +1,133 @@
+import chalk from 'chalk'
+import { inspect } from 'util'
+
+import { LogEntry, LogFormatter } from './interfaces'
+import { LogLevel } from './LogLevel'
+import { toJSON } from './toJSON'
+
+const STYLES = {
+  bigint: 'white',
+  boolean: 'white',
+  date: 'white',
+  module: 'white',
+  name: 'blue',
+  null: 'white',
+  number: 'white',
+  regexp: 'white',
+  special: 'white',
+  string: 'white',
+  symbol: 'white',
+  undefined: 'white',
+}
+
+const INDENT_SIZE = 4
+const INDENT = ' '.repeat(INDENT_SIZE)
+
+export class LogFormatterPretty implements LogFormatter {
+  constructor(
+    private readonly colors: boolean,
+    private readonly utc: boolean,
+  ) {}
+
+  public format(entry: LogEntry): string {
+    const timeOut = this.formatTimePretty(entry.time, this.utc, this.colors)
+    const levelOut = this.formatLevelPretty(entry.level, this.colors)
+    const serviceOut = this.formatServicePretty(entry.service, this.colors)
+    const messageOut = entry.message ? ` ${entry.message}` : ''
+    const paramsOut = this.formatParametersPretty(
+      this.sanitize(
+        entry.resolvedError
+          ? { ...entry.resolvedError, ...entry.parameters }
+          : entry.parameters ?? {},
+      ),
+      this.colors,
+    )
+
+    return `${timeOut} ${levelOut}${serviceOut}${messageOut}${paramsOut}`
+  }
+
+  private formatLevelPretty(level: LogLevel, colors: boolean): string {
+    if (colors) {
+      switch (level) {
+        case 'CRITICAL':
+        case 'ERROR':
+          return chalk.red(chalk.bold(level.toUpperCase()))
+        case 'WARN':
+          return chalk.yellow(chalk.bold(level.toUpperCase()))
+        case 'INFO':
+          return chalk.green(chalk.bold(level.toUpperCase()))
+        case 'DEBUG':
+          return chalk.magenta(chalk.bold(level.toUpperCase()))
+        case 'TRACE':
+          return chalk.gray(chalk.bold(level.toUpperCase()))
+      }
+    }
+    return level.toUpperCase()
+  }
+
+  private formatTimePretty(now: Date, utc: boolean, colors: boolean): string {
+    const h = (utc ? now.getUTCHours() : now.getHours())
+      .toString()
+      .padStart(2, '0')
+    const m = (utc ? now.getUTCMinutes() : now.getMinutes())
+      .toString()
+      .padStart(2, '0')
+    const s = (utc ? now.getUTCSeconds() : now.getSeconds())
+      .toString()
+      .padStart(2, '0')
+    const ms = (utc ? now.getUTCMilliseconds() : now.getMilliseconds())
+      .toString()
+      .padStart(3, '0')
+
+    let result = `${h}:${m}:${s}.${ms}`
+    if (utc) {
+      result += 'Z'
+    }
+
+    return colors ? chalk.gray(result) : result
+  }
+
+  private formatParametersPretty(parameters: object, colors: boolean): string {
+    const oldStyles = inspect.styles
+    inspect.styles = STYLES
+
+    const inspected = inspect(parameters, {
+      colors,
+      breakLength: 80 - INDENT_SIZE,
+      depth: 5,
+    })
+
+    inspect.styles = oldStyles
+
+    if (inspected === '{}') {
+      return ''
+    }
+
+    const indented = inspected
+      .split('\n')
+      .map((x) => INDENT + x)
+      .join('\n')
+
+    if (colors) {
+      return '\n' + chalk.gray(indented)
+    }
+    return '\n' + indented
+  }
+
+  private formatServicePretty(
+    service: string | undefined,
+    colors: boolean,
+  ): string {
+    if (!service) {
+      return ''
+    }
+    return colors
+      ? ` ${chalk.gray('[')} ${chalk.yellow(service)} ${chalk.gray(']')}`
+      : ` [ ${service} ]`
+  }
+
+  private sanitize(parameters: object): object {
+    // eslint-disable-next-line @typescript-eslint/no-unsafe-return
+    return JSON.parse(toJSON(parameters))
+  }
+}