Postgres (#28)

* basic port to postgres

* cap table size

* update ci

* fix env

* fix log

* stream logs

* add data

* fix lint

* update migration

Author: juliangruber

.github/workflows/ci.yml

@@ -6,14 +6,28 @@ on:
 jobs:
   build:
     runs-on: ubuntu-latest
+    services:
+      postgres:
+        image: postgres:latest
+        env:
+          POSTGRES_DB: spark_rewards
+          POSTGRES_PASSWORD: postgres
+          POSTGRES_USER: postgres
+        ports:
+          - 5432:5432
+        options: >-
+          --health-cmd pg_isready
+          --health-interval 10s
+          --health-timeout 5s
+          --health-retries 5
     env:
       SENTRY_ENVIRONMENT: CI
+      DATABASE_URL: postgres://postgres:postgres@localhost:5432/spark_rewards
     steps:
       - uses: actions/checkout@v4
       - uses: actions/setup-node@v4
         with:
           node-version: 20
-      - uses: supercharge/[email protected]
       - run: npm ci
       - run: npm test
 

bin/create-redis-postgres-migration.js

@@ -0,0 +1,27 @@
+import Redis from 'ioredis'
+
+const {
+  REDIS_URL: redisUrl = 'redis://localhost:6379'
+} = process.env
+
+const redisUrlParsed = new URL(redisUrl)
+const redis = new Redis({
+  host: redisUrlParsed.hostname,
+  port: redisUrlParsed.port,
+  username: redisUrlParsed.username,
+  password: redisUrlParsed.password,
+  family: 6 // required for upstash
+})
+
+const rewards = await redis.hgetall('rewards')
+
+console.log('INSERT INTO scheduled_rewards (address, amount)')
+console.log('VALUES')
+console.log(
+  Object.entries(rewards)
+    .map(([address, amount]) => `('${address}', ${amount})`)
+    .join(',\n')
+)
+console.log(';')
+
+process.exit()

bin/spark-rewards.js

@@ -2,8 +2,8 @@ import '../lib/instrument.js'
 import http from 'node:http'
 import { once } from 'node:events'
 import { createHandler } from '../index.js'
-import Redis from 'ioredis'
-import Redlock from 'redlock'
+import pg from 'pg'
+import { migrate } from '../migrations/index.js'
 
 const {
   PORT: port = 3000,
@@ -15,7 +15,7 @@ const {
     '0x646ac6F1941CAb0ce3fE1368e9AD30364a9F51dA', // Miroslav
     '0x3ee4A552b1a6519A266AEFb0514633F289FF2A9F' // Julian
   ].join(','),
-  REDIS_URL: redisUrl = 'redis://localhost:6379'
+  DATABASE_URL
 } = process.env
 
 const logger = {
@@ -24,17 +24,28 @@ const logger = {
   request: ['1', 'true'].includes(requestLogging) ? console.info : () => {}
 }
 
-const redisUrlParsed = new URL(redisUrl)
-const redis = new Redis({
-  host: redisUrlParsed.hostname,
-  port: redisUrlParsed.port,
-  username: redisUrlParsed.username,
-  password: redisUrlParsed.password,
-  family: 6 // required for upstash
+const pgPool = new pg.Pool({
+  connectionString: DATABASE_URL,
+  // allow the pool to close all connections and become empty
+  min: 0,
+  // this values should correlate with service concurrency hard_limit configured in fly.toml
+  // and must take into account the connection limit of our PG server, see
+  // https://fly.io/docs/postgres/managing/configuration-tuning/
+  max: 100,
+  // close connections that haven't been used for one second
+  idleTimeoutMillis: 1000,
+  // automatically close connections older than 60 seconds
+  maxLifetimeSeconds: 60
 })
-const redlock = new Redlock([redis])
+pgPool.on('error', err => {
+  // Prevent crashing the process on idle client errors, the pool will recover
+  // itself. If all connections are lost, the process will still crash.
+  // https://github.com/brianc/node-postgres/issues/1324#issuecomment-308778405
+  console.error('An idle client has experienced an error', err.stack)
+})
+await migrate(pgPool)
 
-const handler = await createHandler({ logger, redis, redlock, signerAddresses })
+const handler = await createHandler({ logger, pgPool, signerAddresses })
 const server = http.createServer(handler)
 server.listen(port, host)
 await once(server, 'listening')