fix(redis): modify caching in redis to protect against failed calls t…

…o AO

If we have a cached value for the arns name in redis, and fail to get the updated one from AO, serve that. This should help protect downtime with AO infra between name resolutions.

Also introduces prometheus and metrics for cache hits.

docker-compose.yml

@@ -12,7 +12,7 @@ services:
       - IO_PROCESS_ID=${IO_PROCESS_ID:-}
       - RUN_RESOLVER=${RUN_RESOLVER:-true}
       - EVALUATION_INTERVAL_MS=${EVALUATION_INTERVAL_MS:-}
-      - ARNS_CACHE_TTL_MS=${RESOLVER_CACHE_TTL_MS:-}
+      - ARNS_CACHE_TTL_MS=${ARNS_CACHE_TTL_MS:-}
       - ARNS_CACHE_PATH=${ARNS_CACHE_PATH:-./data/arns}
       - ARNS_CACHE_TYPE=${ARNS_CACHE_TYPE:-redis}
       - REDIS_CACHE_URL=${REDIS_CACHE_URL:-redis://redis:6379}

nodemon.json

@@ -1,5 +1,5 @@
 {
   "watch": ["src", "docs", ".env"],
   "ext": "ts,yaml,json",
-  "exec": "ts-node --esm -r dotenv/config --project ./tsconfig.json ./src/service.ts"
+  "exec": "NODE_OPTIONS=\"--import=./register.js\" ts-node --esm -r dotenv/config --project ./tsconfig.json ./src/service.ts"
 }

package.json

@@ -20,7 +20,7 @@
     "lmdb": "^3.0.0",
     "middleware-async": "^1.3.5",
     "p-limit": "^4.0.0",
-    "prom-client": "^14.0.1",
+    "prom-client": "^15.1.3",
     "redis": "^4.7.0",
     "swagger-ui-express": "^5.0.0",
     "winston": "^3.7.2",

src/cache/arns-store.ts

@@ -17,28 +17,35 @@
  */
 import winston from 'winston';
 
-import { KVBufferStore } from '../types.js';
+import * as metrics from '../metrics.js';
+import { NameResolver } from '../resolver/arns-resolver.js';
+import { ArNSResolvedData, KVBufferStore } from '../types.js';
 
-export class ArNSStore implements KVBufferStore {
+export class ArNSStore implements KVBufferStore, NameResolver {
   private log: winston.Logger;
   private prefix: string;
   private kvStore: KVBufferStore;
+  private resolver: NameResolver;
 
   constructor({
     log,
+    resolver,
     kvStore,
     prefix = 'ArNS',
   }: {
     log: winston.Logger;
+    resolver: NameResolver;
     kvStore: KVBufferStore;
     prefix?: string;
   }) {
     this.log = log.child({ class: this.constructor.name });
+    this.resolver = resolver;
     this.kvStore = kvStore;
     this.prefix = prefix;
     this.log.info('ArNSStore initialized', {
       prefix,
       kvStore: kvStore.constructor.name,
+      resolver: resolver.constructor.name,
     });
   }
 
@@ -47,12 +54,61 @@ export class ArNSStore implements KVBufferStore {
     return `${this.prefix}|${key}`;
   }
 
+  private serialize(buffer: Buffer, ttlSeconds: number): Buffer {
+    const expirationBuffer = Buffer.allocUnsafe(8);
+    expirationBuffer.writeBigInt64BE(BigInt(Date.now() + ttlSeconds * 1000), 0);
+    const ttlBuffer = Buffer.allocUnsafe(8);
+    ttlBuffer.writeBigInt64BE(BigInt(ttlSeconds * 1000), 0);
+    return Buffer.concat([expirationBuffer, ttlBuffer, buffer]);
+  }
+
+  private deserialize(buffer: Buffer): {
+    expired: boolean;
+    buffer: Buffer;
+    ttlSeconds: number;
+  } {
+    const expirationTimestamp = buffer.readBigInt64BE(0); // 8 bytes for a timestamp
+    const ttlMilliseconds = buffer.readBigInt64BE(8); // 8 bytes for a timestamp
+    return {
+      expired: Date.now() >= Number(expirationTimestamp),
+      ttlSeconds: Number(ttlMilliseconds) / 1000,
+      buffer: buffer.slice(16),
+    };
+  }
+
   async get(key: string): Promise<Buffer | undefined> {
-    return this.kvStore.get(this.hashKey(key));
+    const result = await this.getWithExpirationData(key);
+    if (result === undefined || result.expired) {
+      return undefined;
+    }
+    return result.buffer;
   }
 
-  async set(key: string, value: Buffer, ttlSeconds?: number): Promise<void> {
-    return this.kvStore.set(this.hashKey(key), value, ttlSeconds);
+  private async getWithExpirationData(key: string): Promise<
+    | {
+        buffer: Buffer;
+        ttlSeconds: number;
+        expired: boolean;
+      }
+    | undefined
+  > {
+    const result = await this.kvStore.get(this.hashKey(key));
+    if (result === undefined) {
+      metrics.arnsCacheMiss.inc({
+        cache_type: this.kvStore.constructor.name,
+      });
+      return undefined;
+    }
+    metrics.arnsCacheHit.inc({
+      cache_type: this.kvStore.constructor.name,
+    });
+
+    return this.deserialize(result);
+  }
+
+  async set(key: string, value: Buffer, ttlSeconds: number): Promise<void> {
+    const serialized = this.serialize(value, ttlSeconds);
+    return this.kvStore.set(this.hashKey(key), serialized);
   }
 
   async del(key: string): Promise<void> {
@@ -66,4 +122,54 @@ export class ArNSStore implements KVBufferStore {
   async close(): Promise<void> {
     return this.kvStore.close();
   }
+
+  /**
+   * Resolves a name and updates the cache if it's expired.
+   * @param key - The name to resolve.
+   * @returns The resolved name data.
+   */
+  async resolve(key: string): Promise<ArNSResolvedData | undefined> {
+    const cachedWithExpirationData = await this.getWithExpirationData(key);
+    if (cachedWithExpirationData !== undefined) {
+      if (cachedWithExpirationData.expired) {
+        this.log.debug('Cache expired, resolving name', { key });
+        try {
+          const resolved = await this.resolver.resolve(key);
+          if (resolved) {
+            await this.set(
+              key,
+              Buffer.from(JSON.stringify(resolved)),
+              resolved.ttlSeconds,
+            );
+            this.log.debug('Updated cache', {
+              key,
+              ttlSeconds: resolved.ttlSeconds,
+            });
+            return resolved;
+          }
+        } catch (error: any) {
+          this.log.error('Error resolving name. Falling back to cache', {
+            key,
+            message: error.message,
+            stack: error.stack,
+          });
+        }
+      }
+      return JSON.parse(cachedWithExpirationData.buffer.toString());
+    }
+
+    // if not in cache, resolve it
+    const resolved = await this.resolver.resolve(key);
+    if (!resolved) {
+      return undefined;
+    }
+    // update the cache with the resolved data
+    await this.set(
+      key,
+      Buffer.from(JSON.stringify(resolved)),
+      resolved.ttlSeconds,
+    );
+    this.log.debug('Updated cache', { key, ttlSeconds: resolved.ttlSeconds });
+    return resolved;
+  }
 }

src/cache/redis-kv-store.ts

@@ -18,15 +18,25 @@
 import { RedisClientType, commandOptions, createClient } from 'redis';
 import winston from 'winston';
 
+import * as config from '../config.js';
 import { KVBufferStore } from '../types.js';
 
 export class RedisKvStore implements KVBufferStore {
   private client: RedisClientType;
   private log: winston.Logger;
-  private defaultTtlSeconds?: number;
+  private defaultTtlSeconds: number;
 
-  constructor({ log, redisUrl }: { log: winston.Logger; redisUrl: string }) {
+  constructor({
+    log,
+    redisUrl,
+    defaultTtlSeconds = config.ARNS_CACHE_TTL_MS,
+  }: {
+    log: winston.Logger;
+    redisUrl: string;
+    defaultTtlSeconds?: number;
+  }) {
     this.log = log.child({ class: this.constructor.name });
+    this.defaultTtlSeconds = defaultTtlSeconds;
     this.client = createClient({
       url: redisUrl,
     });
@@ -55,7 +65,12 @@ export class RedisKvStore implements KVBufferStore {
       commandOptions({ returnBuffers: true }),
       key,
     );
-    return value ?? undefined;
+
+    if (!value) {
+      return undefined;
+    }
+
+    return value;
   }
 
   async has(key: string): Promise<boolean> {
@@ -68,13 +83,9 @@ export class RedisKvStore implements KVBufferStore {
     }
   }
 
-  async set(key: string, buffer: Buffer, ttlSeconds?: number): Promise<void> {
-    if (ttlSeconds !== undefined) {
-      await this.client.set(key, buffer, {
-        EX: ttlSeconds ?? this.defaultTtlSeconds,
-      });
-    } else {
-      await this.client.set(key, buffer);
-    }
+  async set(key: string, buffer: Buffer): Promise<void> {
+    await this.client.set(key, buffer, {
+      EX: this.defaultTtlSeconds,
+    });
   }
 }

src/metrics.ts

@@ -0,0 +1,32 @@
+/**
+ * AR.IO ArNS Resolver
+ * Copyright (C) 2023 Permanent Data Solutions, Inc. All Rights Reserved.
+ *
+ * This program is free software: you can redistribute it and/or modify
+ * it under the terms of the GNU Affero General Public License as published by
+ * the Free Software Foundation, either version 3 of the License, or
+ * (at your option) any later version.
+ *
+ * This program is distributed in the hope that it will be useful,
+ * but WITHOUT ANY WARRANTY; without even the implied warranty of
+ * MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE.  See the
+ * GNU Affero General Public License for more details.
+ *
+ * You should have received a copy of the GNU Affero General Public License
+ * along with this program.  If not, see <http://www.gnu.org/licenses/>.
+ */
+import * as promClient from 'prom-client';
+
+export const metrics = new promClient.Registry();
+
+export const arnsCacheHit = new promClient.Counter({
+  name: 'arns_cache_hit',
+  help: 'Number of times the ARNS cache was hit',
+  labelNames: ['cache_type'],
+});
+
+export const arnsCacheMiss = new promClient.Counter({
+  name: 'arns_cache_miss',
+  help: 'Number of times the ARNS cache was missed',
+  labelNames: ['cache_type'],
+});

src/resolver/arns-resolver.ts

@@ -0,0 +1,94 @@
+/**
+ * AR.IO ArNS Resolver
+ * Copyright (C) 2023 Permanent Data Solutions, Inc. All Rights Reserved.
+ *
+ * This program is free software: you can redistribute it and/or modify
+ * it under the terms of the GNU Affero General Public License as published by
+ * the Free Software Foundation, either version 3 of the License, or
+ * (at your option) any later version.
+ *
+ * This program is distributed in the hope that it will be useful,
+ * but WITHOUT ANY WARRANTY; without even the implied warranty of
+ * MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE.  See the
+ * GNU Affero General Public License for more details.
+ *
+ * You should have received a copy of the GNU Affero General Public License
+ * along with this program.  If not, see <http://www.gnu.org/licenses/>.
+ */
+import { ANT, AOProcess, AoClient, AoIORead } from '@ar.io/sdk';
+import { connect } from '@permaweb/aoconnect';
+import winston from 'winston';
+
+import * as config from '../config.js';
+import { ArNSResolvedData } from '../types.js';
+
+export interface NameResolver {
+  resolve(name: string): Promise<ArNSResolvedData | undefined>;
+}
+
+export class ArNSResolver implements NameResolver {
+  private ao: AoClient;
+  private io: AoIORead;
+  private logger: winston.Logger;
+
+  constructor({
+    io,
+    ao = connect({
+      MU_URL: config.AO_MU_URL,
+      CU_URL: config.AO_CU_URL,
+      GRAPHQL_URL: config.AO_GRAPHQL_URL,
+      GATEWAY_URL: config.AO_GATEWAY_URL,
+    }),
+    log,
+  }: {
+    io: AoIORead;
+    ao: AoClient;
+    log: winston.Logger;
+  }) {
+    this.ao = ao;
+    this.io = io;
+    this.logger = log.child({ class: this.constructor.name });
+  }
+
+  async resolve(name: string): Promise<ArNSResolvedData | undefined> {
+    this.logger.debug('Resolving name', { name });
+    const apexName = name.split('_').slice(-1)[0];
+    const record = await this.io.getArNSRecord({ name: apexName });
+
+    if (!record) {
+      this.logger.debug('No record found', { name, apexName });
+      return undefined;
+    }
+
+    // get the ant id and use that to get the record from the cache
+    const antId = record.processId;
+    const ant = ANT.init({
+      process: new AOProcess({
+        processId: antId,
+        ao: this.ao,
+      }),
+    });
+    const undername = name.split('_').slice(0, -1).join('_') || '@';
+    const antRecord = await ant.getRecord({ undername });
+    if (!antRecord) {
+      return undefined;
+    }
+
+    const owner = await ant.getOwner();
+
+    this.logger.debug('Resolved name', {
+      name,
+      antId,
+      antRecord,
+      owner,
+    });
+
+    return {
+      ttlSeconds: antRecord.ttlSeconds,
+      txId: antRecord.transactionId,
+      processId: antId,
+      type: record.type,
+      owner,
+    };
+  }
+}