diff --git a/jest.config.ts b/jest.config.ts index c4089d1a..192281d2 100644 --- a/jest.config.ts +++ b/jest.config.ts @@ -15,5 +15,6 @@ export default { testEnvironment: 'node', setupFiles: ['dotenv/config'], testPathIgnorePatterns: ['/node_modules/', '/dist/', '/test/fixtures/'], - moduleFileExtensions: ['js', 'ts'] + moduleFileExtensions: ['js', 'ts'], + verbose: true }; diff --git a/src/api.ts b/src/api.ts index c30fff20..8ecd7b01 100644 --- a/src/api.ts +++ b/src/api.ts @@ -1,7 +1,9 @@ import express from 'express'; +import { Readable } from 'stream'; import { capture } from '@snapshot-labs/snapshot-sentry'; -import { parseQuery, resize, setHeader, getCacheKey } from './utils'; -import { set, get, streamToBuffer, clear } from './aws'; +import { parseQuery, resize, setHeader } from './utils'; +import { streamToBuffer } from './aws'; +import Cache from './resolvers/cache'; import resolvers from './resolvers'; import constants from './constants.json'; import { rpcError, rpcSuccess } from './helpers/utils'; @@ -33,13 +35,14 @@ router.post('/', async (req, res) => { router.get(`/clear/:type(${TYPE_CONSTRAINTS})/:id`, async (req, res) => { const { type, id } = req.params; try { - const { address, network, w, h, fallback, cb } = await parseQuery(id, type, { - s: constants.max, - fb: req.query.fb, - cb: req.query.cb - }); - const key = getCacheKey({ type, network, address, w, h, fallback, cb }); - const result = await clear(key); + const cache = new Cache( + await parseQuery(id, type, { + s: constants.max, + fb: req.query.fb, + cb: req.query.cb + }) + ); + const result = await cache.clear(); res.status(result ? 200 : 404).json({ status: result ? 'ok' : 'not found' }); } catch (e) { capture(e); @@ -49,42 +52,31 @@ router.get(`/clear/:type(${TYPE_CONSTRAINTS})/:id`, async (req, res) => { router.get(`/:type(${TYPE_CONSTRAINTS})/:id`, async (req, res) => { const { type, id } = req.params; - let address, network, w, h, fallback, cb; + let parsedParams, address, network, w, h, fallback; try { - ({ address, network, w, h, fallback, cb } = await parseQuery(id, type, req.query)); + parsedParams = await parseQuery(id, type, req.query); + ({ address, network, w, h, fallback } = parsedParams); } catch (e) { return res.status(500).json({ status: 'error', error: 'failed to load content' }); } - const key1 = getCacheKey({ - type, - network, - address, - w: constants.max, - h: constants.max, - fallback, - cb - }); - const key2 = getCacheKey({ type, network, address, w, h, fallback, cb }); + const cache = new Cache(parsedParams); // Check resized cache - const cache = await get(`${key1}/${key2}`); - if (cache) { - // console.log('Got cache', address); + const cachedResizedImage = await cache.getResizedImage(); + if (cachedResizedImage) { setHeader(res); - return cache.pipe(res); + return (cachedResizedImage as Readable).pipe(res); } // Check base cache - const base = await get(`${key1}/${key1}`); - let baseImage; - if (base) { - baseImage = await streamToBuffer(base); - // console.log('Got base cache'); - } else { - // console.log('No cache for', key1, base); + const cachedBaseImage = await cache.getBaseImage(); + let baseImage: Buffer; + if (cachedBaseImage) { + baseImage = await streamToBuffer(cachedBaseImage as Readable); + } else { let currentResolvers: string[] = constants.resolvers.avatar; if (type === 'token') currentResolvers = constants.resolvers.token; if (type === 'space') currentResolvers = constants.resolvers.space; @@ -109,17 +101,10 @@ router.get(`/:type(${TYPE_CONSTRAINTS})/:id`, async (req, res) => { res.send(resizedImage); // Store cache - try { - if (!base) { - await set(`${key1}/${key1}`, baseImage); - console.log('Stored base cache', key1); - } - await set(`${key1}/${key2}`, resizedImage); - console.log('Stored cache', address); - } catch (e) { - capture(e); - console.log('Store cache failed', address, e); + if (!cachedBaseImage) { + await cache.setBaseImage(baseImage); } + await cache.setResizedImage(resizedImage); }); export default router; diff --git a/src/aws.ts b/src/aws.ts index 8c9c98fb..42d6788f 100644 --- a/src/aws.ts +++ b/src/aws.ts @@ -1,14 +1,18 @@ import * as AWS from '@aws-sdk/client-s3'; import { Readable } from 'stream'; -let client; +let client: AWS.S3; +const dir = 'stamp-4'; const bucket = process.env.AWS_BUCKET_NAME; const region = process.env.AWS_REGION; const endpoint = process.env.AWS_ENDPOINT || undefined; if (region) client = new AWS.S3({ region, endpoint }); -const dir = 'stamp-4'; -export async function streamToBuffer(stream: Readable) { +export const isConfigured = !!(bucket && region); + +if (isConfigured) client = new AWS.S3({ region, endpoint }); + +export async function streamToBuffer(stream: Readable): Promise { return await new Promise((resolve, reject) => { const chunks: Uint8Array[] = []; stream.on('data', chunk => chunks.push(Buffer.from(chunk))); @@ -17,53 +21,40 @@ export async function streamToBuffer(stream: Readable) { }); } -export async function set(key, value) { - try { - const command = new AWS.PutObjectCommand({ - Bucket: bucket, - Key: `public/${dir}/${key}`, - Body: value, - ContentType: 'image/webp' - }); +export async function set(key: string, value: Buffer) { + const command = new AWS.PutObjectCommand({ + Bucket: bucket, + Key: `public/${dir}/${key}`, + Body: value, + ContentType: 'image/webp' + }); - await client.send(command); - } catch (e) { - console.log('Store cache failed', e); - throw e; - } + return await client.send(command); } -export async function clear(path) { - try { - const listedObjects = await client.listObjectsV2({ - Bucket: bucket, - Prefix: `public/${dir}/${path}` - }); - if (!listedObjects.Contents || listedObjects.Contents.length === 0) return false; - const objs = listedObjects.Contents.map(obj => ({ Key: obj.Key })); - await client.deleteObjects({ - Bucket: bucket, - Delete: { Objects: objs } - }); - if (listedObjects.IsTruncated) await clear(path); - console.log('Cleared cache', path); - return path; - } catch (e) { - console.log('Clear cache failed', e); - throw e; - } +export async function clear(path: string): Promise { + const listedObjects = await client.listObjectsV2({ + Bucket: bucket, + Prefix: `public/${dir}/${path}` + }); + if (!listedObjects.Contents || listedObjects.Contents.length === 0) return false; + const objs = listedObjects.Contents.map(obj => ({ Key: obj.Key })); + await client.deleteObjects({ + Bucket: bucket, + Delete: { Objects: objs } + }); + if (listedObjects.IsTruncated) await clear(path); + return true; } -export async function get(key) { +export async function get(key: string): Promise { try { const command = new AWS.GetObjectCommand({ Bucket: bucket, Key: `public/${dir}/${key}` }); - const { Body } = await client.send(command); - - return Body; + return (await client.send(command)).Body as Readable; } catch (e) { return false; } diff --git a/src/helpers/metrics.ts b/src/helpers/metrics.ts index b468f215..e834f3dc 100644 --- a/src/helpers/metrics.ts +++ b/src/helpers/metrics.ts @@ -31,3 +31,9 @@ export const addressResolversCacheHitCount = new client.Counter({ help: 'Number of hit/miss of the address resolvers cache layer', labelNames: ['status'] }); + +export const imageResolversCacheHitCount = new client.Counter({ + name: 'image_resolvers_cache_hit_count', + help: 'Number of hit/miss of the image resolvers cache layer', + labelNames: ['status'] +}); diff --git a/src/resolvers/cache.ts b/src/resolvers/cache.ts new file mode 100644 index 00000000..241f074e --- /dev/null +++ b/src/resolvers/cache.ts @@ -0,0 +1,110 @@ +import { createHash } from 'crypto'; +import { Readable } from 'stream'; +import { set as setCache, get as getCache, clear as clearCache, isConfigured } from '../aws'; +import constants from '../constants.json'; +import { imageResolversCacheHitCount } from '../helpers/metrics'; +import { capture } from '@snapshot-labs/snapshot-sentry'; + +export function sha256(str: string) { + return createHash('sha256') + .update(str) + .digest('hex'); +} + +type ParamsType = { + type: string; + network: string; + address: string; + w: number; + h: number; + fallback?: string; + cb?: string; +}; + +export default class Cache { + baseImageCacheKey: string; + resizedImageCacheKey: string; + isConfigured: boolean; + + constructor({ type, network, address, w, h, fallback, cb }: ParamsType) { + const data = { type, network, address, w, h }; + if (fallback !== 'blockie') data['fallback'] = fallback; + if (cb) data['cb'] = cb; + + const baseImageKey = this._buildKey({ ...data, w: constants.max, h: constants.max }); + const resizedImageKey = this._buildKey(data); + + this.baseImageCacheKey = `${baseImageKey}/${baseImageKey}`; + this.resizedImageCacheKey = `${baseImageKey}/${resizedImageKey}`; + this.isConfigured = isConfigured; + + if (!this.isConfigured) { + console.log('[cache:resolver] Cache is not configured'); + } + } + + async getBaseImage(): Promise { + return await this._getCache(this.baseImageCacheKey); + } + + async getResizedImage(): Promise { + return await this._getCache(this.resizedImageCacheKey); + } + + async setBaseImage(value: Buffer) { + return await this._setCache(this.baseImageCacheKey, value); + } + + async setResizedImage(value: Buffer) { + return await this._setCache(this.resizedImageCacheKey, value); + } + + async clear(): Promise { + if (!this.isConfigured) return false; + + try { + const result = await clearCache(this.baseImageCacheKey); + + console.log(`[cache:resolver] Cached cleared ${this.baseImageCacheKey}`); + + return result; + } catch (e) { + console.log(`[cache:resolver] Failed to clear cache ${this.baseImageCacheKey}`); + capture(e); + return false; + } + } + + private async _getCache(key: string) { + if (!this.isConfigured) return false; + + try { + console.log(`[cache:resolver] Getting cache ${key}`); + const cache = await getCache(key); + + imageResolversCacheHitCount.inc({ status: cache ? 'HIT' : 'MISS' }, 1); + + return cache; + } catch (e) { + capture(e); + console.log(`[cache:resolver] Failed to get cache ${key}`); + return false; + } + } + + private async _setCache(key: string, value: Buffer) { + if (!this.isConfigured) return false; + + try { + console.log(`[cache:resolver] Setting cache ${key}`); + return await setCache(key, value); + } catch (e) { + capture(e); + console.log(`[cache:resolver] Failed to set cache ${key}`); + } + } + + private _buildKey(params: ParamsType): string { + return sha256(JSON.stringify(params)); + } +} diff --git a/src/utils.ts b/src/utils.ts index 002e3b32..4de44d24 100644 --- a/src/utils.ts +++ b/src/utils.ts @@ -1,4 +1,3 @@ -import { createHash } from 'crypto'; import sharp from 'sharp'; import { Response } from 'express'; import { StaticJsonRpcProvider } from '@ethersproject/providers'; @@ -21,12 +20,6 @@ export function getProvider(network: number): StaticJsonRpcProvider { return providers[`_${network}`]; } -export function sha256(str) { - return createHash('sha256') - .update(str) - .digest('hex'); -} - export async function resize(input, w, h) { return sharp(input) .resize(w, h) @@ -76,6 +69,7 @@ export async function parseQuery(id, type, query) { if (h < 1 || h > maxSize || isNaN(h)) h = size; return { + type, address, network, w, @@ -100,30 +94,6 @@ export function getUrl(url) { return snapshot.utils.getUrl(url, gateway); } -export function getCacheKey({ - type, - network, - address, - w, - h, - fallback, - cb -}: { - type: string; - network: string; - address: string; - w: number; - h: number; - fallback: string; - cb?: string; -}) { - const data = { type, network, address, w, h }; - if (fallback !== 'blockie') data['fallback'] = fallback; - if (cb) data['cb'] = cb; - - return sha256(JSON.stringify(data)); -} - export function setHeader(res: Response, cacheType: 'SHORT_CACHE' | 'LONG_CACHE' = 'LONG_CACHE') { const ttl = cacheType === 'SHORT_CACHE' ? constants.shortTtl : constants.ttl; diff --git a/test/fixtures/sample.webp b/test/fixtures/sample.webp new file mode 100644 index 00000000..122741b6 Binary files /dev/null and b/test/fixtures/sample.webp differ diff --git a/test/integration/resolvers/cache.test.ts b/test/integration/resolvers/cache.test.ts new file mode 100644 index 00000000..1ae61321 --- /dev/null +++ b/test/integration/resolvers/cache.test.ts @@ -0,0 +1,129 @@ +import path from 'path'; +import fs from 'fs'; +import { Readable } from 'stream'; +import Cache from '../../../src/resolvers/cache'; +import { parseQuery } from '../../../src/utils'; +import { isConfigured, set, streamToBuffer } from '../../../src/aws'; +import constants from '../../../src/constants.json'; + +const image_buffer = fs.readFileSync(path.join(__dirname, '../../fixtures/sample.webp')); + +describe('image resolver cache', () => { + let cache: Cache; + + if (!isConfigured) { + it.todo('needs to configure the cache for the tests to run'); + } else { + describe('getBaseImage()', () => { + afterEach(async () => { + await cache.clear(); + }); + + describe('when the image is cached', () => { + it('should return the cached image', async () => { + const parsedQuery = await parseQuery('0x0-test-getbaseimage', 'avatar', {}); + cache = new Cache(parsedQuery); + + await set(cache.baseImageCacheKey, image_buffer); + const result = await cache.getBaseImage(); + return expect(streamToBuffer(result as Readable)).resolves.toEqual(image_buffer); + }); + }); + + describe('when the image is not cached', () => { + it('should return false', async () => { + const parsedQuery = await parseQuery('0x1-test-getbaseimage', 'avatar', {}); + cache = new Cache(parsedQuery); + + return expect(cache.getBaseImage()).resolves.toBe(false); + }); + }); + }); + + describe('getResizedImage()', () => { + afterEach(async () => { + await cache.clear(); + }); + + describe('when the image is cached', () => { + it('should return the cached image', async () => { + const parsedQuery = await parseQuery('0x0-test-getresizedimage', 'avatar', {}); + cache = new Cache(parsedQuery); + + await set(cache.resizedImageCacheKey, image_buffer); + const result = await cache.getResizedImage(); + return expect(streamToBuffer(result as Readable)).resolves.toEqual(image_buffer); + }); + }); + + describe('when the image is not cached', () => { + it('should return false', async () => { + const parsedQuery = await parseQuery('0x1-test-getresizedimage', 'avatar', {}); + cache = new Cache(parsedQuery); + + return expect(cache.getResizedImage()).resolves.toBe(false); + }); + }); + }); + + describe('setBaseImage', () => { + it('should save the image in the cache', async () => { + const parsedQuery = await parseQuery('0x0-set-base-image', 'avatar', {}); + cache = new Cache(parsedQuery); + + return expect(cache.setBaseImage(image_buffer)).resolves.toEqual( + expect.objectContaining({ + $metadata: expect.objectContaining({ httpStatusCode: 200 }) + }) + ); + }); + }); + + describe('setResizedImage()', () => { + it('should save the image in the cache', async () => { + const parsedQuery = await parseQuery('0x0-set-resized-image', 'avatar', {}); + cache = new Cache(parsedQuery); + + return expect(cache.setResizedImage(image_buffer)).resolves.toEqual( + expect.objectContaining({ + $metadata: expect.objectContaining({ httpStatusCode: 200 }) + }) + ); + }); + }); + + describe('clear()', () => { + describe('when the cache exist', () => { + it('should clear the cache', async () => { + const parsedQuery = await parseQuery('0x0-clear-exist', 'avatar', { + s: constants.max, + fb: 'fb-0', + cb: 'cb-0' + }); + cache = new Cache(parsedQuery); + await cache.setBaseImage(image_buffer); + + await new Promise(resolve => { + setTimeout(resolve, 3e3); + }); + + expect(cache.clear()).resolves.toBe(true); + expect(cache.getBaseImage()).resolves.toBe(false); + expect(cache.getResizedImage()).resolves.toBe(false); + }); + }); + + describe('when the cache does not exist', () => { + it('should return false', async () => { + const parsedQuery = await parseQuery('0x0-clear-not-exist', 'avatar', { + s: constants.max, + fb: 'fb-1', + cb: 'cb-1' + }); + cache = new Cache(parsedQuery); + return expect(cache.clear()).resolves.toBe(false); + }); + }); + }); + } +});