feat: tokenlists resolver

package.json

@@ -12,7 +12,9 @@
     "start:test": "dotenv -e test/.env.test yarn dev",
     "test": "PORT=3003 start-server-and-test 'yarn start:test' http://localhost:3003 'dotenv -e test/.env.test yarn jest'",
     "test:integration": "dotenv -e test/.env.test yarn jest --runInBand --collectCoverage=false test/integration",
-    "test:e2e": "PORT=3003 start-server-and-test 'yarn start:test' http://localhost:3003 'dotenv -e test/.env.test jest --runInBand --collectCoverage=false test/e2e/'"
+    "test:e2e": "PORT=3003 start-server-and-test 'yarn start:test' http://localhost:3003 'dotenv -e test/.env.test jest --runInBand --collectCoverage=false test/e2e/'",
+    "test:e2e:tokenlists": "PORT=3003 start-server-and-test 'yarn start:test' http://localhost:3003 'dotenv -e test/.env.test jest --runInBand --collectCoverage=false test/e2e/tokenlists.test.ts'",
+    "test:unit": "yarn jest --runInBand --collectCoverage=false test/unit"
   },
   "dependencies": {
     "@adraffy/ens-normalize": "^1.10.0",

src/constants.json

@@ -6,7 +6,7 @@
   "resolvers": {
     "avatar": ["snapshot", "ens", "lens", "farcaster", "starknet"],
     "user-cover": ["user-cover"],
-    "token": ["trustwallet", "zapper"],
+    "token": ["trustwallet", "zapper", "tokenlists"],
     "space": ["space"],
     "space-cover": ["space-cover"],
     "space-sx": ["space-sx"],

src/helpers/tokenlists.ts

@@ -0,0 +1,177 @@
+import { getAddress } from '@ethersproject/address';
+
+type TokenlistToken = {
+  chainId: number;
+  address: string;
+  logoURI: string;
+};
+
+type AggregatedTokenList = Map<string, string[]>;
+
+const TOKENLISTS_URL =
+  'https://raw.githubusercontent.com/Uniswap/tokenlists-org/master/src/token-lists.json';
+const REQUEST_TIMEOUT = 5000;
+const TTL = 1000 * 60 * 60 * 24;
+let aggregatedTokenList: AggregatedTokenList = new Map();
+let lastUpdateTimestamp: number | undefined;
+let isUpdating = false;
+
+function isTokenlistToken(token: unknown): token is TokenlistToken {
+  if (typeof token !== 'object' || token === null) {
+    return false;
+  }
+
+  const { chainId, address, logoURI } = token as TokenlistToken;
+
+  return typeof chainId === 'number' && typeof address === 'string' && typeof logoURI === 'string';
+}
+
+function isExpired() {
+  return !lastUpdateTimestamp || Date.now() - lastUpdateTimestamp > TTL;
+}
+
+function normalizeUri(uri: string) {
+  if (!uri.startsWith('http') && uri.endsWith('.eth')) {
+    uri = `https://${uri}.limo`;
+  }
+  if (uri.startsWith('ipfs://')) {
+    uri = `https://ipfs.io/ipfs/${uri.slice(7)}`;
+  }
+  return uri;
+}
+
+async function fetchUri(uri: string) {
+  return await fetch(normalizeUri(uri), { signal: AbortSignal.timeout(REQUEST_TIMEOUT) });
+}
+
+async function fetchListUris() {
+  try {
+    const response = await fetchUri(TOKENLISTS_URL);
+    const tokenLists = await response.json();
+    const uris = Object.keys(tokenLists);
+
+    return uris;
+  } catch (e) {
+    return [];
+  }
+}
+
+export async function fetchTokens(listUri: string) {
+  try {
+    const response = await fetchUri(listUri);
+    const { tokens } = await response.json();
+    if (!tokens || !Array.isArray(tokens)) {
+      throw new Error('Invalid token list');
+    }
+    return tokens.filter(isTokenlistToken);
+  } catch (e) {
+    return [];
+  }
+}
+
+const REPLACE_SIZE_REGEXES: { pattern: RegExp; replacement: string }[] = [
+  {
+    pattern: /assets.coingecko.com\/coins\/images\/(\d+)\/(thumb|small)/,
+    replacement: 'assets.coingecko.com/coins/images/$1/large'
+  }
+];
+
+// TODO: Since we do the sorting by keyword match, we should probably not change the URLs in place but add the "large" version to the list of URIs.
+// Might be better for fallback mechanisms, instead of overwriting the version that was fetched.
+export function replaceURIPatterns(uri: string) {
+  for (const { pattern, replacement } of REPLACE_SIZE_REGEXES) {
+    uri = uri.replace(pattern, replacement);
+  }
+  return uri;
+}
+
+const sizeKeywords = [
+  'xxl',
+  'xl',
+  'large',
+  'lg',
+  'big',
+  'medium',
+  'md',
+  'small',
+  'sm',
+  'thumb',
+  'icon',
+  'xs',
+  'xxs'
+];
+
+/**
+ * Sorts URIs by the size keyword in the URI. The order in the array above is the order of the sort.
+ */
+export function sortByKeywordMatch(a: string, b: string) {
+  try {
+    const aPath = new URL(a).pathname;
+    const bPath = new URL(b).pathname;
+
+    const keywordRegex = new RegExp(`\\b(${sizeKeywords.join('|')})\\b`);
+
+    const aMatch = aPath.match(keywordRegex);
+    const bMatch = bPath.match(keywordRegex);
+
+    if (aMatch && bMatch) {
+      return sizeKeywords.indexOf(aMatch[1]) - sizeKeywords.indexOf(bMatch[1]);
+    } else if (aMatch) {
+      return -1;
+    } else if (bMatch) {
+      return 1;
+    } else {
+      return a.localeCompare(b);
+    }
+  } catch (e) {
+    return 0;
+  }
+}
+
+function getTokenKey(address: string, chainId: string) {
+  return `${chainId}-${getAddress(address)}`;
+}
+
+export async function updateExpiredAggregatedTokenList() {
+  if (!isExpired() || isUpdating) {
+    return;
+  }
+
+  isUpdating = true;
+
+  const newTokenMap = new Map<string, string[]>();
+
+  const tokenListUris = await fetchListUris();
+  const tokenLists = await Promise.all(tokenListUris.map(fetchTokens));
+
+  for (const tokens of tokenLists) {
+    for (const token of tokens) {
+      const logoURI = normalizeUri(replaceURIPatterns(token.logoURI));
+      const tokenKey = getTokenKey(token.address, token.chainId.toString());
+
+      const existingToken = newTokenMap.get(tokenKey);
+      if (existingToken) {
+        existingToken.push(logoURI);
+      } else {
+        newTokenMap.set(tokenKey, [logoURI]);
+      }
+    }
+  }
+
+  newTokenMap.forEach(token => token.sort(sortByKeywordMatch));
+
+  aggregatedTokenList = newTokenMap;
+  lastUpdateTimestamp = Date.now();
+  isUpdating = false;
+}
+
+export function findImageUrl(address: string, chainId: string) {
+  const tokenKey = getTokenKey(address, chainId);
+  const token = aggregatedTokenList.get(tokenKey);
+
+  if (!token) {
+    throw new Error('Token not found in aggregated tokenlist');
+  }
+
+  return token[0];
+}

src/index.ts

@@ -20,7 +20,7 @@ app.use(cors({ maxAge: 86400 }));
 app.use(compression());
 app.use('/', api);
 
-app.get('/', (req, res) => {
+app.get('/', (_req, res) => {
   const commit = process.env.COMMIT_HASH ?? undefined;
   res.json({ name, version, commit });
 });

src/resolvers/index.ts

@@ -14,6 +14,7 @@ import lens from './lens';
 import zapper from './zapper';
 import starknet from './starknet';
 import farcaster from './farcaster';
+import tokenlists from './tokenlists';
 
 export default {
   blockie,
@@ -30,5 +31,6 @@ export default {
   lens,
   zapper,
   starknet,
-  farcaster
+  farcaster,
+  tokenlists
 };

src/resolvers/tokenlists.ts

@@ -0,0 +1,16 @@
+import { resize } from '../utils';
+import { max } from '../constants.json';
+import { fetchHttpImage } from './utils';
+import { findImageUrl, updateExpiredAggregatedTokenList } from '../helpers/tokenlists';
+
+export default async function resolve(address: string, chainId: string) {
+  try {
+    await updateExpiredAggregatedTokenList();
+    const url = findImageUrl(address, chainId);
+    const image = await fetchHttpImage(url);
+
+    return await resize(image, max, max);
+  } catch (e) {
+    return false;
+  }
+}

test/e2e/tokenlists.test.ts

@@ -0,0 +1,67 @@
+import sharp from 'sharp';
+import axios from 'axios';
+import crypto from 'crypto';
+
+const HOST = `http://localhost:${process.env.PORT || 3003}`;
+const cUSDC_TOKEN_ADDRESS_ON_MAIN = '0x39AA39c021dfbaE8faC545936693aC917d5E7563';
+const ERC3770_ADDRESS = 'oeth:0xe0BB0D3DE8c10976511e5030cA403dBf4c25165B';
+const EIP155_ADDRESS = 'eip155:1:0x0bc529c00C6401aEF6D220BE8C6Ea1667F6Ad93e';
+
+function getImageFingerprint(input: string) {
+  return crypto
+    .createHash('sha256')
+    .update(input)
+    .digest('hex');
+}
+
+function getImageResponse(identifier: string) {
+  return axios.get(`${HOST}/token/${identifier}?resolver=tokenlists`, {
+    responseType: 'arraybuffer',
+    headers: {
+      'Cache-Control': 'no-cache',
+      Pragma: 'no-cache',
+      Expires: '0'
+    }
+  });
+}
+
+// tokenlists resolver needs a moment on first run
+// there's probably a better way to handle this
+jest.setTimeout(60_000);
+
+describe('tokenlist resolver', () => {
+  it('returns an image for standard address', async () => {
+    const response = await getImageResponse(cUSDC_TOKEN_ADDRESS_ON_MAIN);
+
+    expect(response.status).toBe(200);
+    expect(response.headers['content-type']).toBe('image/webp');
+  });
+
+  it('returns correct image for ERC3770 address', async () => {
+    const response = await getImageResponse(ERC3770_ADDRESS);
+
+    const image = sharp(response.data);
+    const imageBuffer = await image.raw().toBuffer();
+
+    const fingerprint = getImageFingerprint(imageBuffer.toString('hex'));
+    const expectedFingerprint = 'ac601f072065d4d03e6ef906c1dc3074d7ad52b9c715d0db6941ec89bf2073a1';
+
+    expect(response.status).toBe(200);
+    expect(response.headers['content-type']).toBe('image/webp');
+    expect(fingerprint).toBe(expectedFingerprint);
+  });
+
+  it('returns an image for EIP155 address', async () => {
+    const response = await getImageResponse(EIP155_ADDRESS);
+
+    const image = sharp(response.data);
+    const imageBuffer = await image.raw().toBuffer();
+
+    const fingerprint = getImageFingerprint(imageBuffer.toString('hex'));
+    const expectedFingerprint = '8118786398e4756b2b7e8e224ec2bb5cbe3b26ee93ceff3b19d40f81c8ce45a2';
+
+    expect(response.status).toBe(200);
+    expect(response.headers['content-type']).toBe('image/webp');
+    expect(fingerprint).toBe(expectedFingerprint);
+  });
+});

test/unit/tokenlists.test.ts

@@ -0,0 +1,53 @@
+import { replaceURIPatterns, sortByKeywordMatch } from '../../src/helpers/tokenlists';
+
+jest.setTimeout(60_000);
+
+describe('tokenlists helper', () => {
+  it('replaceURIPatterns should replace known image size related parts in URLs', () => {
+    const uris = [
+      'https://assets.coingecko.com/coins/images/123/thumb',
+      'https://assets.coingecko.com/coins/images/456/small'
+    ];
+
+    const expectedUris = [
+      'https://assets.coingecko.com/coins/images/123/large',
+      'https://assets.coingecko.com/coins/images/456/large'
+    ];
+
+    uris.forEach((uri, i) => {
+      expect(replaceURIPatterns(uri)).toBe(expectedUris[i]);
+    });
+  });
+
+  it('sortByKeywordMatch should sort URIs by size keywords', () => {
+    const uris = [
+      'https://assets.coingecko.com/coins/images/123/thumb',
+      'https://assets.coingecko.com/coins/images/2021/xxs',
+      'https://assets.coingecko.com/coins/images/456-small',
+      'https://assets.coingecko.com/coins/images/789/medium',
+      'https://assets.coingecko.com/coins/images/1011/large',
+      'https://assets.xl.coingecko.com/coins/images/1213',
+      'https://assets.coingecko.com/coins/images/1415/xxl',
+      'https://assets.coingecko.com/coins/images/1617/icon',
+      'https://assets.coingecko.com/coins/images/2021/lg/logo.png',
+      'https://assets.coingecko.com/coins/images/2021/md-logo.png'
+    ];
+
+    const expectedUris = [
+      'https://assets.coingecko.com/coins/images/1415/xxl',
+      'https://assets.coingecko.com/coins/images/1011/large',
+      'https://assets.coingecko.com/coins/images/2021/lg/logo.png',
+      'https://assets.coingecko.com/coins/images/789/medium',
+      'https://assets.coingecko.com/coins/images/2021/md-logo.png',
+      'https://assets.coingecko.com/coins/images/456-small',
+      'https://assets.coingecko.com/coins/images/123/thumb',
+      'https://assets.coingecko.com/coins/images/1617/icon',
+      'https://assets.coingecko.com/coins/images/2021/xxs',
+
+      // no keyword, should be at the end (domain part should be ignored)
+      'https://assets.xl.coingecko.com/coins/images/1213'
+    ];
+
+    expect(uris.sort(sortByKeywordMatch)).toEqual(expectedUris);
+  });
+});

yarn.lock

@@ -6290,7 +6290,7 @@ [email protected]:
   resolved "https://registry.yarnpkg.com/scrypt-js/-/scrypt-js-3.0.1.tgz#d314a57c2aef69d1ad98a138a21fe9eafa9ee312"
   integrity sha512-cdwTTnqPu0Hyvf5in5asVdZocVDTNRmR7XEcJuIzMjJeSHybHl7vpB66AzwTaIg6CLSbtjcxc8fqcySfnTkccA==
 
-[email protected], semver@^7.3.2, semver@^7.3.5, semver@^7.3.7, semver@^7.5.3:
+[email protected], semver@^7.3.2, semver@^7.3.5, semver@^7.5.3:
   version "7.6.0"
   resolved "https://registry.yarnpkg.com/semver/-/semver-7.6.0.tgz#1a46a4db4bffcccd97b743b5005c8325f23d4e2d"
   integrity sha512-EnwXhrlwXMk9gKu5/flx5sv/an57AkRplG3hTK68W7FRDN+k+OWBj65M7719OkA82XLBxrcX0KSHj+X5COhOVg==
@@ -6307,6 +6307,11 @@ semver@^6.0.0, semver@^6.1.2, semver@^6.3.0, semver@^6.3.1:
   resolved "https://registry.yarnpkg.com/semver/-/semver-6.3.1.tgz#556d2ef8689146e46dcea4bfdd095f3434dffcb4"
   integrity sha512-BR7VvDCVHO+q2xBEWskxS6DJE1qRnb7DxzUrogb71CWoSficBxYsiAGd+Kl0mmq/MprG9yArRkyrQxTO6XjMzA==
 
+semver@^7.3.7:
+  version "7.6.3"
+  resolved "https://registry.yarnpkg.com/semver/-/semver-7.6.3.tgz#980f7b5550bc175fb4dc09403085627f9eb33143"
+  integrity sha512-oVekP1cKtI+CTDvHWYFUcMtsK/00wmAEfyqKfNdARm8u1wNVhSgaX7A8d4UuIlUI5e84iEwOhs7ZPYRmzU9U6A==
+
 semver@~7.0.0:
   version "7.0.0"
   resolved "https://registry.yarnpkg.com/semver/-/semver-7.0.0.tgz#5f3ca35761e47e05b206c6daff2cf814f0316b8e"