diff --git a/package.json b/package.json index 627f7f70..f1f53516 100644 --- a/package.json +++ b/package.json @@ -12,7 +12,9 @@ "start:test": "dotenv -e test/.env.test yarn dev", "test": "PORT=3003 start-server-and-test 'yarn start:test' http://localhost:3003 'dotenv -e test/.env.test yarn jest'", "test:integration": "dotenv -e test/.env.test yarn jest --runInBand --collectCoverage=false test/integration", - "test:e2e": "PORT=3003 start-server-and-test 'yarn start:test' http://localhost:3003 'dotenv -e test/.env.test jest --runInBand --collectCoverage=false test/e2e/'" + "test:e2e": "PORT=3003 start-server-and-test 'yarn start:test' http://localhost:3003 'dotenv -e test/.env.test jest --runInBand --collectCoverage=false test/e2e/'", + "test:e2e:tokenlists": "PORT=3003 start-server-and-test 'yarn start:test' http://localhost:3003 'dotenv -e test/.env.test jest --runInBand --collectCoverage=false test/e2e/tokenlists.test.ts'", + "test:unit": "yarn jest --runInBand --collectCoverage=false test/unit" }, "dependencies": { "@adraffy/ens-normalize": "^1.10.0", diff --git a/src/constants.json b/src/constants.json index a2bb44c7..98979a9c 100644 --- a/src/constants.json +++ b/src/constants.json @@ -6,7 +6,7 @@ "resolvers": { "avatar": ["snapshot", "ens", "lens", "farcaster", "starknet"], "user-cover": ["user-cover"], - "token": ["trustwallet", "zapper"], + "token": ["trustwallet", "zapper", "tokenlists"], "space": ["space"], "space-cover": ["space-cover"], "space-sx": ["space-sx"], diff --git a/src/helpers/tokenlists.ts b/src/helpers/tokenlists.ts new file mode 100644 index 00000000..2d59a82c --- /dev/null +++ b/src/helpers/tokenlists.ts @@ -0,0 +1,177 @@ +import { getAddress } from '@ethersproject/address'; + +type TokenlistToken = { + chainId: number; + address: string; + logoURI: string; +}; + +type AggregatedTokenList = Map; + +const TOKENLISTS_URL = + 'https://raw.githubusercontent.com/Uniswap/tokenlists-org/master/src/token-lists.json'; +const REQUEST_TIMEOUT = 5000; +const TTL = 1000 * 60 * 60 * 24; +let aggregatedTokenList: AggregatedTokenList = new Map(); +let lastUpdateTimestamp: number | undefined; +let isUpdating = false; + +function isTokenlistToken(token: unknown): token is TokenlistToken { + if (typeof token !== 'object' || token === null) { + return false; + } + + const { chainId, address, logoURI } = token as TokenlistToken; + + return typeof chainId === 'number' && typeof address === 'string' && typeof logoURI === 'string'; +} + +function isExpired() { + return !lastUpdateTimestamp || Date.now() - lastUpdateTimestamp > TTL; +} + +function normalizeUri(uri: string) { + if (!uri.startsWith('http') && uri.endsWith('.eth')) { + uri = `https://${uri}.limo`; + } + if (uri.startsWith('ipfs://')) { + uri = `https://ipfs.io/ipfs/${uri.slice(7)}`; + } + return uri; +} + +async function fetchUri(uri: string) { + return await fetch(normalizeUri(uri), { signal: AbortSignal.timeout(REQUEST_TIMEOUT) }); +} + +async function fetchListUris() { + try { + const response = await fetchUri(TOKENLISTS_URL); + const tokenLists = await response.json(); + const uris = Object.keys(tokenLists); + + return uris; + } catch (e) { + return []; + } +} + +export async function fetchTokens(listUri: string) { + try { + const response = await fetchUri(listUri); + const { tokens } = await response.json(); + if (!tokens || !Array.isArray(tokens)) { + throw new Error('Invalid token list'); + } + return tokens.filter(isTokenlistToken); + } catch (e) { + return []; + } +} + +const REPLACE_SIZE_REGEXES: { pattern: RegExp; replacement: string }[] = [ + { + pattern: /assets.coingecko.com\/coins\/images\/(\d+)\/(thumb|small)/, + replacement: 'assets.coingecko.com/coins/images/$1/large' + } +]; + +// TODO: Since we do the sorting by keyword match, we should probably not change the URLs in place but add the "large" version to the list of URIs. +// Might be better for fallback mechanisms, instead of overwriting the version that was fetched. +export function replaceURIPatterns(uri: string) { + for (const { pattern, replacement } of REPLACE_SIZE_REGEXES) { + uri = uri.replace(pattern, replacement); + } + return uri; +} + +const sizeKeywords = [ + 'xxl', + 'xl', + 'large', + 'lg', + 'big', + 'medium', + 'md', + 'small', + 'sm', + 'thumb', + 'icon', + 'xs', + 'xxs' +]; + +/** + * Sorts URIs by the size keyword in the URI. The order in the array above is the order of the sort. + */ +export function sortByKeywordMatch(a: string, b: string) { + try { + const aPath = new URL(a).pathname; + const bPath = new URL(b).pathname; + + const keywordRegex = new RegExp(`\\b(${sizeKeywords.join('|')})\\b`); + + const aMatch = aPath.match(keywordRegex); + const bMatch = bPath.match(keywordRegex); + + if (aMatch && bMatch) { + return sizeKeywords.indexOf(aMatch[1]) - sizeKeywords.indexOf(bMatch[1]); + } else if (aMatch) { + return -1; + } else if (bMatch) { + return 1; + } else { + return a.localeCompare(b); + } + } catch (e) { + return 0; + } +} + +function getTokenKey(address: string, chainId: string) { + return `${chainId}-${getAddress(address)}`; +} + +export async function updateExpiredAggregatedTokenList() { + if (!isExpired() || isUpdating) { + return; + } + + isUpdating = true; + + const newTokenMap = new Map(); + + const tokenListUris = await fetchListUris(); + const tokenLists = await Promise.all(tokenListUris.map(fetchTokens)); + + for (const tokens of tokenLists) { + for (const token of tokens) { + const logoURI = normalizeUri(replaceURIPatterns(token.logoURI)); + const tokenKey = getTokenKey(token.address, token.chainId.toString()); + + const existingToken = newTokenMap.get(tokenKey); + if (existingToken) { + existingToken.push(logoURI); + } else { + newTokenMap.set(tokenKey, [logoURI]); + } + } + } + + newTokenMap.forEach(token => token.sort(sortByKeywordMatch)); + + aggregatedTokenList = newTokenMap; + lastUpdateTimestamp = Date.now(); + isUpdating = false; +} + +export function findImageUrl(address: string, chainId: string) { + const tokenKey = getTokenKey(address, chainId); + const token = aggregatedTokenList.get(tokenKey); + + if (!token) { + throw new Error('Token not found in aggregated tokenlist'); + } + + return token[0]; +} diff --git a/src/index.ts b/src/index.ts index 9a9a39df..3a0221b7 100644 --- a/src/index.ts +++ b/src/index.ts @@ -20,7 +20,7 @@ app.use(cors({ maxAge: 86400 })); app.use(compression()); app.use('/', api); -app.get('/', (req, res) => { +app.get('/', (_req, res) => { const commit = process.env.COMMIT_HASH ?? undefined; res.json({ name, version, commit }); }); diff --git a/src/resolvers/index.ts b/src/resolvers/index.ts index c578d91c..1b330594 100644 --- a/src/resolvers/index.ts +++ b/src/resolvers/index.ts @@ -14,6 +14,7 @@ import lens from './lens'; import zapper from './zapper'; import starknet from './starknet'; import farcaster from './farcaster'; +import tokenlists from './tokenlists'; export default { blockie, @@ -30,5 +31,6 @@ export default { lens, zapper, starknet, - farcaster + farcaster, + tokenlists }; diff --git a/src/resolvers/tokenlists.ts b/src/resolvers/tokenlists.ts new file mode 100644 index 00000000..d4b13e2a --- /dev/null +++ b/src/resolvers/tokenlists.ts @@ -0,0 +1,16 @@ +import { resize } from '../utils'; +import { max } from '../constants.json'; +import { fetchHttpImage } from './utils'; +import { findImageUrl, updateExpiredAggregatedTokenList } from '../helpers/tokenlists'; + +export default async function resolve(address: string, chainId: string) { + try { + await updateExpiredAggregatedTokenList(); + const url = findImageUrl(address, chainId); + const image = await fetchHttpImage(url); + + return await resize(image, max, max); + } catch (e) { + return false; + } +} diff --git a/test/e2e/tokenlists.test.ts b/test/e2e/tokenlists.test.ts new file mode 100644 index 00000000..a2b91ab5 --- /dev/null +++ b/test/e2e/tokenlists.test.ts @@ -0,0 +1,67 @@ +import sharp from 'sharp'; +import axios from 'axios'; +import crypto from 'crypto'; + +const HOST = `http://localhost:${process.env.PORT || 3003}`; +const cUSDC_TOKEN_ADDRESS_ON_MAIN = '0x39AA39c021dfbaE8faC545936693aC917d5E7563'; +const ERC3770_ADDRESS = 'oeth:0xe0BB0D3DE8c10976511e5030cA403dBf4c25165B'; +const EIP155_ADDRESS = 'eip155:1:0x0bc529c00C6401aEF6D220BE8C6Ea1667F6Ad93e'; + +function getImageFingerprint(input: string) { + return crypto + .createHash('sha256') + .update(input) + .digest('hex'); +} + +function getImageResponse(identifier: string) { + return axios.get(`${HOST}/token/${identifier}?resolver=tokenlists`, { + responseType: 'arraybuffer', + headers: { + 'Cache-Control': 'no-cache', + Pragma: 'no-cache', + Expires: '0' + } + }); +} + +// tokenlists resolver needs a moment on first run +// there's probably a better way to handle this +jest.setTimeout(60_000); + +describe('tokenlist resolver', () => { + it('returns an image for standard address', async () => { + const response = await getImageResponse(cUSDC_TOKEN_ADDRESS_ON_MAIN); + + expect(response.status).toBe(200); + expect(response.headers['content-type']).toBe('image/webp'); + }); + + it('returns correct image for ERC3770 address', async () => { + const response = await getImageResponse(ERC3770_ADDRESS); + + const image = sharp(response.data); + const imageBuffer = await image.raw().toBuffer(); + + const fingerprint = getImageFingerprint(imageBuffer.toString('hex')); + const expectedFingerprint = 'ac601f072065d4d03e6ef906c1dc3074d7ad52b9c715d0db6941ec89bf2073a1'; + + expect(response.status).toBe(200); + expect(response.headers['content-type']).toBe('image/webp'); + expect(fingerprint).toBe(expectedFingerprint); + }); + + it('returns an image for EIP155 address', async () => { + const response = await getImageResponse(EIP155_ADDRESS); + + const image = sharp(response.data); + const imageBuffer = await image.raw().toBuffer(); + + const fingerprint = getImageFingerprint(imageBuffer.toString('hex')); + const expectedFingerprint = '8118786398e4756b2b7e8e224ec2bb5cbe3b26ee93ceff3b19d40f81c8ce45a2'; + + expect(response.status).toBe(200); + expect(response.headers['content-type']).toBe('image/webp'); + expect(fingerprint).toBe(expectedFingerprint); + }); +}); diff --git a/test/unit/tokenlists.test.ts b/test/unit/tokenlists.test.ts new file mode 100644 index 00000000..2bae38da --- /dev/null +++ b/test/unit/tokenlists.test.ts @@ -0,0 +1,53 @@ +import { replaceURIPatterns, sortByKeywordMatch } from '../../src/helpers/tokenlists'; + +jest.setTimeout(60_000); + +describe('tokenlists helper', () => { + it('replaceURIPatterns should replace known image size related parts in URLs', () => { + const uris = [ + 'https://assets.coingecko.com/coins/images/123/thumb', + 'https://assets.coingecko.com/coins/images/456/small' + ]; + + const expectedUris = [ + 'https://assets.coingecko.com/coins/images/123/large', + 'https://assets.coingecko.com/coins/images/456/large' + ]; + + uris.forEach((uri, i) => { + expect(replaceURIPatterns(uri)).toBe(expectedUris[i]); + }); + }); + + it('sortByKeywordMatch should sort URIs by size keywords', () => { + const uris = [ + 'https://assets.coingecko.com/coins/images/123/thumb', + 'https://assets.coingecko.com/coins/images/2021/xxs', + 'https://assets.coingecko.com/coins/images/456-small', + 'https://assets.coingecko.com/coins/images/789/medium', + 'https://assets.coingecko.com/coins/images/1011/large', + 'https://assets.xl.coingecko.com/coins/images/1213', + 'https://assets.coingecko.com/coins/images/1415/xxl', + 'https://assets.coingecko.com/coins/images/1617/icon', + 'https://assets.coingecko.com/coins/images/2021/lg/logo.png', + 'https://assets.coingecko.com/coins/images/2021/md-logo.png' + ]; + + const expectedUris = [ + 'https://assets.coingecko.com/coins/images/1415/xxl', + 'https://assets.coingecko.com/coins/images/1011/large', + 'https://assets.coingecko.com/coins/images/2021/lg/logo.png', + 'https://assets.coingecko.com/coins/images/789/medium', + 'https://assets.coingecko.com/coins/images/2021/md-logo.png', + 'https://assets.coingecko.com/coins/images/456-small', + 'https://assets.coingecko.com/coins/images/123/thumb', + 'https://assets.coingecko.com/coins/images/1617/icon', + 'https://assets.coingecko.com/coins/images/2021/xxs', + + // no keyword, should be at the end (domain part should be ignored) + 'https://assets.xl.coingecko.com/coins/images/1213' + ]; + + expect(uris.sort(sortByKeywordMatch)).toEqual(expectedUris); + }); +}); diff --git a/yarn.lock b/yarn.lock index b67bfcd0..61433110 100644 --- a/yarn.lock +++ b/yarn.lock @@ -6290,7 +6290,7 @@ scrypt-js@3.0.1: resolved "https://registry.yarnpkg.com/scrypt-js/-/scrypt-js-3.0.1.tgz#d314a57c2aef69d1ad98a138a21fe9eafa9ee312" integrity sha512-cdwTTnqPu0Hyvf5in5asVdZocVDTNRmR7XEcJuIzMjJeSHybHl7vpB66AzwTaIg6CLSbtjcxc8fqcySfnTkccA== -semver@7.x, semver@^7.3.2, semver@^7.3.5, semver@^7.3.7, semver@^7.5.3: +semver@7.x, semver@^7.3.2, semver@^7.3.5, semver@^7.5.3: version "7.6.0" resolved "https://registry.yarnpkg.com/semver/-/semver-7.6.0.tgz#1a46a4db4bffcccd97b743b5005c8325f23d4e2d" integrity sha512-EnwXhrlwXMk9gKu5/flx5sv/an57AkRplG3hTK68W7FRDN+k+OWBj65M7719OkA82XLBxrcX0KSHj+X5COhOVg== @@ -6307,6 +6307,11 @@ semver@^6.0.0, semver@^6.1.2, semver@^6.3.0, semver@^6.3.1: resolved "https://registry.yarnpkg.com/semver/-/semver-6.3.1.tgz#556d2ef8689146e46dcea4bfdd095f3434dffcb4" integrity sha512-BR7VvDCVHO+q2xBEWskxS6DJE1qRnb7DxzUrogb71CWoSficBxYsiAGd+Kl0mmq/MprG9yArRkyrQxTO6XjMzA== +semver@^7.3.7: + version "7.6.3" + resolved "https://registry.yarnpkg.com/semver/-/semver-7.6.3.tgz#980f7b5550bc175fb4dc09403085627f9eb33143" + integrity sha512-oVekP1cKtI+CTDvHWYFUcMtsK/00wmAEfyqKfNdARm8u1wNVhSgaX7A8d4UuIlUI5e84iEwOhs7ZPYRmzU9U6A== + semver@~7.0.0: version "7.0.0" resolved "https://registry.yarnpkg.com/semver/-/semver-7.0.0.tgz#5f3ca35761e47e05b206c6daff2cf814f0316b8e"