[Internal] Tokenlist pairs update: Only manual update; use exclude/include config; no tokenlist_base (trustwallet#5451)

* Force include and exclude implementation.

* Rename update to updateAuto

* UpdateManual hooks

* UpdateManual hook fix

* Update existing tokenlist.json file, use exclude/include config.

* Force include only pairs against the main currency.

* PS config adjustment

* Remove tokenlist_base files

* Lint fix

* Remove all pairs first.

Co-authored-by: Catenocrypt <[email protected]>

Author: 2 people

.github/workflows/periodic-update.yml

@@ -5,7 +5,7 @@ on:
     - cron:  '0 7,19 * * *'
   workflow_dispatch:
 jobs:
-  periodic-update:
+  periodic-auto-update:
     runs-on: ubuntu-latest
     steps:
       - uses: actions/checkout@v2
@@ -18,7 +18,7 @@ jobs:
       - name: Install Dependencies
         run: npm ci
       - name: Run scripts
-        run: npm run update
+        run: npm run updateAuto
       - name: Show update result (diff)
         if: success()
         run: |

blockchains/ethereum/tokenlist_base.json

@@ -9,7 +9,8 @@
     "check-sanity": "ts-node ./script/entrypoint/check-sanity",
     "fix": "ts-node ./script/entrypoint/fix",
     "fix-sanity": "ts-node ./script/entrypoint/fix-sanity",
-    "update": "ts-node ./script/entrypoint/update",
+    "updateAuto": "ts-node ./script/entrypoint/updateAuto",
+    "update": "ts-node ./script/entrypoint/updateManual",
     "lint": "npx eslint . --ext .js,.jsx,.ts,.tsx",
     "lint:fix": "npx eslint . --ext .js,.jsx,.ts,.tsx --fix"
   },

blockchains/smartchain/tokenlist_base.json

@@ -125,7 +125,7 @@ export class BinanceAction implements ActionInterface {
         ];
     }
 
-    async update(): Promise<void> {
+    async updateAuto(): Promise<void> {
         // retrieve missing token images; BEP2 (bep8 not supported)
         const bep2InfoList = await retrieveBep2AssetList();
         const denylist: string[] = readJsonFile(getChainDenylistPath(binanceChain)) as string[];

package.json

@@ -8,6 +8,7 @@ import {
 } from "../generic/subgraph";
 import { Ethereum } from "../generic/blockchains";
 import {
+    parseForceList,
     rebuildTokenlist,
     TokenItem
 } from "../generic/tokenlists";
@@ -21,14 +22,17 @@ const PrimaryTokens: string[] = ["WETH", "ETH"];
 async function retrieveUniswapPairs(): Promise<PairInfo[]> {
     console.log(`Retrieving pairs from Uniswap, limit liquidity USD ${config.Uniswap_MinLiquidity}  volume ${config.Uniswap_MinVol24}  txcount ${config.Uniswap_MinTxCount24}`);
 
+    console.log(`  forceIncludeList: ${config.Uniswap_ForceInclude}`);
+    const includeList = parseForceList(config.Uniswap_ForceInclude);
+
     const pairs = await getTradingPairs(config.Uniswap_TradingPairsUrl, config.Uniswap_TradingPairsQuery);
     const filtered: PairInfo[] = [];
     pairs.forEach(x => {
         try {
             if (typeof(x) === "object") {
                 const pairInfo = x as PairInfo;
                 if (pairInfo) {
-                    if (checkTradingPair(pairInfo, config.Uniswap_MinLiquidity, config.Uniswap_MinVol24, config.Uniswap_MinTxCount24, PrimaryTokens)) {
+                    if (checkTradingPair(pairInfo, config.Uniswap_MinLiquidity, config.Uniswap_MinVol24, config.Uniswap_MinTxCount24, PrimaryTokens, includeList)) {
                         filtered.push(pairInfo);
                     }
                 }
@@ -71,15 +75,15 @@ async function generateTokenlist(): Promise<void> {
         }
         pairs2.push([tokenItem0, tokenItem1]);
     });
-    await rebuildTokenlist(Ethereum, pairs2, "Ethereum");
+    await rebuildTokenlist(Ethereum, pairs2, "Ethereum", config.Uniswap_ForceExclude);
 }
 
 export class EthereumAction implements ActionInterface {
     getName(): string { return "Ethereum"; }
 
     getSanityChecks(): CheckStepInterface[] { return []; }
 
-    async update(): Promise<void> {
-        //await generateTokenlist();
+    async updateManual(): Promise<void> {
+        await generateTokenlist();
     }
 }

script/blockchain/binance.ts

@@ -8,6 +8,7 @@ import {
 } from "../generic/subgraph";
 import { SmartChain } from "../generic/blockchains";
 import {
+    parseForceList,
     rebuildTokenlist,
     TokenItem
 } from "../generic/tokenlists";
@@ -20,14 +21,17 @@ const PrimaryTokens: string[] = ["WBNB", "BNB"];
 async function retrievePancakeSwapPairs(): Promise<PairInfo[]> {
     console.log(`Retrieving pairs from PancakeSwap, limit liquidity USD ${config.PancakeSwap_MinLiquidity}  volume ${config.PancakeSwap_MinVol24}  txcount ${config.PancakeSwap_MinTxCount24}`);
 
+    console.log(`  forceIncludeList: ${config.PancakeSwap_ForceInclude}`);
+    const includeList = parseForceList(config.PancakeSwap_ForceInclude);
+
     const pairs = await getTradingPairs(config.PancakeSwap_TradingPairsUrl, config.PancakeSwap_TradingPairsQuery);
     const filtered: PairInfo[] = [];
     pairs.forEach(x => {
         try {
             if (typeof(x) === "object") {
                 const pairInfo = x as PairInfo;
                 if (pairInfo) {
-                    if (checkTradingPair(pairInfo, config.PancakeSwap_MinLiquidity, config.PancakeSwap_MinVol24, config.PancakeSwap_MinTxCount24, PrimaryTokens)) {
+                    if (checkTradingPair(pairInfo, config.PancakeSwap_MinLiquidity, config.PancakeSwap_MinVol24, config.PancakeSwap_MinTxCount24, PrimaryTokens, includeList)) {
                         filtered.push(pairInfo);
                     }
                 }
@@ -70,15 +74,15 @@ async function generateTokenlist(): Promise<void> {
         }
         pairs2.push([tokenItem0, tokenItem1]);
     });
-    await rebuildTokenlist(SmartChain, pairs2, "Smart Chain");
+    await rebuildTokenlist(SmartChain, pairs2, "Smart Chain", config.PancakeSwap_ForceExclude);
 }
 
 export class SmartchainAction implements ActionInterface {
     getName(): string { return "Binance Smartchain"; }
 
     getSanityChecks(): CheckStepInterface[] { return []; }
 
-    async update(): Promise<void> {
-        //await generateTokenlist();
+    async updateManual(): Promise<void> {
+        await generateTokenlist();
     }
 }

script/blockchain/ethereum.ts

@@ -96,7 +96,7 @@ export class TezosAction implements ActionInterface {
         ];
     }
 
-    async update(): Promise<void> {
+    async updateAuto(): Promise<void> {
         await gen_validators_tezos();
     }
 }