Merge branch 'master' into feat-alias-vote-and-propose

package.json

@@ -32,7 +32,7 @@
     "@snapshot-labs/pineapple": "^1.1.0",
     "@snapshot-labs/snapshot-metrics": "^1.4.1",
     "@snapshot-labs/snapshot-sentry": "^1.5.5",
-    "@snapshot-labs/snapshot.js": "^0.11.21",
+    "@snapshot-labs/snapshot.js": "^0.11.26",
     "bluebird": "^3.7.2",
     "connection-string": "^1.0.1",
     "cors": "^2.8.5",

scripts/init_leaderboard_counters.ts

@@ -0,0 +1,92 @@
+import 'dotenv/config';
+import db from '../src/helpers/mysql';
+
+// Usage: yarn ts-node scripts/init_leaderboard_counters.ts --pivot TIMESTAMP
+async function main() {
+  let pivot: number | null = null;
+
+  process.argv.forEach((arg, index) => {
+    if (arg === '--pivot') {
+      if (!process.argv[index + 1]) throw new Error('Pivot timestamp is missing');
+      console.log('Filtered by votes.created >=', process.argv[index + 1]);
+      pivot = +process.argv[index + 1].trim();
+    }
+  });
+
+  if (!pivot) {
+    const firstVoted = await db.queryAsync(
+      'SELECT created FROM votes ORDER BY created ASC LIMIT 1'
+    );
+    if (!firstVoted.length) throw new Error('No votes found in the database');
+    pivot = firstVoted[0].created as number;
+  }
+
+  await processVotesCount(pivot);
+}
+
+async function processVotesCount(pivot: number) {
+  const processedVoters = new Set<string>();
+  const batchWindow = 60 * 60 * 24; // 1 day
+  let _pivot = pivot;
+
+  while (_pivot < Date.now() / 1000) {
+    console.log(`Processing voters from ${_pivot} to ${_pivot + batchWindow}`);
+    const votersId = await db
+      .queryAsync(
+        `SELECT voter FROM votes WHERE created >= ?
+      AND created < ?
+      ORDER BY created ASC`,
+        [_pivot, _pivot + batchWindow]
+      )
+      .map(v => v.voter);
+    const startTs = +new Date() / 1000;
+    let count = 0;
+    const newVoters = Array.from(new Set<string>(votersId.values())).filter(
+      v => !processedVoters.has(v)
+    );
+
+    console.log(`Found ${newVoters.length} new voters`);
+
+    for (const id of newVoters) {
+      processedVoters.add(id);
+
+      process.stdout.write(`\n${id} `);
+      const votes = await db.queryAsync(
+        'SELECT space, voter, COUNT(voter) as votes_count, MAX(created) as last_vote FROM votes WHERE voter = ? AND created > ? AND created <= ? GROUP BY space',
+        [id, _pivot, _pivot + batchWindow]
+      );
+
+      votes.forEach(async vote => {
+        await db.queryAsync(
+          `
+            INSERT INTO leaderboard (vote_count, last_vote, user, space)
+            VALUES(?, ?, ?, ?)
+            ON DUPLICATE KEY UPDATE vote_count = ?, last_vote = ?
+            `,
+          [vote.votes_count, vote.last_vote, id, vote.space, vote.votes_count, vote.last_vote]
+        );
+
+        process.stdout.write('.');
+      });
+
+      count += 1;
+    }
+
+    _pivot = _pivot + batchWindow;
+    console.log(
+      `\nProcessed ${count} voters (${Math.round(count / (+new Date() / 1000 - startTs))} voters/s)`
+    );
+  }
+
+  console.log(`\nProcessed ${processedVoters.size} voters in total`);
+}
+
+(async () => {
+  try {
+    await main();
+    process.exit(0);
+  } catch (e) {
+    console.error(e);
+    process.exit(1);
+  }
+})();

src/helpers/actions.ts

@@ -1,5 +1,7 @@
+import snapshot from '@snapshot-labs/snapshot.js';
 import db from './mysql';
 import { jsonParse } from './utils';
+import { NETWORK_WHITELIST, defaultNetwork } from '../writer/follow';
 
 export async function addOrUpdateSpace(space: string, settings: any) {
   if (!settings?.name) return false;
@@ -37,7 +39,16 @@ export async function getProposal(space, id) {
   return proposal;
 }
 
-export async function getSpace(id: string, includeDeleted = false) {
+export async function getSpace(id: string, includeDeleted = false, network = defaultNetwork) {
+  if (NETWORK_WHITELIST.includes(network) && network !== defaultNetwork) {
+    const spaceExist = await sxSpaceExists(id);
+    if (!spaceExist) return false;
+
+    return {
+      network: 0
+    };
+  }
+
   const query = `SELECT settings, deleted, flagged, verified, turbo, hibernated FROM spaces WHERE id = ? AND deleted in (?) LIMIT 1`;
   const spaces = await db.queryAsync(query, [id, includeDeleted ? [0, 1] : [0]]);
 
@@ -53,6 +64,21 @@ export async function getSpace(id: string, includeDeleted = false) {
   };
 }
 
+export async function sxSpaceExists(spaceId: string): Promise<boolean> {
+  const { space } = await snapshot.utils.subgraphRequest(
+    'https://api.studio.thegraph.com/query/23545/sx/version/latest',
+    {
+      space: {
+        __args: {
+          id: spaceId
+        },
+        id: true
+      }
+    }
+  );
+  return !!space?.id;
+}
+
 export function refreshProposalsCount(spaces?: string[], users?: string[]) {
   const whereFilters = ['spaces.deleted = 0'];
   const params: string[][] = [];

src/helpers/alias.ts

@@ -1,7 +1,10 @@
 import db from './mysql';
 
-export async function isValidAlias(from, alias): Promise<boolean> {
-  const query = 'SELECT * FROM aliases WHERE address = ? AND alias = ? LIMIT 1';
-  const results = await db.queryAsync(query, [from, alias]);
+export async function isValidAlias(address: string, alias: string): Promise<boolean> {
+  const thirtyDaysAgo = Math.floor(new Date().getTime() / 1000) - 30 * 24 * 60 * 60;
+
+  const query =
+    'SELECT address, alias FROM aliases WHERE address = ? AND alias = ? AND created > ? LIMIT 1';
+  const results = await db.queryAsync(query, [address, alias, thirtyDaysAgo]);
   return !!results[0];
 }

src/helpers/envelope.json

@@ -38,7 +38,8 @@
                   "type": "number"
                 },
                 "space": {
-                  "type": "string"
+                  "type": "string",
+                  "pattern": "^[^ ]+$"
                 }
               },
               "required": [

src/helpers/utils.ts

@@ -6,6 +6,7 @@ import { Response } from 'express';
 import fetch from 'node-fetch';
 import { URL } from 'url';
 import snapshot from '@snapshot-labs/snapshot.js';
+import { capture } from '@snapshot-labs/snapshot-sentry';
 import { BigNumber } from '@ethersproject/bignumber';
 
 export const DEFAULT_NETWORK = process.env.DEFAULT_NETWORK ?? '1';
@@ -71,7 +72,13 @@ export function hasStrategyOverride(strategies: any[]) {
     '"api-v2-override"'
   ];
   const strategiesStr = JSON.stringify(strategies).toLowerCase();
-  return keywords.some(keyword => strategiesStr.includes(`"name":${keyword}`));
+  if (keywords.some(keyword => strategiesStr.includes(`"name":${keyword}`))) return true;
+  // Check for split-delegation with delegationOverride
+  const splitDelegation = strategies.filter(strategy => strategy.name === 'split-delegation');
+  return (
+    splitDelegation.length > 0 &&
+    splitDelegation.some(strategy => strategy.params?.delegationOverride)
+  );
 }
 
 export function validateChoices({ type, choices }): boolean {
@@ -181,3 +188,9 @@ export const getQuorum = async (options: any, network: string, blockTag: number)
       throw new Error(`Unsupported quorum strategy: ${strategy}`);
   }
 };
+
+export function captureError(e: any, context?: any, ignoredErrorCodes?: number[]) {
+  if (ignoredErrorCodes?.includes(e.code)) return;
+
+  capture(e, context);
+}

src/ingestor.ts

@@ -49,6 +49,9 @@ export default async function ingestor(req) {
     if (message.timestamp > overTs || message.timestamp < underTs)
       return Promise.reject('wrong timestamp');
 
+    if (message.proposal && message.proposal.includes(' '))
+      return Promise.reject('proposal cannot contain whitespace');
+
     if (domain.name !== NAME || domain.version !== VERSION) return Promise.reject('wrong domain');
 
     // Ignore EIP712Domain type, it's not used
@@ -60,7 +63,8 @@ export default async function ingestor(req) {
 
     if (!['settings', 'alias', 'profile'].includes(type)) {
       if (!message.space) return Promise.reject('unknown space');
-      const space = await getSpace(message.space);
+
+      const space = await getSpace(message.space, false, message.network);
       if (!space) return Promise.reject('unknown space');
       network = space.network;
     }

src/writer/delete-proposal.ts

@@ -1,4 +1,4 @@
-import { getProposal, getSpace, refreshVotesCount } from '../helpers/actions';
+import { getProposal, getSpace } from '../helpers/actions';
 import { jsonParse } from '../helpers/utils';
 import db from '../helpers/mysql';
 
@@ -21,19 +21,32 @@ export async function verify(body): Promise<any> {
 export async function action(body): Promise<void> {
   const msg = jsonParse(body.msg);
   const proposal = await getProposal(msg.space, msg.payload.proposal);
+  const voters = await db.queryAsync(`SELECT voter FROM votes WHERE proposal = ?`, [
+    msg.payload.proposal
+  ]);
   const id = msg.payload.proposal;
 
-  await db.queryAsync(
-    `
+  let queries = `
     DELETE FROM proposals WHERE id = ? LIMIT 1;
     DELETE FROM votes WHERE proposal = ?;
     UPDATE leaderboard
       SET proposal_count = GREATEST(proposal_count - 1, 0)
       WHERE user = ? AND space = ?
       LIMIT 1;
-  `,
-    [id, id, proposal.author, msg.space]
-  );
+  `;
 
-  await refreshVotesCount([msg.space]);
+  const parameters = [id, id, proposal.author, msg.space];
+
+  if (voters.length > 0) {
+    queries += `
+    UPDATE leaderboard SET vote_count = GREATEST(vote_count - 1, 0)
+      WHERE user IN (?) AND space = ?;
+  `;
+    parameters.push(
+      voters.map(voter => voter.voter),
+      msg.space
+    );
+  }
+
+  await db.queryAsync(queries, parameters);
 }

src/writer/follow.ts

@@ -2,7 +2,8 @@ import { FOLLOWS_LIMIT_PER_USER } from '../helpers/limits';
 import db from '../helpers/mysql';
 
 const MAINNET_NETWORK_WHITELIST = ['s', 'eth', 'matic', 'arb1', 'oeth', 'sn'];
-const TESTNET_NETWORK_WHITELIST = ['s-tn', 'gor', 'sep', 'linea-testnet', 'sn-tn', 'sn-sep'];
+const TESTNET_NETWORK_WHITELIST = ['s-tn', 'sep', 'linea-testnet', 'sn-sep'];
+export const NETWORK_WHITELIST = [...MAINNET_NETWORK_WHITELIST, ...TESTNET_NETWORK_WHITELIST];
 
 export const getFollowsCount = async (follower: string): Promise<number> => {
   const query = `SELECT COUNT(*) AS count FROM follows WHERE follower = ?`;

src/writer/proposal.ts

@@ -1,7 +1,7 @@
 import snapshot from '@snapshot-labs/snapshot.js';
 import networks from '@snapshot-labs/snapshot.js/src/networks.json';
 import kebabCase from 'lodash/kebabCase';
-import { getQuorum, jsonParse, validateChoices } from '../helpers/utils';
+import { captureError, getQuorum, jsonParse, validateChoices } from '../helpers/utils';
 import db from '../helpers/mysql';
 import { getSpace } from '../helpers/actions';
 import log from '../helpers/log';
@@ -151,8 +151,8 @@ export async function verify(body): Promise<any> {
       }
 
       if (!isValid) return Promise.reject('validation failed');
-    } catch (e) {
-      capture(e, { space: msg.space, address: body.address });
+    } catch (e: any) {
+      captureError(e, { space: msg.space, address: body.address }, [504]);
       log.warn(
         `[writer] Failed to check proposal validation, ${msg.space}, ${
           body.address

src/writer/vote.ts

@@ -1,11 +1,10 @@
 import snapshot from '@snapshot-labs/snapshot.js';
 import kebabCase from 'lodash/kebabCase';
-import { hasStrategyOverride, jsonParse } from '../helpers/utils';
+import { captureError, hasStrategyOverride, jsonParse } from '../helpers/utils';
 import { getProposal } from '../helpers/actions';
 import db from '../helpers/mysql';
 import { updateProposalAndVotes } from '../scores';
 import log from '../helpers/log';
-import { capture } from '@snapshot-labs/snapshot-sentry';
 
 const scoreAPIUrl = process.env.SCORE_API_URL || 'https://score.snapshot.org';
 
@@ -66,7 +65,7 @@ export async function verify(body): Promise<any> {
       );
       if (!validate) return Promise.reject('failed vote validation');
     } catch (e) {
-      capture(e, { contexts: { input: { space: msg.space, address: body.address } } });
+      captureError(e, { contexts: { input: { space: msg.space, address: body.address } } }, [504]);
       log.warn(
         `[writer] Failed to check vote validation, ${msg.space}, ${body.address}, ${JSON.stringify(
           e
@@ -88,8 +87,8 @@ export async function verify(body): Promise<any> {
       { url: scoreAPIUrl }
     );
     if (vp.vp === 0) return Promise.reject('no voting power');
-  } catch (e) {
-    capture(e, { contexts: { input: { space: msg.space, address: body.address } } });
+  } catch (e: any) {
+    captureError(e, { contexts: { input: { space: msg.space, address: body.address } } }, [504]);
     log.warn(
       `[writer] Failed to check voting power (vote), ${msg.space}, ${body.address}, ${
         proposal.snapshot
@@ -194,8 +193,8 @@ export async function action(body, ipfs, receipt, id, context): Promise<void> {
   try {
     const result = await updateProposalAndVotes(proposalId);
     if (!result) log.warn(`[writer] updateProposalAndVotes() false, ${proposalId}`);
-  } catch (e) {
-    capture(e, { contexts: { input: { space: msg.space, id: proposalId } } });
+  } catch (e: any) {
+    captureError(e, { contexts: { input: { space: msg.space, id: proposalId } } }, [504]);
     log.warn(`[writer] updateProposalAndVotes() failed, ${msg.space}, ${proposalId}`);
   }
 }

test/fixtures/space.ts

@@ -35,6 +35,24 @@ export const spacesSqlFixtures: Record<string, any>[] = [
       network: '1',
       strategies: [{ name: 'basic' }]
     }
+  },
+  {
+    id: 'test-deleted.eth',
+    name: 'Test deleted space',
+    verified: 0,
+    flagged: 0,
+    deleted: 1,
+    hibernated: 0,
+    turbo: 0,
+    created: 1649844547,
+    updated: 1649844547,
+    settings: {
+      name: 'Test deleted space',
+      admins: ['0x87D68ecFBcF53c857ABf494728Cf3DE1016b27B0'],
+      symbol: 'TEST2',
+      network: '1',
+      strategies: [{ name: 'basic' }]
+    }
   }
 ];