From 9c554754924515b9d7cece4e113045201265efd6 Mon Sep 17 00:00:00 2001 From: Wan Qi Chen <495709+wa0x6e@users.noreply.github.com> Date: Thu, 24 Aug 2023 23:12:05 +0900 Subject: [PATCH 1/5] feat: add user_space table to collect user's votes and proposals count per space --- scripts/generate_user_space_counters.ts | 31 +++++++++++++++++++++++++ src/helpers/schema.sql | 11 +++++++++ 2 files changed, 42 insertions(+) create mode 100644 scripts/generate_user_space_counters.ts diff --git a/scripts/generate_user_space_counters.ts b/scripts/generate_user_space_counters.ts new file mode 100644 index 00000000..8e06d112 --- /dev/null +++ b/scripts/generate_user_space_counters.ts @@ -0,0 +1,31 @@ +import 'dotenv/config'; +import db from '../src/helpers/mysql'; + +// Usage: yarn ts-node scripts/generate_user_space_counters.ts +async function main() { + console.log('Inserting/Updating proposals_count'); + const proposalsCountRes = await db.queryAsync( + `INSERT INTO user_space (proposals_count, user_id, space_id) + (select * from (select count(id) as proposals_count, author, space from proposals group by author, space) as t) + ON DUPLICATE KEY UPDATE proposals_count = t.proposals_count` + ); + console.log(proposalsCountRes); + + console.log('Inserting/Updating votes_count'); + const votesCountRes = await db.queryAsync( + `INSERT INTO user_space (votes_count, user_id, space_id) + (select * from (select count(id) as votes_count, voter, space from votes group by voter, space) as t) + ON DUPLICATE KEY UPDATE votes_count = t.votes_count` + ); + console.log(votesCountRes); +} + +(async () => { + try { + await main(); + process.exit(0); + } catch (e) { + console.error(e); + process.exit(1); + } +})(); diff --git a/src/helpers/schema.sql b/src/helpers/schema.sql index eea3849a..a4c69274 100644 --- a/src/helpers/schema.sql +++ b/src/helpers/schema.sql @@ -142,3 +142,14 @@ CREATE TABLE statements ( INDEX created (created), INDEX updated (updated) ); + +CREATE TABLE user_space ( + user_id varchar(64) NOT NULL, + space_id varchar(64) NOT NULL, + votes_count smallint unsigned NOT NULL DEFAULT '0', + proposals_count smallint unsigned NOT NULL DEFAULT '0', + UNIQUE KEY user_id_space_id (user_id,space_id), + INDEX space_id (space_id), + INDEX votes_count (votes_count), + INDEX proposals_count (proposals_count) +); From 8723558e48caa0ed18ebaf3620e613a60ae6577e Mon Sep 17 00:00:00 2001 From: Wan Qi Chen <495709+wa0x6e@users.noreply.github.com> Date: Thu, 24 Aug 2023 23:38:22 +0900 Subject: [PATCH 2/5] fix: batch the votes_count computation --- scripts/generate_user_space_counters.ts | 34 ++++++++++++++++++++----- 1 file changed, 28 insertions(+), 6 deletions(-) diff --git a/scripts/generate_user_space_counters.ts b/scripts/generate_user_space_counters.ts index 8e06d112..ef83766f 100644 --- a/scripts/generate_user_space_counters.ts +++ b/scripts/generate_user_space_counters.ts @@ -12,12 +12,34 @@ async function main() { console.log(proposalsCountRes); console.log('Inserting/Updating votes_count'); - const votesCountRes = await db.queryAsync( - `INSERT INTO user_space (votes_count, user_id, space_id) - (select * from (select count(id) as votes_count, voter, space from votes group by voter, space) as t) - ON DUPLICATE KEY UPDATE votes_count = t.votes_count` - ); - console.log(votesCountRes); + let page = 0; + const batchSize = 1000; + const results: any[] = []; + + while (true) { + const result = await db.queryAsync('SELECT id from spaces ORDER BY RAND () LIMIT ? OFFSET ?', [ + batchSize, + page * batchSize + ]); + + if (result.length === 0) { + break; + } + + page += 1; + results.push(result); + } + + for (const index in results) { + console.log(`Processing batch ${index + 1}/${results.length}`); + const votesCountRes = await db.queryAsync( + `INSERT INTO user_space (votes_count, user_id, space_id) + (select * from (select count(id) as votes_count, voter, space from votes where space IN (?) group by voter, space) as t) + ON DUPLICATE KEY UPDATE votes_count = t.votes_count`, + [results[index].map(d => d.id)] + ); + console.log(votesCountRes); + } } (async () => { From 68d9dc5c005003efa0991987b54c12533f4b7b22 Mon Sep 17 00:00:00 2001 From: Wan Qi Chen <495709+wa0x6e@users.noreply.github.com> Date: Sun, 27 Aug 2023 00:26:15 +0900 Subject: [PATCH 3/5] fix: remove insert scripts --- scripts/generate_user_space_counters.ts | 53 ------------------------- 1 file changed, 53 deletions(-) delete mode 100644 scripts/generate_user_space_counters.ts diff --git a/scripts/generate_user_space_counters.ts b/scripts/generate_user_space_counters.ts deleted file mode 100644 index ef83766f..00000000 --- a/scripts/generate_user_space_counters.ts +++ /dev/null @@ -1,53 +0,0 @@ -import 'dotenv/config'; -import db from '../src/helpers/mysql'; - -// Usage: yarn ts-node scripts/generate_user_space_counters.ts -async function main() { - console.log('Inserting/Updating proposals_count'); - const proposalsCountRes = await db.queryAsync( - `INSERT INTO user_space (proposals_count, user_id, space_id) - (select * from (select count(id) as proposals_count, author, space from proposals group by author, space) as t) - ON DUPLICATE KEY UPDATE proposals_count = t.proposals_count` - ); - console.log(proposalsCountRes); - - console.log('Inserting/Updating votes_count'); - let page = 0; - const batchSize = 1000; - const results: any[] = []; - - while (true) { - const result = await db.queryAsync('SELECT id from spaces ORDER BY RAND () LIMIT ? OFFSET ?', [ - batchSize, - page * batchSize - ]); - - if (result.length === 0) { - break; - } - - page += 1; - results.push(result); - } - - for (const index in results) { - console.log(`Processing batch ${index + 1}/${results.length}`); - const votesCountRes = await db.queryAsync( - `INSERT INTO user_space (votes_count, user_id, space_id) - (select * from (select count(id) as votes_count, voter, space from votes where space IN (?) group by voter, space) as t) - ON DUPLICATE KEY UPDATE votes_count = t.votes_count`, - [results[index].map(d => d.id)] - ); - console.log(votesCountRes); - } -} - -(async () => { - try { - await main(); - process.exit(0); - } catch (e) { - console.error(e); - process.exit(1); - } -})(); From 10debae038e1f48ee2ec96a3a8bb9c2689de98c4 Mon Sep 17 00:00:00 2001 From: Wan Qi Chen <495709+wa0x6e@users.noreply.github.com> Date: Sun, 27 Aug 2023 22:54:49 +0900 Subject: [PATCH 4/5] fix: rename table to a more descriptive name --- src/helpers/schema.sql | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/src/helpers/schema.sql b/src/helpers/schema.sql index a4c69274..203f22bb 100644 --- a/src/helpers/schema.sql +++ b/src/helpers/schema.sql @@ -143,7 +143,7 @@ CREATE TABLE statements ( INDEX updated (updated) ); -CREATE TABLE user_space ( +CREATE TABLE user_space_activities ( user_id varchar(64) NOT NULL, space_id varchar(64) NOT NULL, votes_count smallint unsigned NOT NULL DEFAULT '0', From fe3711bb54896a80fa5a8ddeed7e24f590912ff1 Mon Sep 17 00:00:00 2001 From: Wan Qi Chen <495709+wa0x6e@users.noreply.github.com> Date: Thu, 31 Aug 2023 23:50:43 +0900 Subject: [PATCH 5/5] chore: fix case --- src/helpers/schema.sql | 8 ++++---- 1 file changed, 4 insertions(+), 4 deletions(-) diff --git a/src/helpers/schema.sql b/src/helpers/schema.sql index 203f22bb..8a6bf58c 100644 --- a/src/helpers/schema.sql +++ b/src/helpers/schema.sql @@ -144,10 +144,10 @@ CREATE TABLE statements ( ); CREATE TABLE user_space_activities ( - user_id varchar(64) NOT NULL, - space_id varchar(64) NOT NULL, - votes_count smallint unsigned NOT NULL DEFAULT '0', - proposals_count smallint unsigned NOT NULL DEFAULT '0', + user_id VARCHAR(64) NOT NULL, + space_id VARCHAR(64) NOT NULL, + votes_count SMALLINT UNSIGNED NOT NULL DEFAULT '0', + proposals_count SMALLINT UNSIGNED NOT NULL DEFAULT '0', UNIQUE KEY user_id_space_id (user_id,space_id), INDEX space_id (space_id), INDEX votes_count (votes_count),