From 9d860e78c1d8df78a942054e9e3ef42afe1236b1 Mon Sep 17 00:00:00 2001 From: Nodari Chkuaselidze Date: Mon, 18 Mar 2024 14:30:35 +0400 Subject: [PATCH 1/2] walletdb: migration to v3. --- lib/wallet/migrations.js | 72 ++++++++++- lib/wallet/walletdb.js | 2 +- test/wallet-migration-test.js | 225 +++++++++++++++++++++++++++++++++- 3 files changed, 294 insertions(+), 5 deletions(-) diff --git a/lib/wallet/migrations.js b/lib/wallet/migrations.js index 26eef7f95..d359d7bce 100644 --- a/lib/wallet/migrations.js +++ b/lib/wallet/migrations.js @@ -74,7 +74,7 @@ class MigrateMigrations extends AbstractMigration { /** * Run change address migration. - * Applies to WalletDB v0 + * Applies to WalletDB v1 */ class MigrateChangeAddress extends AbstractMigration { @@ -160,7 +160,7 @@ class MigrateChangeAddress extends AbstractMigration { /** * Migrate account for new lookahead entry. - * Applies to WalletDB v1 + * Applies to WalletDB v1 -> v2 */ class MigrateAccountLookahead extends AbstractMigration { @@ -245,6 +245,11 @@ class MigrateAccountLookahead extends AbstractMigration { } } +/** + * Recalculate TXDB balances. + * Applies to WalletDB v2 + */ + class MigrateTXDBBalances extends AbstractMigration { /** * Create TXDB Balance migration object. @@ -289,6 +294,65 @@ class MigrateTXDBBalances extends AbstractMigration { } } +/** + * Migrate WDB to v3. Drop TXDB and migrate to new WDB layout. + * Applies to WalletDB v2 + */ + +class MigrateWDBv3 extends AbstractMigration { + constructor(options) { + super(options); + + this.options = options; + this.logger = options.logger.context('wdb-migrate-v3'); + this.db = options.db; + this.ldb = options.ldb; + } + + /** + * We always migrate. + * @returns {Promise} + */ + + async check() { + return types.MIGRATE; + } + + /** + * Actual migration + * @param {Batch} b + * @param {WalletMigrationResult} pending + * @returns {Promise} + */ + + async migrate(b, pending) { + const removeRange = (opt) => { + return this.ldb.iterator(opt).each(key => b.del(key)); + }; + + // Remove mappings and TXDB Entries. + await this.db.deepClean(); + + // Remove the sync progress. + await removeRange({ + gte: layout.h.min(), + lte: layout.h.max() + }); + + await this.db.saveGenesis(); + + // Now rewrite genesis and version. + await this.db.writeVersion(b, 3); + } + + static info() { + return { + name: 'Migrate WDB to v3', + description: 'Drop TXDB and migrate to new WDB layout' + }; + } +} + /** * Wallet migration results. * @alias module:blockchain.WalletMigrationResult @@ -423,7 +487,8 @@ exports.migrations = { 0: MigrateMigrations, 1: MigrateChangeAddress, 2: MigrateAccountLookahead, - 3: MigrateTXDBBalances + 3: MigrateTXDBBalances, + 4: MigrateWDBv3 }; // Expose migrations @@ -431,5 +496,6 @@ exports.MigrateChangeAddress = MigrateChangeAddress; exports.MigrateMigrations = MigrateMigrations; exports.MigrateAccountLookahead = MigrateAccountLookahead; exports.MigrateTXDBBalances = MigrateTXDBBalances; +exports.MigrateWDBv3 = MigrateWDBv3; module.exports = exports; diff --git a/lib/wallet/walletdb.js b/lib/wallet/walletdb.js index 790087b41..efc7e6802 100644 --- a/lib/wallet/walletdb.js +++ b/lib/wallet/walletdb.js @@ -62,7 +62,7 @@ class WalletDB extends EventEmitter { this.feeRate = this.options.feeRate; this.db = bdb.create(this.options); this.name = 'wallet'; - this.version = 2; + this.version = 3; // chain state. this.hasStateCache = false; diff --git a/test/wallet-migration-test.js b/test/wallet-migration-test.js index 8dfc4371e..41d2f77bd 100644 --- a/test/wallet-migration-test.js +++ b/test/wallet-migration-test.js @@ -6,6 +6,7 @@ const random = require('bcrypto/lib/random'); const Network = require('../lib/protocol/network'); const rules = require('../lib/covenants/rules'); const Coin = require('../lib/primitives/coin'); +const MTX = require('../lib/primitives/mtx'); const WalletDB = require('../lib/wallet/walletdb'); const layouts = require('../lib/wallet/layout'); const TXDB = require('../lib/wallet/txdb'); @@ -20,6 +21,7 @@ const { } = require('../lib/migrations/migrator'); const {migrationError} = require('./util/migrations'); const {rimraf, testdir} = require('./util/common'); +const wutil = require('./util/wallet'); const NETWORK = 'regtest'; const network = Network.get(NETWORK); @@ -703,7 +705,7 @@ describe('Wallet Migrations', function() { }); }); - describe('Migrate txdb (integration)', function() { + describe('Migrate txdb balances (integration)', function() { const location = testdir('walet-txdb-refresh'); const migrationsBAK = WalletMigrator.migrations; @@ -949,6 +951,227 @@ describe('Wallet Migrations', function() { await walletDB.close(); }); }); + + describe('Migrate WDB to v3', function() { + const location = testdir('wallet-migrate-v3'); + const migrationsBAK = WalletMigrator.migrations; + + const walletOptions = { + prefix: location, + memory: false, + network + }; + + const NAME = rules.grindName(10, 1, network); + const wallets = new Set(); + const accounts = new Map(); + const blocks = []; + + let walletDB, ldb; + before(async () => { + WalletMigrator.migrations = {}; + await fs.mkdirp(location); + }); + + after(async () => { + WalletMigrator.migrations = migrationsBAK; + await rimraf(location); + }); + + beforeEach(async () => { + walletDB = new WalletDB(walletOptions); + ldb = walletDB.db; + }); + + afterEach(async () => { + if (ldb.opened) + await ldb.close(); + }); + + it('should have db entries', async () => { + // Faking old entries is not necessary, as the migration + // will reset most of the db entries. + await walletDB.open(); + + const wallet1 = walletDB.primary; + const wallet2 = await walletDB.create({ + id: 'wallet2' + }); + + wallets.add('primary'); + wallets.add('wallet2'); + + await wallet1.createAccount({ name: 'account1' }); + await wallet2.createAccount({ name: 'account2' }); + + // write older version. + const b = ldb.batch(); + writeVersion(b, 'wallet', 2); + await b.write(); + + // fund first wallet. + const derAddr = await wallet1.receiveAddress(); + const mtx = new MTX(); + mtx.addInput(wutil.dummyInput()); + for (let i = 0; i < 20; i++) + mtx.addOutput(derAddr, 5e6); + + const addrs = []; + + addrs.push((await wallet1.receiveAddress(0))); + addrs.push((await wallet1.receiveAddress(1))); + addrs.push((await wallet2.receiveAddress(0))); + addrs.push((await wallet2.receiveAddress(1))); + + const block1 = wutil.fakeBlock(1); + await walletDB.addBlock(block1, [mtx.toTX()]); + blocks.push([block1, [mtx.toTX()]]); + + const fundMTX = await wallet1.createTX({ + outputs: addrs.map(addr => ({ address: addr, value: 3e6 })) + }); + await wallet1.sign(fundMTX); + + const block2 = wutil.fakeBlock(2); + await walletDB.addBlock(block2, [fundMTX.toTX()]); + blocks.push([block2, [fundMTX.toTX()]]); + + // create open + const openMTX = await wallet1.createOpen(NAME); + await wallet1.sign(openMTX); + const block3 = wutil.fakeBlock(3); + await walletDB.addBlock(block3, [openMTX.toTX()]); + blocks.push([block3, [openMTX.toTX()]]); + + for (let i = 0; i < network.names.treeInterval + 1; i++) { + const block = wutil.fakeBlock(walletDB.state.height + 1); + await walletDB.addBlock(block, []); + blocks.push([block, []]); + } + + // create bid + const bidMTX = await wallet2.createBid(NAME, 1e6, 2e6); + await wallet2.sign(bidMTX); + const bidBlock = wutil.fakeBlock(walletDB.state.height + 1); + await walletDB.addBlock(bidBlock, [bidMTX.toTX()]); + blocks.push([bidBlock, [bidMTX.toTX()]]); + + // Collect info + const w1def = await wallet1.getAccount(0); + const w1acc = await wallet1.getAccount(1); + + const w2def = await wallet2.getAccount(0); + const w2acc = await wallet2.getAccount(1); + + accounts.set('primary', [w1def.getJSON(), w1acc.getJSON()]); + accounts.set('wallet2', [w2def.getJSON(), w2acc.getJSON()]); + + const bidsAfter = await wallet2.getBids(rules.hashName(NAME)); + assert.strictEqual(bidsAfter.length, 1); + + // blind value should have recovered. + const bid = bidsAfter[0]; + assert.strictEqual(bid.value, 1e6); + assert.strictEqual(bid.own, true); + + await walletDB.close(); + }); + + it('should enable migration to WDB v3.', async () => { + WalletMigrator.migrations = { + 0: WalletMigrator.MigrateWDBv3 + }; + }); + + it('should migrate', async () => { + walletDB.options.walletMigrate = 0; + + await walletDB.open(); + + // chain sync was reset. + assert.strictEqual(walletDB.state.height, 0); + const lastTip = blocks[blocks.length - 1][0].height; + for (let i = 0; i < lastTip; i++) { + const block = await walletDB.getBlock(i); + assert.strictEqual(block, null); + } + + // Wallet information should be the same. + const wids = await walletDB.getWallets(); + assert.strictEqual(wallets.size, wids.length); + + for (const wid of wids) { + assert(wallets.has(wid)); + const wallet = await walletDB.get(wid); + const accs = await wallet.getAccounts(); + + const testAccounts = accounts.get(wid); + assert.strictEqual(accs.length, testAccounts.length); + + for (const testAcc of testAccounts) { + const name = testAcc.name; + const acc = await wallet.getAccount(name); + + assert.deepStrictEqual(acc.getJSON(), testAcc); + } + } + + const assertBalanceEmpty = (balance) => { + assert.strictEqual(balance.tx, 0); + assert.strictEqual(balance.coin, 0); + assert.strictEqual(balance.confirmed, 0); + assert.strictEqual(balance.unconfirmed, 0); + assert.strictEqual(balance.clocked, 0); + assert.strictEqual(balance.ulocked, 0); + }; + + // balances should be reset. + for (const wid of wids) { + const wallet = await walletDB.get(wid); + const balance = await wallet.getBalance(); + assertBalanceEmpty(balance); + + for (const name of await wallet.getAccounts()) { + const balance = await wallet.getBalance(name); + assertBalanceEmpty(balance); + } + } + + const wallet1 = walletDB.primary; + const wallet2 = await walletDB.get('wallet2'); + + for (const wallet of [wallet1, wallet2]) { + assert.strictEqual((await wallet.getHistory()).length, 0); + assert.strictEqual((await wallet.getCoins()).length, 0); + assert.strictEqual((await wallet.getCredits()).length, 0); + assert.strictEqual((await wallet.getPending()).length, 0); + assert.strictEqual((await wallet.getRange(-1, { + start: 0 + })).length, 0); + assert.strictEqual((await wallet.getBlocks()).length, 0); + } + + // no bids + const nameHash = rules.hashName(NAME); + const bids = await wallet2.getBids(nameHash); + assert.strictEqual(bids.length, 0); + + // resend blocks + for (const [block, txs] of blocks) { + await walletDB.addBlock(block, txs); + } + + const bidsAfter = await wallet2.getBids(nameHash); + assert.strictEqual(bidsAfter.length, 1); + + // blind value should have recovered. + const bid = bidsAfter[0]; + assert.strictEqual(bid.value, 1e6); + assert.strictEqual(bid.own, true); + + await walletDB.close(); + }); + }); }); function writeVersion(b, name, version) { From 8548b0e886dbdca4c494e03e70a53807144c28ef Mon Sep 17 00:00:00 2001 From: Nodari Chkuaselidze Date: Sat, 30 Mar 2024 19:38:58 +0400 Subject: [PATCH 2/2] test: future proof migration test. --- test/wallet-migration-test.js | 1 + 1 file changed, 1 insertion(+) diff --git a/test/wallet-migration-test.js b/test/wallet-migration-test.js index 41d2f77bd..f8c4b7b88 100644 --- a/test/wallet-migration-test.js +++ b/test/wallet-migration-test.js @@ -1086,6 +1086,7 @@ describe('Wallet Migrations', function() { it('should migrate', async () => { walletDB.options.walletMigrate = 0; + walletDB.version = 3; await walletDB.open(); // chain sync was reset.