wdb: use layout back up instead of wdb layout. Generate migration data.

lib/wallet/migrations.js

@@ -27,7 +27,6 @@ const {
   oldLayout
 } = require('../migrations/migrator');
 const layouts = require('./layout');
-const layout = layouts.wdb;
 
 /**
  * Switch to new migrations layout.
@@ -341,6 +340,7 @@ class MigrateAccountLookahead extends AbstractMigration {
     this.logger = options.logger.context('account-lookahead-migration');
     this.db = options.db;
     this.ldb = options.ldb;
+    this.layout = MigrateAccountLookahead.layout();
   }
 
   /**
@@ -358,10 +358,11 @@ class MigrateAccountLookahead extends AbstractMigration {
    */
 
   async migrate(b) {
+    const wlayout = this.layout.wdb;
     const wids = await this.ldb.keys({
-      gte: layout.W.min(),
-      lte: layout.W.max(),
-      parse: key => layout.W.decode(key)[0]
+      gte: wlayout.W.min(),
+      lte: wlayout.W.max(),
+      parse: key => wlayout.W.decode(key)[0]
     });
 
     for (const wid of wids)
@@ -371,14 +372,15 @@ class MigrateAccountLookahead extends AbstractMigration {
   }
 
   async migrateWallet(b, wid) {
+    const wlayout = this.layout.wdb;
     const accounts = await this.ldb.keys({
-      gte: layout.a.min(wid),
-      lte: layout.a.max(wid),
-      parse: key => layout.a.decode(key)[1]
+      gte: wlayout.a.min(wid),
+      lte: wlayout.a.max(wid),
+      parse: key => wlayout.a.decode(key)[1]
     });
 
     for (const accID of accounts) {
-      const key = layout.a.encode(wid, accID);
+      const key = wlayout.a.encode(wid, accID);
       const rawAccount = await this.ldb.get(key);
       const newRaw = this.accountEncode(rawAccount);
       b.put(key, newRaw);
@@ -408,6 +410,18 @@ class MigrateAccountLookahead extends AbstractMigration {
       description: 'Account lookahead now supports up to 2^32 - 1'
     };
   }
+
+  static layout() {
+    return {
+      wdb: {
+        // W[wid] -> wallet id
+        W: bdb.key('W', ['uint32']),
+
+        // a[wid][index] -> account
+        a: bdb.key('a', ['uint32', 'uint32'])
+      }
+    };
+  }
 }
 
 class MigrateTXDBBalances extends AbstractMigration {

test/data/migrations/wallet-2-account-lookahead-gen.js

@@ -0,0 +1,87 @@
+'use strict';
+
+/**
+ * This migration is for v4 -> v5.
+ */
+
+const bdb = require('bdb');
+const Network = require('../../../lib/protocol/network');
+const WalletDB = require('../../../lib/wallet/walletdb');
+const wutils = require('../../util/wallet');
+const mutils = require('../../util/migrations');
+
+const layout = {
+  wdb: {
+    // W[wid] -> wallet id
+    W: bdb.key('W', ['uint32']),
+
+    // a[wid][index] -> account
+    a: bdb.key('a', ['uint32', 'uint32'])
+  }
+};
+
+const NETWORK = Network.get('regtest');
+const LOOKAHEAD1 = 10;
+const LOOKAHEAD2 = 200;
+
+const wallet1priv = 'rprvKE8qsHtkmUxUSPQdn2sFKFUcKyUQz9pKQhxjEWecnXg9hgJMsmJXcw'
+  + 'J77SqmHT1R6mcuNqVPzgT2EoGStsXaUN92VJKhQWUB6uZdL8gAZvez';
+const wallet2priv = 'rprvKE8qsHtkmUxUSR4jE7Lti9XV77hv7xxacAShw5MvxY6RfsAYVeB1WL'
+  + 'WtjiebDmqTruVJxmMeQUMkk61e83WDZbZidDnNPhHyQpeEwxjuSZuG';
+const rpub = 'rpubKBAPj83PkcGYSz3GR9xfzXBfTFTPtERb2x7fKvbikH9utGMvr6HDd4sTxt5zo'
+  + 'arw4bzzgH1VDzBUoX9fotzmPrrngFyLMz3ozAi1ozAbJjSY';
+
+(async () => {
+  const wdb = new WalletDB({
+    network: NETWORK,
+    memory: true
+  });
+
+  await wdb.open();
+
+  const wallet1 = await wdb.create({
+    id: 'wallet1',
+    master: wallet1priv,
+    lookahead: LOOKAHEAD1
+  });
+
+  await wallet1.createAccount({
+    name: 'alt',
+    lookahead: LOOKAHEAD2
+  });
+
+  const wallet2 = await wdb.create({
+    id: 'wallet2',
+    master: wallet2priv,
+    lookahead: LOOKAHEAD2,
+    m: 1,
+    n: 2
+  });
+
+  await wallet2.addSharedKey(0, rpub);
+
+  for (let i = 0; i < 100; i++) {
+    const entry = wutils.nextEntry(wdb);
+    await wdb.addBlock(entry, []);
+  }
+
+  console.log(JSON.stringify({
+    data: await getMigrationDump(wdb)
+  }, null, 2));
+
+  await wdb.close();
+})().catch((e) => {
+    console.error(e);
+});
+
+async function getMigrationDump(wdb) {
+  const prefixes = [];
+
+  // skip primary wallet.
+  for (let i = 1; i < 3; i++) {
+    prefixes.push(layout.wdb.W.encode(i).toString('hex'));
+    prefixes.push(layout.wdb.a.encode(i, 0).slice(0, 5).toString('hex'));
+  }
+
+  return await mutils.dumpWDB(wdb, prefixes);
+}

test/data/migrations/wallet-2-account-lookahead-gen.json

@@ -0,0 +1,17 @@
+{
+  "before": {
+    "5700000001": "0777616c6c657431",
+    "5700000002": "0777616c6c657432",
+    "610000000100000000": "0100010101000000010000000a03875c90a480000000fdc9691244b5dc0636ce0eabd905556c333e208b2815e463a6e65db29833c15d024a4400453207366ed8567bf78fdda5dccc1a0f8b1398f9925b744cc563575a7400",
+    "610000000100000001": "010001010100000001000000c803875c90a480000001fc7d26586b3e6e082df9d01051258b8809960d5434d1abfe088ab507a0cfbbf003f44993e0ed4638fb5d32a272cca586ffe85040755d95568e01ca5c4845e27c9e00",
+    "610000000200000000": "010101020100000001000000c8034a0c1e1180000000bda10baaccd9189fb3a56e2950ad198c45611cd3dbf0f154f699fa1c9ddc4b89022e05837732c62feb49896bc8b3cd8fd2f1b9b6bc415074ba31d858a810bbb29401032ac4ec93800000008921a0a732ac09e78b4f51464880ca9fc3e09a129564c0d8aef9fb1fd2ff175302f2b8aebdc25d17815420662ff8e2b9e24202a9b628f07ce2671cedac5f8001e1"
+
+  },
+  "after": {
+    "5700000001": "0777616c6c657431",
+    "5700000002": "0777616c6c657432",
+    "610000000100000000": "0100010101000000010000000a00000003875c90a480000000fdc9691244b5dc0636ce0eabd905556c333e208b2815e463a6e65db29833c15d024a4400453207366ed8567bf78fdda5dccc1a0f8b1398f9925b744cc563575a7400",
+    "610000000100000001": "010001010100000001000000c800000003875c90a480000001fc7d26586b3e6e082df9d01051258b8809960d5434d1abfe088ab507a0cfbbf003f44993e0ed4638fb5d32a272cca586ffe85040755d95568e01ca5c4845e27c9e00",
+    "610000000200000000": "010101020100000001000000c8000000034a0c1e1180000000bda10baaccd9189fb3a56e2950ad198c45611cd3dbf0f154f699fa1c9ddc4b89022e05837732c62feb49896bc8b3cd8fd2f1b9b6bc415074ba31d858a810bbb29401032ac4ec93800000008921a0a732ac09e78b4f51464880ca9fc3e09a129564c0d8aef9fb1fd2ff175302f2b8aebdc25d17815420662ff8e2b9e24202a9b628f07ce2671cedac5f8001e1"
+  }
+}

test/wallet-migration-test.js

@@ -372,8 +372,87 @@ describe('Wallet Migrations', function() {
     });
   });
 
+  describe('Migrate change address (data)', function() {
+    const location = testdir('wallet-lookahead-data');
+    const migrationsBAK = WalletMigrator.migrations;
+    const data = require('./data/migrations/wallet-2-account-lookahead-gen.json');
+    const Migration = WalletMigrator.MigrateAccountLookahead;
+
+    const walletOptions = {
+      prefix: location,
+      memory: false,
+      network
+    };
+
+    let wdb, ldb;
+    before(async () => {
+      WalletMigrator.migrations = {};
+      await fs.mkdirp(location);
+
+      wdb = new WalletDB(walletOptions);
+      ldb = wdb.db;
+
+      await ldb.open();
+      await fillEntries(ldb, data.before);
+      await ldb.close();
+    });
+
+    after(async () => {
+      WalletMigrator.migrations = migrationsBAK;
+      await rimraf(location);
+    });
+
+    it('should have before entries', async () => {
+      wdb.version = 1;
+      try {
+        // We don't care that new wallet can't decode old data.
+        // It will still run migrations.
+        await wdb.open();
+      } catch (e) {
+        ;
+      }
+      await checkVersion(ldb, layouts.wdb.V.encode(), 1);
+      await checkEntries(ldb, data.before);
+      await wdb.close();
+    });
+
+    it('should enable wallet migration', () => {
+      WalletMigrator.migrations = {
+        0: Migration
+      };
+    });
+
+    it('should fail without migrate flag', async () => {
+      const expectedError = migrationError(WalletMigrator.migrations, [0],
+          wdbFlagError(0));
+
+      await assert.rejects(async () => {
+        await wdb.open();
+      }, {
+        message: expectedError
+      });
+
+      await ldb.close();
+    });
+
+    it('should migrate', async () => {
+      wdb.options.walletMigrate = 0;
+
+      try {
+        // We don't care that new wallet can't decode old data.
+        // It will still run migrations.
+        await wdb.open();
+      } catch (e) {
+        ;
+      }
+      await checkVersion(ldb, layouts.wdb.V.encode(), 2);
+      await checkEntries(ldb, data.after);
+      await wdb.close();
+    });
+  });
+
   describe('Migrate account lookahead (integration)', function () {
-    const location = testdir('wallet-change');
+    const location = testdir('wallet-lookahead');
     const migrationsBAK = WalletMigrator.migrations;
     const TEST_LOOKAHEAD = 150;