Replace dumb JSON encoding by more sophisticated encoding.

The new encoding can remember types and saves space when storing Uint8Arrays.
Also increases the version number and simplifies access to JDB objects in tests.

gulpfile.js

@@ -35,6 +35,8 @@ const sources = {
     generic: [
         './src/main/generic/utils/ArrayUtils.js',
         './src/main/generic/utils/BTree.js',
+        './src/main/generic/utils/BufferUtils.js',
+        './src/main/generic/utils/JSONUtils.js',
         './src/main/generic/utils/Log.js',
         './src/main/generic/utils/LRUMap.js',
         './src/main/generic/utils/ObjectUtils.js',
@@ -211,7 +213,7 @@ gulp.task('build-node-istanbul', ['build-istanbul'], function () {
 
 gulp.task('test', ['watch'], function () {
     gulp.run(jasmine({
-        files: ['dist/web.js'].concat(sources.test.generic).concat(sources.test.browser)
+        files: ['./src/test/platform/browser/spec.js', 'dist/web.js'].concat(sources.test.generic).concat(sources.test.browser)
     }));
 });
 

karma.conf.js

@@ -15,6 +15,7 @@ module.exports = function (config) {
 
         // list of files / patterns to load in the browser
         files: [
+            'src/test/platform/browser/spec.js',
             'dist/web.js',
             'src/test/**/DummyBackend.js',
             'src/test/generic/TestCodec.js',
@@ -114,10 +115,10 @@ module.exports = function (config) {
     for (const version of [57, 58, 59, 60, 61, 62]) sauceLabsConfig('Windows 10', 'chrome', `${version}.0`, 'Chrome', version);
 
     if (process.env.USE_BABEL) {
-        configuration.files[0] = 'dist/web-babel.js';
+        configuration.files[1] = 'dist/web-babel.js';
     }
     if (process.env.USE_ISTANBUL) {
-        configuration.files[0] = 'dist/web-istanbul.js';
+        configuration.files[1] = 'dist/web-istanbul.js';
         configuration.reporters.push('coverage');
         configuration.coverageReporter = {
             reporters: [

package.json

@@ -1,6 +1,6 @@
 {
   "name": "jungle-db",
-  "version": "0.3.6",
+  "version": "0.3.7",
   "main": "dist/node.js",
   "private": true,
   "scripts": {
@@ -20,6 +20,8 @@
     "node": ">=7.9.0"
   },
   "dependencies": {
+    "atob": "^2.0.3",
+    "btoa": "^1.1.2",
     "fs": "^0.0.1-security",
     "level-sublevel": "^6.6.1",
     "leveldown": "^1.6.0",

src/loader/nodejs/index.prefix.js

@@ -1,5 +1,8 @@
 module.exports = {};
 
+const atob = require('atob');
+const btoa = require('btoa');
+
 const Class = {
     register: clazz => {
         module.exports[clazz.prototype.constructor.name] = clazz;

src/main/generic/utils/BufferUtils.js

@@ -0,0 +1,19 @@
+class BufferUtils {
+     /**
+     * @param {*} buffer
+     * @return {string}
+     */
+    static toBase64(buffer) {
+        return btoa(String.fromCharCode(...new Uint8Array(buffer)));
+    }
+
+    /**
+     * @param {string} base64
+     * @return {SerialBuffer}
+     */
+    static fromBase64(base64) {
+        return Uint8Array.from(atob(base64), c => c.charCodeAt(0));
+    }
+}
+
+Class.register(BufferUtils);

src/main/generic/utils/JSONUtils.js

@@ -0,0 +1,42 @@
+class JSONUtils {
+    static stringify(value) {
+        return JSON.stringify(value, JSONUtils.jsonifyType);
+    }
+
+    static parse(value) {
+        return JSON.parse(value, JSONUtils.parseType);
+    }
+
+    static parseType(key, value) {
+        if (value[JSONUtils.TYPE_SYMBOL]) {
+            switch (value[JSONUtils.TYPE_SYMBOL]) {
+                case 'Uint8Array':
+                    return BufferUtils.fromBase64(value[JSONUtils.VALUE_SYMBOL]);
+                case 'Set':
+                    return Set.from(value[JSONUtils.VALUE_SYMBOL]);
+            }
+        }
+        return value;
+    }
+
+    static jsonifyType(key, value) {
+        if (value instanceof Uint8Array) {
+            return JSONUtils.typedObject('Uint8Array', BufferUtils.toBase64(value));
+        }
+        if (value instanceof Set) {
+            return JSONUtils.typedObject('Set', Array.from(value));
+        }
+        return value;
+    }
+
+    static typedObject(type, value) {
+        const obj = {};
+        obj[JSONUtils.TYPE_SYMBOL] = type;
+        obj[JSONUtils.VALUE_SYMBOL] = value;
+        return obj;
+    }
+}
+JSONUtils.TYPE_SYMBOL = '__';
+JSONUtils.VALUE_SYMBOL = 'value';
+
+Class.register(JSONUtils);

src/main/platform/nodejs/JungleDB.js

@@ -281,16 +281,8 @@ class JungleDB {
  * @type {{encode: function(val:*):*, decode: function(val:*):*, buffer: boolean, type: string}}
  */
 JungleDB.JSON_ENCODING = {
-    encode: val => JSON.stringify(val, (k, v) => {
-        if (v instanceof Uint8Array) {
-            return Array.from(v);
-        }
-        if (v instanceof Set) {
-            return Array.from(v);
-        }
-        return v;
-    }),
-    decode: JSON.parse,
+    encode: JSONUtils.stringify,
+    decode: JSONUtils.parse,
     buffer: false,
     type: 'json'
 };

src/test/generic/CombinedTransaction.spec.js

@@ -2,10 +2,10 @@ describe('CombinedTransaction', () => {
     let backend1, backend2, objectStore1, objectStore2;
 
     beforeEach((done) => {
-        backend1 = new JDB.InMemoryBackend('');
-        backend2 = new JDB.InMemoryBackend('');
-        objectStore1 = new JDB.ObjectStore(backend1, null);
-        objectStore2 = new JDB.ObjectStore(backend2, null);
+        backend1 = new InMemoryBackend('');
+        backend2 = new InMemoryBackend('');
+        objectStore1 = new ObjectStore(backend1, null);
+        objectStore2 = new ObjectStore(backend2, null);
 
         (async function () {
             // Add 10 objects.
@@ -26,21 +26,21 @@ describe('CombinedTransaction', () => {
             let tx2 = objectStore2.transaction();
             await tx1.remove('key6');
             await tx2.remove('key6');
-            new JDB.CombinedTransaction(tx1, tx2);
+            new CombinedTransaction(tx1, tx2);
 
             expect(await tx1.commit()).toBe(true);
-            expect(await tx2.state).toBe(JDB.Transaction.STATE.COMMITTED);
+            expect(await tx2.state).toBe(Transaction.STATE.COMMITTED);
             expect(await backend1.get('key6')).toBe(undefined);
             expect(await backend2.get('key6')).toBe(undefined);
 
             tx1 = objectStore1.transaction();
             tx2 = objectStore2.transaction();
             await tx1.remove('key7');
             await tx2.remove('key7');
-            new JDB.CombinedTransaction(tx1, tx2);
+            new CombinedTransaction(tx1, tx2);
 
             expect(await tx2.abort()).toBe(true);
-            expect(await tx1.state).toBe(JDB.Transaction.STATE.ABORTED);
+            expect(await tx1.state).toBe(Transaction.STATE.ABORTED);
             expect(await backend1.get('key7')).toBe('value7');
             expect(await backend2.get('key7')).toBe('value7');
         })().then(done, done.fail);
@@ -52,7 +52,7 @@ describe('CombinedTransaction', () => {
             const tx2 = objectStore1.transaction();
             await tx1.remove('key0');
             try {
-                await JDB.JungleDB.commitCombined(tx1, tx2);
+                await JungleDB.commitCombined(tx1, tx2);
                 expect(false).toBe(true);
             } catch (e) {
                 expect(true).toBe(true);
@@ -69,7 +69,7 @@ describe('CombinedTransaction', () => {
             await tx2.remove('key6');
 
             // Commit both (which should immediately update the backend as well).
-            expect(await JDB.JungleDB.commitCombined(tx1, tx2)).toBe(true);
+            expect(await JungleDB.commitCombined(tx1, tx2)).toBe(true);
             expect(await backend1.get('key6')).toBe(undefined);
             expect(await backend2.get('key6')).toBe(undefined);
         })().then(done, done.fail);
@@ -85,7 +85,7 @@ describe('CombinedTransaction', () => {
             await tx2.remove('key6');
 
             // Commit two of them, which should be successful.
-            expect(await JDB.JungleDB.commitCombined(tx1, tx2)).toBe(true);
+            expect(await JungleDB.commitCombined(tx1, tx2)).toBe(true);
             // Hence the object store will be updated.
             expect(await objectStore1.get('key6')).toBe(undefined);
             expect(await objectStore2.get('key6')).toBe(undefined);
@@ -117,7 +117,7 @@ describe('CombinedTransaction', () => {
             const tx5 = objectStore2.transaction();
 
             // Commit tx4 and tx1.
-            expect(await JDB.JungleDB.commitCombined(tx1, tx4)).toBe(true);
+            expect(await JungleDB.commitCombined(tx1, tx4)).toBe(true);
             // Hence the object store will be updated.
             expect(await objectStore1.get('key6')).toBe(undefined);
             expect(await objectStore2.get('key6')).toBe(undefined);
@@ -160,7 +160,7 @@ describe('CombinedTransaction', () => {
 
             // Cannot commit combined transactions including a nested transaction.
             try {
-                await JDB.JungleDB.commitCombined(nested, tx2);
+                await JungleDB.commitCombined(nested, tx2);
                 expect(false).toBe(true);
             } catch (e) {
                 expect(true).toBe(true);
@@ -180,7 +180,7 @@ describe('CombinedTransaction', () => {
             expect(await tx3.commit()).toBe(true);
 
             // Commit and fail (not all tx are committable because of conflict).
-            expect(await JDB.JungleDB.commitCombined(tx1, tx2)).toBe(false);
+            expect(await JungleDB.commitCombined(tx1, tx2)).toBe(false);
             expect(await objectStore1.get('key6')).toBe('value6');
             expect(await objectStore2.get('key6')).toBe('value6');
             expect(await backend1.get('key6')).toBe('value6');
@@ -199,7 +199,7 @@ describe('CombinedTransaction', () => {
             await tx3.remove('key6');
 
             // Commit two transactions.
-            expect(await JDB.JungleDB.commitCombined(tx1, tx3)).toBe(true);
+            expect(await JungleDB.commitCombined(tx1, tx3)).toBe(true);
             // Nothing should be flushed, OS3 should be updated.
             expect(await objectStore1.get('key6')).toBe(undefined);
             expect(await objectStore2.get('key6')).toBe(undefined);
@@ -213,7 +213,7 @@ describe('CombinedTransaction', () => {
             await tx5.put('test', 'successful');
 
             // Next, commit remaining transactions (except tx4) in combination.
-            expect(await JDB.JungleDB.commitCombined(tx4, tx5)).toBe(true);
+            expect(await JungleDB.commitCombined(tx4, tx5)).toBe(true);
             // And everything should be updated, nothing flushed.
             expect(await objectStore1.get('key6')).toBe(undefined);
             expect(await objectStore1.get('test')).toBe('successful');
@@ -256,7 +256,7 @@ describe('CombinedTransaction', () => {
             // Commit both (which should immediately update the backend as well).
             expect(await tx1.get('key6')).toBe('newvalue6');
             expect(await tx2.get('key6')).toBe('newvalue6');
-            expect(await JDB.JungleDB.commitCombined(tx1, tx2)).toBe(true);
+            expect(await JungleDB.commitCombined(tx1, tx2)).toBe(true);
             expect(await objectStore2.get('key6')).toBe('newvalue6');
             expect(await objectStore1.get('key6')).toBe('newvalue6');
         })().then(done, done.fail);

src/test/generic/CombinedTransactionPlatform.spec.js

@@ -2,7 +2,7 @@ describe('CombinedTransactionPlatform', () => {
     let jdb, backend1, backend2, objectStore1, objectStore2;
 
     beforeEach((done) => {
-        jdb = new JDB.JungleDB('test', 1);
+        jdb = new JungleDB('test', 1);
         objectStore1 = jdb.createObjectStore('testStore1');
         objectStore2 = jdb.createObjectStore('testStore2');
 
@@ -33,21 +33,21 @@ describe('CombinedTransactionPlatform', () => {
             let tx2 = objectStore2.transaction();
             await tx1.remove('key6');
             await tx2.remove('key6');
-            new JDB.CombinedTransaction(tx1, tx2);
+            new CombinedTransaction(tx1, tx2);
 
             expect(await tx1.commit()).toBe(true);
-            expect(await tx2.state).toBe(JDB.Transaction.STATE.COMMITTED);
+            expect(await tx2.state).toBe(Transaction.STATE.COMMITTED);
             expect(await backend1.get('key6')).toBe(undefined);
             expect(await backend2.get('key6')).toBe(undefined);
 
             tx1 = objectStore1.transaction();
             tx2 = objectStore2.transaction();
             await tx1.remove('key7');
             await tx2.remove('key7');
-            new JDB.CombinedTransaction(tx1, tx2);
+            new CombinedTransaction(tx1, tx2);
 
             expect(await tx2.abort()).toBe(true);
-            expect(await tx1.state).toBe(JDB.Transaction.STATE.ABORTED);
+            expect(await tx1.state).toBe(Transaction.STATE.ABORTED);
             expect(await backend1.get('key7')).toBe('value7');
             expect(await backend2.get('key7')).toBe('value7');
         })().then(done, done.fail);
@@ -59,7 +59,7 @@ describe('CombinedTransactionPlatform', () => {
             const tx2 = objectStore1.transaction();
             await tx1.remove('key0');
             try {
-                await JDB.JungleDB.commitCombined(tx1, tx2);
+                await JungleDB.commitCombined(tx1, tx2);
                 expect(false).toBe(true);
             } catch (e) {
                 expect(true).toBe(true);
@@ -76,7 +76,7 @@ describe('CombinedTransactionPlatform', () => {
             await tx2.remove('key6');
 
             // Commit both (which should immediately update the backend as well).
-            expect(await JDB.JungleDB.commitCombined(tx1, tx2)).toBe(true);
+            expect(await JungleDB.commitCombined(tx1, tx2)).toBe(true);
             expect(await backend1.get('key6')).toBe(undefined);
             expect(await backend2.get('key6')).toBe(undefined);
         })().then(done, done.fail);
@@ -92,7 +92,7 @@ describe('CombinedTransactionPlatform', () => {
             await tx2.remove('key6');
 
             // Commit two of them, which should be successful.
-            expect(await JDB.JungleDB.commitCombined(tx1, tx2)).toBe(true);
+            expect(await JungleDB.commitCombined(tx1, tx2)).toBe(true);
             // Hence the object store will be updated.
             expect(await objectStore1.get('key6')).toBe(undefined);
             expect(await objectStore2.get('key6')).toBe(undefined);
@@ -124,7 +124,7 @@ describe('CombinedTransactionPlatform', () => {
             const tx5 = objectStore2.transaction();
 
             // Commit tx4 and tx1.
-            expect(await JDB.JungleDB.commitCombined(tx1, tx4)).toBe(true);
+            expect(await JungleDB.commitCombined(tx1, tx4)).toBe(true);
             // Hence the object store will be updated.
             expect(await objectStore1.get('key6')).toBe(undefined);
             expect(await objectStore2.get('key6')).toBe(undefined);
@@ -167,7 +167,7 @@ describe('CombinedTransactionPlatform', () => {
 
             // Cannot commit combined transactions including a nested transaction.
             try {
-                await JDB.JungleDB.commitCombined(nested, tx2);
+                await JungleDB.commitCombined(nested, tx2);
                 expect(false).toBe(true);
             } catch (e) {
                 expect(true).toBe(true);
@@ -187,7 +187,7 @@ describe('CombinedTransactionPlatform', () => {
             expect(await tx3.commit()).toBe(true);
 
             // Commit and fail (not all tx are committable because of conflict).
-            expect(await JDB.JungleDB.commitCombined(tx1, tx2)).toBe(false);
+            expect(await JungleDB.commitCombined(tx1, tx2)).toBe(false);
             expect(await objectStore1.get('key6')).toBe('value6');
             expect(await objectStore2.get('key6')).toBe('value6');
             expect(await backend1.get('key6')).toBe('value6');
@@ -206,7 +206,7 @@ describe('CombinedTransactionPlatform', () => {
             await tx3.remove('key6');
 
             // Commit two transactions.
-            expect(await JDB.JungleDB.commitCombined(tx1, tx3)).toBe(true);
+            expect(await JungleDB.commitCombined(tx1, tx3)).toBe(true);
             // Nothing should be flushed, OS3 should be updated.
             expect(await objectStore1.get('key6')).toBe(undefined);
             expect(await objectStore2.get('key6')).toBe(undefined);
@@ -220,7 +220,7 @@ describe('CombinedTransactionPlatform', () => {
             await tx5.put('test', 'successful');
 
             // Next, commit remaining transactions (except tx4) in combination.
-            expect(await JDB.JungleDB.commitCombined(tx4, tx5)).toBe(true);
+            expect(await JungleDB.commitCombined(tx4, tx5)).toBe(true);
             // And everything should be updated, nothing flushed.
             expect(await objectStore1.get('key6')).toBe(undefined);
             expect(await objectStore1.get('test')).toBe('successful');
@@ -263,7 +263,7 @@ describe('CombinedTransactionPlatform', () => {
             // Commit both (which should immediately update the backend as well).
             expect(await tx1.get('key6')).toBe('newvalue6');
             expect(await tx2.get('key6')).toBe('newvalue6');
-            expect(await JDB.JungleDB.commitCombined(tx1, tx2)).toBe(true);
+            expect(await JungleDB.commitCombined(tx1, tx2)).toBe(true);
             expect(await objectStore2.get('key6')).toBe('newvalue6');
             expect(await objectStore1.get('key6')).toBe('newvalue6');
         })().then(done, done.fail);