Fix DB issues with object keys (#1125)

Fix DB issues with object keys

apps/server/src/common/db/collections-registry.ts

@@ -0,0 +1,6 @@
+import { Collection } from 'lokijs';
+import { App } from '@flogo-web/core';
+
+export const collections: {
+  apps?: Collection<App>;
+} = {};

apps/server/src/common/db/db.spec.ts

@@ -0,0 +1,48 @@
+import { initDb, persistedDb } from './db';
+
+let collection: Collection<any>;
+
+beforeEach(async () => {
+  await initDb({ persist: false });
+  collection = persistedDb.addCollection('test');
+});
+
+describe('db library integration', () => {
+  it('should save object with dots in its keys', function() {
+    const inserted = collection.insert({
+      'my.object': {
+        'is.nested': {
+          foo: [
+            {
+              'bar.baz': true,
+            },
+          ],
+        },
+      },
+    });
+
+    expect(inserted).toMatchObject({
+      'my.object': {
+        'is.nested': {
+          foo: [
+            {
+              'bar.baz': true,
+            },
+          ],
+        },
+      },
+    });
+
+    expect(collection.get(inserted.$loki)).toMatchObject({
+      'my.object': {
+        'is.nested': {
+          foo: [
+            {
+              'bar.baz': true,
+            },
+          ],
+        },
+      },
+    });
+  });
+});

apps/server/src/common/db/db.ts

@@ -0,0 +1,57 @@
+import Loki from 'lokijs';
+
+import { App } from '@flogo-web/core';
+import { config } from '../../config';
+import { collections } from './collections-registry';
+
+const dbPath = config.apps.dbPath;
+
+export let persistedDb: Loki;
+// todo: use by non-persistent collections like contributions
+const memoryDb = new Loki('mem.db', { adapter: new Loki.LokiMemoryAdapter() });
+
+export function initDb({ persist = true, autosave = true } = {}) {
+  return new Promise(resolve => {
+    persistedDb = new Loki(dbPath, {
+      adapter: persist ? new Loki.LokiFsAdapter() : new Loki.LokiMemoryAdapter(),
+      autoload: true,
+      autosave,
+      autoloadCallback: afterInitDb(resolve),
+      autosaveInterval: 4000,
+    });
+  });
+}
+
+export function flushAndCloseDb() {
+  if (persistedDb) {
+    return new Promise((resolve, reject) => {
+      persistedDb.save(err => {
+        if (err) {
+          return reject(err);
+        }
+
+        persistedDb.close(err2 => {
+          if (err2) {
+            return reject(err2);
+          }
+          resolve();
+        });
+      });
+    });
+  }
+}
+
+function afterInitDb(signalReadyFn: Function) {
+  return () => {
+    let apps = persistedDb.getCollection<App>('apps');
+    if (apps == null) {
+      apps = persistedDb.addCollection('apps', {
+        unique: ['id'],
+        indices: ['name'],
+        clone: true,
+      });
+    }
+    collections.apps = apps;
+    signalReadyFn();
+  };
+}

apps/server/src/common/db/index.ts

@@ -1,3 +1,4 @@
-export { apps } from './apps';
-export { indexer } from './indexer';
 export { contributionsDBService } from './contributions';
+
+export { initDb, flushAndCloseDb } from './db';
+export { collections } from './collections-registry';

apps/server/src/config/app-config.js

@@ -21,8 +21,6 @@ const FLOW_TESTER_PORT = process.env.FLOGO_FLOW_TESTER_PORT || '8080';
 
 const DB_DIR = process.env.FLOGO_WEB_DBDIR || path.resolve(LOCAL_DIR, 'db');
 
-console.log(`localDir=${LOCAL_DIR},publicDir=${PUBLIC_DIR}`);
-
 const logLevel = process.env.FLOGO_WEB_LOGLEVEL || 'debug';
 
 const appPort = process.env.PORT || 3303;
@@ -69,7 +67,7 @@ const config = {
   /* apps module config */
   // TODO: consolidate and cleanup
   apps: {
-    dbPath: path.resolve(DB_DIR, 'apps.db'),
+    dbPath: path.resolve(DB_DIR, 'flogo.db'),
   },
   indexer: {
     dbPath: path.resolve(DB_DIR, 'indexer.db'),

apps/server/src/configure-engines.ts

@@ -1,22 +1,18 @@
 import { rootContainer, installDefaults } from './init';
-import { config } from './config/app-config';
-import { TOKENS } from './core';
-import { Database } from './common/database.service';
+import { config } from './config';
 import { getInitializedEngine } from './modules/engine';
 import { syncTasks } from './modules/contrib-install-controller/sync-tasks';
 import { AppsService } from './modules/apps';
+import { initDb, flushAndCloseDb } from './common/db';
 
-getInitializedEngine(config.defaultEngine.path, { forceCreate: false })
+initDb()
+  .then(() => getInitializedEngine(config.defaultEngine.path, { forceCreate: false }))
   .then(engine => syncTasks(engine))
   .then(() => {
     console.log('[log] init test engine done');
     return installDefaults(rootContainer.resolve(AppsService));
   })
-  .then(() => {
-    const apps = rootContainer.get<Database>(TOKENS.AppsDb);
-    const indexer = rootContainer.get<Database>(TOKENS.ResourceIndexerDb);
-    return Promise.all([apps.compact(), indexer.compact()]);
-  })
+  .then(() => flushAndCloseDb())
   .catch(error => {
     console.error(error);
     console.error(error.stack);

apps/server/src/core/models/app.ts

@@ -5,7 +5,7 @@ export function constructApp(inputData, generateId?: () => string): App {
   const now = new Date().toISOString();
   return {
     ...inputData,
-    _id: inputData._id || generateId(),
+    id: inputData.id || generateId(),
     name: inputData.name.trim(),
     createdAt: now,
     updatedAt: null,

apps/server/src/injector/persistence/module.ts

@@ -1,8 +1,9 @@
 import { ContainerModule, interfaces } from 'inversify';
-import { apps, indexer } from '../../common/db';
+import { collections } from '../../common/db';
+import { indexer } from '../../common/db/indexer';
 import { TOKENS } from '../../core';
 
 export const PersistenceModule = new ContainerModule((bind: interfaces.Bind) => {
-  bind(TOKENS.AppsDb).toConstantValue(apps);
+  bind(TOKENS.AppsDb).toDynamicValue(() => collections.apps);
   bind(TOKENS.ResourceIndexerDb).toConstantValue(indexer);
 });

apps/server/src/modules/apps/app-importer.ts

@@ -31,6 +31,6 @@ export class AppImporter {
       shortid.generate,
       contributions
     );
-    return { _id: id, ...newApp };
+    return { id, ...newApp };
   }
 }