Merge pull request #15 from hypermodules/multi-key-index

Stub out multi-key index

Author: bcomnes

.gitignore

@@ -35,3 +35,4 @@ jspm_packages
 
 # Optional REPL history
 .node_repl_history
+.vscode

CHANGELOG.md

@@ -5,6 +5,10 @@ All notable changes to this project will be documented in this file.
 The format is based on [Keep a Changelog](http://keepachangelog.com/)
 and this project adheres to [Semantic Versioning](http://semver.org/).
 
+## 1.1.0 - 2018-03-08
+
+* Add multi-key indexing.  Multi-key reducers can return an array of keys to index per `put`, `del`, or `batch`. Thanks [@louiscenter](https://github.com/louiscenter) and [@substack](https://github.com/substack) for the idea.
+
 ## 1.0.5
 
 * Fix index cleaning bug.  Indexes should now clean up missing lookups.

README.md

@@ -1,4 +1,4 @@
-# level-auto-index
+****# level-auto-index
 
 Automatic secondary indexing for leveldb and subleveldown.
 
@@ -32,7 +32,8 @@ var db = level()
 var posts = sub(db, 'posts', {valueEncoding: 'json'})
 var idx = {
   title: sub(db, 'title'),
-  length: sub(db, 'length')
+  length: sub(db, 'length'),
+  tag: sub(db, 'tag')
 }
 
 // add a title index
@@ -44,10 +45,19 @@ posts.byLength = AutoIndex(posts, idx.length, function (post) {
   return post.body.length + '!' + post.id
 })
 
+// Create multiple index keys on an index
+posts.byTag = AutoIndex(posts, idx.tag, function (post) {
+    if (!post || !post.tags || !Array.isArray(post.tags)) return
+    return post.tags.map(function (tag) {
+      return [tag, post.id].join('!')
+    })
+  }, { multi: true })
+
 posts.put('1337', {
   id: '1337',
   title: 'a title',
-  body: 'lorem ipsum'
+  body: 'lorem ipsum',
+  tags: [ 'foo', 'bar', 'baz' ]
 }, function (err) {
   if (err) throw err
 
@@ -87,7 +97,7 @@ posts.put('1337', {
 
 ## API
 
-### AutoIndex(db, idb, reduce)
+### AutoIndex(db, idb, reduce, opts)
 
 Automatically index a `db` level into the `idb` level using a `reduce` function that creates the index key.  The `db` and `idb` levels should be in isolated key namespaces, either by being two different levels or [`mafintosh/subleveldown`](https://github.com/mafintosh/subleveldown) partitions.  The `db` hook is mutated by [`hypermodules/level-hookdown`](https://github.com/hypermodules/level-hookdown) to set up the prehooks used for indexing.  Only `db` keys are stored as values to save space and reduce data redundancy.
 
@@ -102,6 +112,16 @@ function reducer (value) {
 }
 ```
 
+Available opts:
+
+```js
+{
+  multi: false // Reducer returns an array of keys to associate with the primary key
+}
+```
+
+Multi-key index's are for when you you want to write multiple index entries into an index.  This is useful for 'tag' fields, where a document may have `n` tags per document, and you would like to index documents by 'tag'.  When creating a multi-key index, your reducer must return an array of keys to index by.
+
 ### AutoIndex#get(key, opts, cb)
 
 Get the value that has been indexed with `key`.
@@ -129,6 +149,20 @@ function keyReducer (reducerString) {
 
 For a higher level api for creating secondary indexes see [hypermodules/level-idx](https://github.com/hypermodules/level-idx).
 
+### AutoIndex.keyReducer(string)
+
+A shortcut reducer for simplistic multi-key indexing.  You might need more than this.
+
+```js
+function multiKeyReducer (multiFieldName, primaryKeyFieldName) {
+  return function multiKeyrdc (document) {
+    if (!document || !document[multiFieldName] || !Array.isArray(document[multiFieldName])) return
+    return document[multiFieldName].map(function (tag) {
+      return [tag, document[primaryKeyFieldName]].join('!')
+    })
+  }
+```
+
 ### AutoIndex#db
 
 The level instance that we are indexing.

index.js

@@ -1,8 +1,7 @@
 var extend = require('xtend')
 var hook = require('level-hookdown')
 var existy = require('existy')
-var Transform =
-  require('stream').Transform || require('readable-stream').Transform
+var Transform = require('stream').Transform || require('readable-stream').Transform
 var isArray = Array.isArray
 
 module.exports = AutoIndex
@@ -17,24 +16,51 @@ function existyKeys (operation) {
   return existy(operation.key)
 }
 
-function AutoIndex (db, idb, reduce) {
+function empty (v) {
+  return !!v
+}
+
+function AutoIndex (db, idb, reduce, opts) {
+  if (!opts) opts = {}
   var hdb = !db.prehooks || !isArray(db.prehooks) ? hook(db) : db
+  var multiKey = opts.multi
 
   if (typeof reduce !== 'function') {
-    throw new Error('Reduce argument must be a string or function')
+    throw new Error('Reduce argument must be a function')
   }
 
   function index (operation, cb) {
     var key
+    var keyBatch
     if (operation.type === 'put') {
       key = reduce(operation.value)
-      if (key) return idb.put(key, operation.key, cb)
+      if (key && multiKey) {
+        if (!Array.isArray(key)) throw new Error('Reducer must return an array of keys for a multiKey index')
+        keyBatch = key.filter(empty).map(
+          function (k) {
+            return { key: k, value: operation.key }
+          }
+        )
+        return idb.batch(keyBatch, cb)
+      } else if (key) {
+        return idb.put(key, operation.key, cb)
+      }
       return process.nextTick(cb)
     } else if (operation.type === 'del') {
       db.get(operation.key, function (err, value) {
-        if (err && err.type === 'NotFoundError') {
-          key = reduce(operation.value)
-          if (key) return idb.del(key, cb)
+        if (!err || err.type === 'NotFoundError') {
+          key = reduce(value)
+          if (key && multiKey) {
+            if (!Array.isArray(key)) throw new Error('Reducer must return an array of keys for a multiKey index')
+            keyBatch = key.filter(empty).map(
+              function (k) {
+                return { key: k, type: 'del' }
+              }
+            )
+            return idb.batch(keyBatch, cb)
+          } else if (key) {
+            return idb.del(key, key, cb)
+          }
           return cb()
         } else if (err) {
           cb(err)
@@ -47,6 +73,16 @@ function AutoIndex (db, idb, reduce) {
       var idxBatch = operation.array.filter(puts).map(function (opr) {
         return extend(opr, {key: reduce(opr.value), value: opr.key})
       })
+      if (multiKey) {
+        idxBatch = idxBatch.reduce(function (flattened, opr) {
+          if (!opr.key) return flattened
+          if (!Array.isArray(opr.key)) throw new Error('Reducer must return an array of keys for a multiKey index')
+          opr.key.filter(empty).forEach(function (k) {
+            flattened.push(extend(opr, { key: k }))
+          })
+          return flattened
+        }, [])
+      }
       idb.batch(idxBatch.filter(existyKeys), cb)
     }
   }
@@ -149,3 +185,14 @@ function keyReducer (reducerString) {
   }
   return keyRdc
 }
+
+module.exports.multiKeyReducer = multiKeyReducer
+
+function multiKeyReducer (multiFieldName, primaryKeyFieldName) {
+  return function multiKeyrdc (document) {
+    if (!document || !document[multiFieldName] || !Array.isArray(document[multiFieldName])) return
+    return document[multiFieldName].map(function (tag) {
+      return [tag, document[primaryKeyFieldName]].join('!')
+    })
+  }
+}

package.json

@@ -7,7 +7,8 @@
     "test": "run-s test:*",
     "test:deps": "dependency-check ./package.json",
     "test:lint": "standard | snazzy",
-    "test:tape": "tape test/* | tap-format-spec"
+    "test:tape": "tape test/* | tap-format-spec",
+    "debug": "node --nolazy --inspect-brk=9229 node_modules/.bin/tape test/*"
   },
   "repository": {
     "type": "git",
@@ -36,6 +37,7 @@
   "devDependencies": {
     "@tap-format/spec": "^0.2.0",
     "bytewise": "^1.1.0",
+    "concat-stream": "^1.6.1",
     "dependency-check": "^3.0.0",
     "memdb": "^1.3.1",
     "memdown": "^2.0.0",

test/multi-key.js

@@ -0,0 +1,113 @@
+var level = require('memdb')
+var AutoIndex = require('..')
+var sub = require('subleveldown')
+var test = require('tape')
+var concat = require('concat-stream')
+var multiKeyReducer = AutoIndex.multiKeyReducer
+
+test('multi-key index', function (t) {
+  var db = level()
+  var index = {
+    tags: sub(db, 'tags')
+  }
+
+  var posts = sub(db, 'posts', {valueEncoding: 'json'})
+  posts.byTag = AutoIndex(posts, index.tags, multiKeyReducer('tags', 'id'), { multi: true })
+
+  var postData = [
+    {
+      title: 'a title',
+      id: 0,
+      body: 'lorem ipsum',
+      tags: [ 'foo', 'bar', 'baz' ]
+    },
+    {
+      title: 'second title',
+      id: 1,
+      body: 'second body',
+      tags: []
+    },
+    {
+      title: 'third title',
+      id: 2,
+      body: 'third body'
+    },
+    {
+      title: 'fourth title',
+      body: 'fourth body',
+      id: 3,
+      tags: ['bing', 'google', 'foo']
+    }
+  ]
+
+  var batch = postData.map(function (post) {
+    return {
+      type: 'put',
+      key: post.id,
+      value: post
+    }
+  })
+
+  const singlePost = {
+    title: 'some title',
+    body: 'some body',
+    id: 20,
+    tags: ['bing', 'google', 'foo', 'bleep', 'bloop']
+  }
+
+  posts.put(singlePost.id, singlePost, function (err) {
+    t.error(err)
+    posts.batch(batch, function (err) {
+      t.error(err)
+
+      var tagIndexStream = posts.byTag.createReadStream({ gt: 'foo!', lt: 'foo!~' })
+      var concatStream = concat(handleResults)
+      tagIndexStream.on('error', handleError)
+      tagIndexStream.pipe(concatStream)
+
+      function handleResults (data) {
+        t.equal(data.length, 3)
+        t.equal(data[0].key, '0')
+        t.deepEqual(data[0].value, postData[0])
+        t.equal(data[1].key, '20')
+        t.deepEqual(data[1].value, singlePost)
+        t.equal(data[2].key, '3')
+        t.deepEqual(data[2].value, postData[3])
+        posts.del(3, function (err) {
+          t.error(err)
+          secondTest()
+        })
+      }
+
+      function secondTest () {
+        var tagIndexStream = index.tags.createReadStream()
+        var concatStream = concat(handleResults)
+        tagIndexStream.on('error', handleError)
+        tagIndexStream.pipe(concatStream)
+        function handleResults (data) {
+          t.equal(data.length, 8)
+          thirdTest()
+        }
+      }
+
+      function thirdTest () {
+        var tagIndexStream = posts.byTag.createReadStream({ gt: 'foo!', lt: 'foo!~' })
+        var concatStream = concat(handleResults)
+        tagIndexStream.on('error', handleError)
+        tagIndexStream.pipe(concatStream)
+        function handleResults (data) {
+          t.equal(data.length, 2)
+          t.deepEqual(data[0].value, postData[0])
+          t.equal(data[1].key, '20')
+          t.deepEqual(data[1].value, singlePost)
+          t.end()
+        }
+      }
+
+      function handleError (err) {
+        t.error(err)
+        t.end()
+      }
+    })
+  })
+})