Custom separator

index.js

@@ -44,6 +44,7 @@ function HyperTrie (storage, key, opts) {
   this.metadata = opts.metadata || null
   this.hash = opts.hash || null
   this.valueEncoding = opts.valueEncoding ? codecs(opts.valueEncoding) : null
+  this.sep = opts.sep || '/'
   this.alwaysUpdate = !!opts.alwaysUpdate
   this.alwaysReconnect = !!opts.alwaysReconnect
   this.subtype = opts.subtype
@@ -249,6 +250,9 @@ HyperTrie.prototype.list = function (prefix, opts, cb) {
 
 HyperTrie.prototype.iterator = function (prefix, opts) {
   if (isOptions(prefix)) return this.iterator('', prefix)
+  opts = Object.assign({}, opts, {
+    sep: this.sep
+  })
   return new Iterator(this, prefix, opts)
 }
 
@@ -277,6 +281,9 @@ HyperTrie.prototype.createDiffStream = function (other, prefix, opts) {
 
 HyperTrie.prototype.get = function (key, opts, cb) {
   if (typeof opts === 'function') return this.get(key, null, opts)
+  opts = Object.assign({}, opts, {
+    sep: this.sep
+  })
   return new Get(this, key, opts, cb)
 }
 
@@ -292,6 +299,7 @@ HyperTrie.prototype.batch = function (ops, cb) {
 HyperTrie.prototype.put = function (key, value, opts, cb) {
   if (typeof opts === 'function') return this.put(key, value, null, opts)
   opts = Object.assign({}, opts, {
+    sep: this.sep,
     batch: null,
     del: 0
   })
@@ -301,6 +309,7 @@ HyperTrie.prototype.put = function (key, value, opts, cb) {
 HyperTrie.prototype.del = function (key, opts, cb) {
   if (typeof opts === 'function') return this.del(key, null, opts)
   opts = Object.assign({}, opts, {
+    sep: this.sep,
     batch: null
   })
   return new Delete(this, key, opts, cb)
@@ -320,17 +329,16 @@ HyperTrie.prototype.getBySeq = function (seq, opts, cb) {
   if (typeof opts === 'function') return this.getBySeq(seq, null, opts)
   if (seq < 1) return process.nextTick(cb, null, null)
   const self = this
-
   const cached = this._cache.get(seq)
   if (cached) return process.nextTick(onnode, null, cached)
   this.feed.get(seq, opts, onnode)
 
   function onnode (err, val) {
     if (err) return cb(err)
-    const node = Node.decode(val, seq, self.valueEncoding, self.hash)
+    const node = Node.decode(val, seq, self.valueEncoding, self.hash, self.sep)
     self._cache.set(seq, val)
     // early exit for the key: '' nodes we write to reset the db
-    if (!node.value && !node.key) return cb(null, null)
+    if (node.value === null && node.key === '') return cb(null, null)
     cb(null, node)
   }
 }

lib/batch.js

@@ -63,7 +63,6 @@ Batch.prototype._start = function () {
 Batch.prototype._update = function () {
   var i = 0
   const self = this
-
   loop(null, null)
 
   function loop (err, head) {
@@ -73,6 +72,6 @@ Batch.prototype._update = function () {
 
     const {type, key, value, hidden, flags} = self._ops[i++]
     if (type === 'del') self._op = new Delete(self._db, key, { batch: self, hidden }, loop)
-    else self._op = new Put(self._db, key, value === undefined ? null : value, { batch: self, del: 0, hidden, flags }, loop)
+    else self._op = new Put(self._db, key, value === undefined ? null : value, { batch: self, del: 0, hidden, flags, sep: self._db.sep }, loop)
   }
 }

lib/del.js

@@ -3,15 +3,15 @@ const Node = require('./node')
 
 module.exports = Delete
 
-function Delete (db, key, { batch, condition = null, hidden = false, closest = false }, cb) {
+function Delete (db, key, { batch, condition = null, hidden = false, closest = false, sep }, cb) {
   this._db = db
   this._key = key
   this._callback = cb
   this._release = null
   this._put = null
   this._batch = batch
   this._condition = condition
-  this._node = new Node({key, flags: hidden ? Node.Flags.HIDDEN : 0}, null, null, db.hash)
+  this._node = new Node({key, flags: hidden ? Node.Flags.HIDDEN : 0}, null, null, db.hash, sep)
   this._length = this._node.length
   this._returnClosest = closest
   this._closest = 0

lib/get.js

@@ -4,7 +4,7 @@ module.exports = Get
 
 function Get (db, key, opts, cb) {
   this._db = db
-  this._node = new Node({key, flags: (opts && opts.hidden) ? Node.Flags.HIDDEN : 0}, 0, null, db.hash)
+  this._node = new Node({key, flags: (opts && opts.hidden) ? Node.Flags.HIDDEN : 0}, 0, null, db.hash, opts.sep)
   this._callback = cb
   this._prefix = !!(opts && opts.prefix)
   this._closest = !!(opts && opts.closest)

lib/history.js

@@ -43,6 +43,7 @@ History.prototype._next = function (cb) {
 
   function done (err, node) {
     if (err) return cb(err)
+    if (!node) return cb(null, null)
     cb(null, node.final())
   }
 }

lib/iterator.js

@@ -20,7 +20,7 @@ function Iterator (db, prefix, opts) {
   }
 
   this._checkpoint = (opts && opts.checkpoint) || null
-  this._prefix = Node.normalizeKey(prefix || '')
+  this._prefix = Node.normalizeKey(prefix || '', opts.sep)
   this._recursive = !opts || opts.recursive !== false
   this._order = (opts && opts.reverse) ? REVERSE_SORT_ORDER : SORT_ORDER
   this._random = !!(opts && opts.random)
@@ -36,7 +36,7 @@ function Iterator (db, prefix, opts) {
   this._error = null
   this._gt = !!(opts && opts.gt)
   this._needsSort = []
-  this._options = opts ? { extension: opts.extension, wait: opts.wait, timeout: opts.timeout, hidden: !!opts.hidden, onseq: opts.onseq, onwait: null } : { onwait: null }
+  this._options = opts ? { extension: opts.extension, wait: opts.wait, timeout: opts.timeout, hidden: !!opts.hidden, onseq: opts.onseq, onwait: null, sep: opts.sep } : { onwait: null, sep: opts.sep }
   this._flags = (this._recursive ? 1 : 0) | (this._order === REVERSE_SORT_ORDER ? 2 : 0) | (this._gt ? 4 : 0) | ((this._options && this._options.hidden) ? 8 : 0)
   if (this._extensionState) this._options.onwait = this._sendExt.bind(this)
 }

lib/node.js

@@ -11,11 +11,11 @@ const Flags = {
   HIDDEN: 1
 }
 
-function Node (data, seq, enc, userHash) {
+function Node (data, seq, enc, userHash, sep) {
   this.seq = seq || 0
-  this.key = normalizeKey(data.key)
+  this.key = normalizeKey(data.key, sep)
   this.value = data.value !== undefined ? data.value : null
-  this.keySplit = split(this.key)
+  this.keySplit = split(this.key, sep)
   this.hash = userHash ? userHash(this.key) : hash(this.keySplit)
   this.trie = data.trieBuffer ? trie.decode(data.trieBuffer) : (data.trie || [])
   this.trieBuffer = null
@@ -88,8 +88,8 @@ Node.prototype.collides = function (node, i) {
   return this.keySplit[j] !== node.keySplit[j]
 }
 
-Node.decode = function (buf, seq, enc, hash) {
-  return new Node(messages.Node.decode(buf), seq, enc, hash)
+Node.decode = function (buf, seq, enc, hash, sep) {
+  return new Node(messages.Node.decode(buf), seq, enc, hash, sep)
 }
 
 Node.terminator = function (i) {
@@ -110,16 +110,16 @@ function hash (keys) {
   return buf
 }
 
-function split (key) {
-  const list = key.split('/')
+function split (key, sep) {
+  const list = key.split(sep)
   if (list[0] === '') list.shift()
   if (list[list.length - 1] === '') list.pop()
   return list
 }
 
-function normalizeKey (key) {
+function normalizeKey (key, sep) {
   if (!key.length) return ''
-  return key[0] === '/' ? key.slice(1) : key
+  return key[0] === sep ? key.slice(1) : key
 }
 
 function defaultStylize (val) {

lib/put.js

@@ -20,7 +20,7 @@ function Put (db, key, value, opts, cb) {
   // The flags are shifted in order to both hide the internal flags and support user-defined flags.
   flags = (flags << 8) | (hidden ? Node.Flags.HIDDEN : 0)
 
-  this._node = new Node({key, value, valueBuffer, flags}, 0, db.valueEncoding, db.hash)
+  this._node = new Node({key, value, valueBuffer, flags}, 0, db.valueEncoding, db.hash, opts.sep)
   this._callback = cb
   this._release = null
   this._batch = batch

test/separator.js

@@ -0,0 +1,171 @@
+const tape = require('tape')
+const create = require('./helpers/create')
+
+const sep = Buffer.alloc(1)
+
+tape('basic iteration', function (t) {
+  const db = create(null, {sep})
+  const vals = ['a', 'b', 'c']
+  const expected = toMap(vals)
+
+  put(db, vals, function (err) {
+    t.error(err, 'no error')
+    all(db.iterator(), function (err, map) {
+      t.error(err, 'no error')
+      t.same(map, expected, 'iterated all values')
+      t.end()
+    })
+  })
+})
+
+tape('iterate a big db', function (t) {
+  const db = create(null, {sep})
+  const vals = range(1000, '#')
+  const expected = toMap(vals)
+
+  put(db, vals, function (err) {
+    t.error(err, 'no error')
+    all(db.iterator(), function (err, map) {
+      t.error(err, 'no error')
+      t.same(map, expected, 'iterated all values')
+      t.end()
+    })
+  })
+})
+
+tape('prefix basic iteration', function (t) {
+  const db = create(null, {sep})
+  var vals = ['foo' + sep + 'a', 'foo' + sep + 'b', 'foo' + sep + 'c']
+  const expected = toMap(vals)
+
+  vals = vals.concat(['a', 'b', 'c'])
+
+  put(db, vals, function (err) {
+    t.error(err, 'no error')
+    all(db.iterator('foo'), function (err, map) {
+      t.error(err, 'no error')
+      t.same(map, expected, 'iterated all values')
+      t.end()
+    })
+  })
+})
+
+tape('empty prefix iteration', function (t) {
+  const db = create(null, {sep})
+  const vals = ['foo/a', 'foo/b', 'foo/c']
+  const expected = {}
+
+  put(db, vals, function (err) {
+    t.error(err, 'no error')
+    all(db.iterator('bar'), function (err, map) {
+      t.error(err, 'no error')
+      t.same(map, expected, 'iterated all values')
+      t.end()
+    })
+  })
+})
+
+tape('prefix iterate a big db', function (t) {
+  var vals = range(1000, 'foo' + sep + '#')
+  const db = create(null, {sep})
+  const expected = toMap(vals)
+
+  vals = vals.concat(range(1000, '#'))
+
+  put(db, vals, function (err) {
+    t.error(err, 'no error')
+    all(db.iterator('foo'), function (err, map) {
+      t.error(err, 'no error')
+      t.same(map, expected, 'iterated all values')
+      t.end()
+    })
+  })
+})
+
+tape('non recursive iteration', function (t) {
+  const db = create(null, {sep})
+  const vals = [
+    'a',
+    'a' + sep + 'b' + sep + 'c' + sep + 'd',
+    'a' + sep + 'b',
+    'b',
+    'b' + sep + 'b' + sep + 'c',
+    'c' + sep + 'a',
+    'c'
+  ]
+
+  put(db, vals, function (err) {
+    t.error(err, 'no error')
+    all(db.iterator({recursive: false}), function (err, map) {
+      t.error(err, 'no error')
+      // console.log('map', map)
+      const keys = Object.keys(map).map(k => k.split(sep)[0])
+      t.same(keys.sort(), ['a', 'b', 'c'], 'iterated all values')
+      t.end()
+    })
+  })
+})
+
+tape('mixed nested and non nexted iteration', function (t) {
+  const db = create(null, {sep})
+  const vals = ['a', 'a' + sep + 'a', 'a' + sep + 'b', 'a' + sep + 'c', 'a' + sep + 'a' + sep + 'a', 'a' + sep + 'a' + sep + 'b', 'a' + sep + 'a' + sep + 'c']
+  const expected = toMap(vals)
+
+  put(db, vals, function (err) {
+    t.error(err, 'no error')
+    all(db.iterator(), function (err, map) {
+      t.error(err, 'no error')
+      t.same(map, expected, 'iterated all values')
+      t.end()
+    })
+  })
+})
+
+tape('list buffers an iterator', function (t) {
+  const db = create(null, {sep})
+
+  put(db, ['a', 'b', 'b' + sep + 'c'], function (err) {
+    t.error(err, 'no error')
+    db.list(function (err, all) {
+      t.error(err, 'no error')
+      t.same(all.map(v => v.key).sort(), ['a', 'b', 'b' + sep + 'c'])
+      db.list('b', {gt: true}, function (err, all) {
+        t.error(err, 'no error')
+        t.same(all.length, 1)
+        t.same(all[0].key, 'b' + sep + 'c')
+        t.end()
+      })
+    })
+  })
+})
+
+function range (n, v) {
+  // #0, #1, #2, ...
+  return new Array(n).join('.').split('.').map((a, i) => v + i)
+}
+
+function toMap (list) {
+  const map = {}
+  for (var i = 0; i < list.length; i++) {
+    map[list[i]] = list[i]
+  }
+  return map
+}
+
+function all (ite, cb) {
+  const vals = {}
+
+  ite.next(function loop (err, node) {
+    if (err) return cb(err)
+    if (!node) return cb(null, vals)
+    const key = Array.isArray(node) ? node[0].key : node.key
+    // console.log('node', node)
+    if (vals[key]) return cb(new Error('duplicate node for ' + key))
+    vals[key] = Array.isArray(node) ? node.map(n => n.value).sort() : node.value
+    ite.next(loop)
+  })
+}
+
+function put (db, vals, cb) {
+  db.batch(vals.map(v => ({key: v, value: v})), cb)
+}