From 18b419bd084970673f281cf9c3f224f30baa5047 Mon Sep 17 00:00:00 2001 From: Alexander Popov Date: Wed, 4 Sep 2013 18:03:34 +0400 Subject: [PATCH 01/27] fix for https://github.com/mostlyserious/riak-js/issues/180 - allow use in query. Also support for max_results and continuation params --- lib/http-client.js | 12 ++++++++++-- lib/http-meta.js | 5 ++++- lib/meta.js | 5 ++++- 3 files changed, 18 insertions(+), 4 deletions(-) diff --git a/lib/http-client.js b/lib/http-client.js index a7c6fa5..66db4a1 100644 --- a/lib/http-client.js +++ b/lib/http-client.js @@ -358,13 +358,21 @@ HttpClient.prototype.query = function(bucket, query, options, callback) { value = value[0]; } - var type = typeof value == 'number' ? 'int' : 'bin', - key = "index/" + field + "_" + type + "/" + encodeURIComponent(value); + if ( field == "$bucket" ){ + key = "index/" + field + "/" + encodeURIComponent(value); + } + else{ + var type = typeof value == 'number' ? 'int' : 'bin', + key = "index/" + field + "_" + type + "/" + encodeURIComponent(value); + } if (end) key += "/" + encodeURIComponent(end); this.get(bucket, key, meta, function(err, data, _meta) { + var continuation = data.continuation; data = data ? data.keys : undefined; + if ( continuation && data ) + data.continuation = continuation; meta.callback(err, data, _meta); }); diff --git a/lib/http-meta.js b/lib/http-meta.js index e80d10c..5be748f 100644 --- a/lib/http-meta.js +++ b/lib/http-meta.js @@ -222,7 +222,10 @@ HttpMeta.queryProperties = [ 'vtag', 'returnbody', 'chunked', - 'buckets' + 'buckets', + 'max_results', + 'return_terms', + 'continuation', ] module.exports = HttpMeta; diff --git a/lib/meta.js b/lib/meta.js index d8ef892..974fc22 100644 --- a/lib/meta.js +++ b/lib/meta.js @@ -216,7 +216,10 @@ Meta.keywords = [ 'operation', 'pool', '_pool', - 'agent' + 'agent', + 'continuation', + 'max_results', + 'return_terms' ] module.exports = Meta; From 08385dfda89f2b5cf44ba466a40dd69557452380 Mon Sep 17 00:00:00 2001 From: Alexander Popov Date: Wed, 4 Sep 2013 18:22:07 +0400 Subject: [PATCH 02/27] fix for https://github.com/mostlyserious/riak-js/issues/180 - allow use /start/end index --- lib/http-client.js | 12 ++++-------- 1 file changed, 4 insertions(+), 8 deletions(-) diff --git a/lib/http-client.js b/lib/http-client.js index 66db4a1..eb2843c 100644 --- a/lib/http-client.js +++ b/lib/http-client.js @@ -357,14 +357,10 @@ HttpClient.prototype.query = function(bucket, query, options, callback) { end = value[1]; value = value[0]; } - - if ( field == "$bucket" ){ - key = "index/" + field + "/" + encodeURIComponent(value); - } - else{ - var type = typeof value == 'number' ? 'int' : 'bin', - key = "index/" + field + "_" + type + "/" + encodeURIComponent(value); - } + + + var typedIndex = field != "$bucket" && field != "$key"; + var key = "index/" + field + ( typedIndex? ( "_" + (typeof value == 'number' ? 'int' : 'bin') ): "" ) + "/" + encodeURIComponent(value); if (end) key += "/" + encodeURIComponent(end); From 5e2c3cfd007102e5fd69672f68ff3b09fd5b0ad2 Mon Sep 17 00:00:00 2001 From: Alexander Popov Date: Wed, 4 Sep 2013 19:12:07 +0400 Subject: [PATCH 03/27] tests for special indexes , . test for max_results and continuation --- test/http-client-test.js | 39 +++++++++++++++++++++++++++++++++++++++ 1 file changed, 39 insertions(+) diff --git a/test/http-client-test.js b/test/http-client-test.js index 080dea7..86fe2e1 100644 --- a/test/http-client-test.js +++ b/test/http-client-test.js @@ -324,6 +324,45 @@ describe('http-client-tests', function() { }); }); + it('Special Secondary Indices', function (done) { + db.save(bucket, 1500, + { email: 'fran@gmail.com', age: 28 }, + function (err, data, meta) { + should.not.exist(err); + db.save(bucket, 1501, + { email: 'bob@gmail.com', age: 29 }, + function (err, data, meta) { + should.not.exist(err); + db.save(bucket, 1502, + { email: 'joe@gmail.com', age: 30 }, + function (err, data, meta) { + should.not.exist(err); + async.parallel([ + function (callback) { + db.query(bucket, { "$key": [1500, 1501] }, function (err, results) { + should.not.exist(err); + should.exist(results); + results.length.should.equal(2); + results[0].should.equal('1500'); + results[1].should.equal('1501'); + callback(); + }); + }, function (callback) { + db.query(bucket, { "$bucket": '_' }, {max_results: 2}, function (err, results) { + should.not.exist(err); + should.exist(results); + results.length.should.equal(2); + should.exist(results.continuation); + callback(); + }); + }], function () { + done(); + }); + }); + }); + }); + }); + it('Buckets is an Array', function(done) { db.buckets(function(err, buckets) { should.not.exist(err); From edbb778b492da5889a33038bed8a5eebf9b0bbd4 Mon Sep 17 00:00:00 2001 From: Philipp Fehre Date: Mon, 21 Oct 2013 11:37:18 +0200 Subject: [PATCH 04/27] parse index structure from response add the indices to meta.index on get as they are defined when setting them via ```meta.index = { foo: 1, bar: "baz" }``` on save. FIXES #183 --- lib/http-meta.js | 51 +++++++++++++++++++++++++++++++++++++++- test/http-client-test.js | 14 +++++++++++ 2 files changed, 64 insertions(+), 1 deletion(-) diff --git a/lib/http-meta.js b/lib/http-meta.js index e80d10c..107f145 100644 --- a/lib/http-meta.js +++ b/lib/http-meta.js @@ -38,9 +38,15 @@ HttpMeta.prototype.loadResponse = function(response) { this.links = linkUtils.stringToLinks(response.client._httpMessage._headers.link); } + // 2ii + var indices = indexUtils.extract(headers); + if (indices) { + this.index = indices; + } + // etag -- replace any quotes in the string if (headers.etag) this.etag = headers.etag.replace(/"/g, ''); - + // location if (headers.location) { var match = headers.location.match(/^\/([^\/]+)(?:\/([^\/]+))?\/([^\/]+)$/); @@ -247,6 +253,49 @@ var _encodeUri = function(component, encodeUri) { return encodeURIComponent(component.replace(/\+/g, "%20")); } +var indexUtils = { + Matcher: /x-riak-index-(.*)_(.*)/, + + /** + * Parses indices from a response + * @param {Object} headers + * @return {Object} with indices in `Meta` representation. + * @api private + */ + extract: function(headers) { + var rawIndices; + if (rawIndices = indexUtils.propsAndMatchesByRegex(indexUtils.Matcher, headers)) { + var indices = {}; + rawIndices.forEach(function(index) { + var name = index.match[1]; + var type = index.match[2]; + var value = index.value; + if (type == "int") value = parseInt(value); + indices[name] = value; + }) + return indices; + } else { + return false; + } + }, + + propsAndMatchesByRegex: function(r, obj) { + var m; + var key; + var res = []; + for (key in obj) { + if (m = r.exec(key)) { + res.push({ match: m, value: obj[key] }); + } + } + if (res.length > 0) { + return res; + } else { + return false; + } + } +} + var linkUtils = { /** diff --git a/test/http-client-test.js b/test/http-client-test.js index 080dea7..2009da0 100644 --- a/test/http-client-test.js +++ b/test/http-client-test.js @@ -112,6 +112,20 @@ describe('http-client-tests', function() { }); }); + it('Fetching a document with 2ii', function(done) { + var index = { type: 'dude', number: 1 }; + db.save(bucket, 'indexed_dude', + { name: 'Indexed Dude' }, + { index: index }, + function(_err, _data, _meta) { + db.get(bucket, 'indexed_dude', function(err, data, meta) { + should.exist(meta.index); + meta.index.should.eql(index); + done(); + }) + }) + }) + it('Fetching a document with links', function(done) { db.get(bucket, 'other@gmail.com', function(err, data, meta) { should.not.exist(err); From dc6f75fe575b887105a09a1678aac8bcbae16e6c Mon Sep 17 00:00:00 2001 From: Alexander Popov Date: Tue, 12 Nov 2013 15:59:17 +0400 Subject: [PATCH 05/27] better error handling in case of buffer is JSON. Show message instead of Object[Object], because buffer is often looks like: { phase: 0 error: "[{<<"lineno">>,466},{<<"message">>,<<"SyntaxError: syntax error">>},{<<"source">... (length: 92)" input: "{ok,{r_object,<<"users">>,<<"b1f0e0b64f8143a6a45977db8e29ff89">>,[{r_content,{di... (length: 515)" } --- lib/http-request.js | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/lib/http-request.js b/lib/http-request.js index e2e05a1..a5239a7 100644 --- a/lib/http-request.js +++ b/lib/http-request.js @@ -236,7 +236,7 @@ HttpRequest.prototype.handleRequestEnd = function(buffer) { // deal with errors if (this.meta.statusCode >= 400) { var err = new Error(); - err.message = buffer && buffer.toString().trim(); + err.message = buffer && JSON.stringify( buffer ).trim(); err.statusCode = this.meta.statusCode; buffer = err; } From 22e8c8b75a459fcbf93847980d159a015072ffde Mon Sep 17 00:00:00 2001 From: Philipp Fehre Date: Tue, 26 Nov 2013 10:08:43 +0100 Subject: [PATCH 06/27] added .jshintrc to keep a certain coding style --- .jshintrc | 3 +++ 1 file changed, 3 insertions(+) create mode 100644 .jshintrc diff --git a/.jshintrc b/.jshintrc new file mode 100644 index 0000000..9bf7d78 --- /dev/null +++ b/.jshintrc @@ -0,0 +1,3 @@ +{ + "es5": true +} From 8e50767668927346ed95c1ebeb6bc4a6dd8b2ac3 Mon Sep 17 00:00:00 2001 From: Philipp Fehre Date: Tue, 26 Nov 2013 10:09:15 +0100 Subject: [PATCH 07/27] fix missing semicolons either we use asi or we don't mixing is bad --- test/http-client-test.js | 8 ++++---- 1 file changed, 4 insertions(+), 4 deletions(-) diff --git a/test/http-client-test.js b/test/http-client-test.js index 080dea7..5072c5f 100644 --- a/test/http-client-test.js +++ b/test/http-client-test.js @@ -131,7 +131,7 @@ describe('http-client-tests', function() { { bucket: bucket, key: 'test@gmail.com' } ]}, function(err, data, meta) { should.exist(meta.links); - meta.links.should.have.length(1) + meta.links.should.have.length(1); done(); }); @@ -265,7 +265,7 @@ describe('http-client-tests', function() { }, function() { var buf = [], keys = function(keys) { - buf = buf.concat(keys) + buf = buf.concat(keys); }, end = function() { // keys come in random order, need to sort @@ -398,7 +398,7 @@ describe('http-client-tests', function() { var CustomMeta = function() { var args = Array.prototype.slice.call(arguments); HttpMeta.apply(this, args); -} +}; util.inherits(CustomMeta, HttpMeta); @@ -406,4 +406,4 @@ CustomMeta.prototype.parse = function(data) { var result = HttpMeta.prototype.parse.call(this, data); if (result instanceof Object) result.intercepted = true; return result; -} +}; From bd9d006887f337765d96184f5cf425a96124a33f Mon Sep 17 00:00:00 2001 From: Philipp Fehre Date: Sat, 30 Nov 2013 09:43:02 +0100 Subject: [PATCH 08/27] missing ; --- test/http-client-pool-test.js | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/test/http-client-pool-test.js b/test/http-client-pool-test.js index d41aef1..5a527b1 100644 --- a/test/http-client-pool-test.js +++ b/test/http-client-pool-test.js @@ -35,6 +35,6 @@ describe('http-client-pool', function () { done(); }); }); - }) -}) + }); +}); From 4738bb244556b1efac2839538d4ca553b9ebfe3d Mon Sep 17 00:00:00 2001 From: Philipp Fehre Date: Sat, 30 Nov 2013 12:15:11 +0100 Subject: [PATCH 09/27] make the tests cleanup after themself, comment out search tests Since just dumping data in Riak without ever cleaning up is not nice I added a cleanup for all writen keys in the tests. This should also make sure that we don't have dependend tests Also commeted out search tests relying in pre solar functionaliy --- test/http-client-instrumentation-test.js | 15 ++++++++---- test/http-client-pool-test.js | 16 +++++++++---- test/http-client-search-test.js | 24 +++++++++---------- test/http-client-test.js | 7 ++++++ test/http-mapreduce-client-test.js | 7 +++++- test/protocol-buffers-client-test.js | 20 +++++++++++----- test/protocol-buffers-mapreduce-test.js | 13 ++++++---- test/protocol-buffers-search-test.js | 15 +++++++----- test/test_helper.js | 30 ++++++++++++++++++++++++ 9 files changed, 106 insertions(+), 41 deletions(-) create mode 100644 test/test_helper.js diff --git a/test/http-client-instrumentation-test.js b/test/http-client-instrumentation-test.js index f1dee82..9f44a4f 100644 --- a/test/http-client-instrumentation-test.js +++ b/test/http-client-instrumentation-test.js @@ -1,7 +1,8 @@ var HttpClient = require('../lib/http-client'), HttpMeta = require('../lib/http-meta'), should = require('should'), - util = require('util'); + util = require('util'), + helpers = require('./test_helper.js'); var db, events = [], listener, bucket; @@ -11,16 +12,16 @@ describe('http-client-instrumentation-tests', function() { listener = { "riak.request.start": function(event) { - events.push(event) + events.push(event); }, "riak.request.response": function(event) { - events.push(event) + events.push(event); }, "riak.request.finish": function(event) { - events.push(event) + events.push(event); }, "riak.request.end": function(event) { - events.push(event) + events.push(event); } }; @@ -32,6 +33,10 @@ describe('http-client-instrumentation-tests', function() { done(); }); + after(function (done) { + helpers.cleanupBucket(bucket, done); + }); + it('Create an object', function(done) { db.save(bucket, 'someone@gmail.com', {name: 'Someone Else'}, function(err, doc, meta) { diff --git a/test/http-client-pool-test.js b/test/http-client-pool-test.js index 5a527b1..851d634 100644 --- a/test/http-client-pool-test.js +++ b/test/http-client-pool-test.js @@ -1,11 +1,13 @@ var HttpClient = require('../lib/http-client'), HttpMeta = require('../lib/http-meta'), - should = require('should'); + should = require('should'), + helpers = require('./test_helper.js'); -var db; +var db, bucket; describe('http-client-pool', function () { before(function(done) { + bucket = 'languages'; db = new HttpClient({ pool: { servers: [ 'localhost:8098', @@ -16,9 +18,13 @@ describe('http-client-pool', function () { done(); }); + after(function (done) { + helpers.cleanupBucket(bucket, done); + }); + it('Creates an object', function(done) { - db.save('languages', 'erlang', {type: 'functional'}, function(err) { - db.get('languages', 'erlang', function(err, data) { + db.save(bucket, 'erlang', {type: 'functional'}, function(err) { + db.get(bucket, 'erlang', function(err, data) { should.not.exist(err); data.type.should.equal('functional'); done(); @@ -27,7 +33,7 @@ describe('http-client-pool', function () { }); it('Fetches the streamed object', function(done) { - db.get('languages', 'erlang', {stream: true}, function(err, response, meta) { + db.get(bucket, 'erlang', {stream: true}, function(err, response, meta) { should.not.exist(err); response.on('data', function(data) { data = JSON.parse(String(data)); diff --git a/test/http-client-search-test.js b/test/http-client-search-test.js index 006e257..399f79e 100644 --- a/test/http-client-search-test.js +++ b/test/http-client-search-test.js @@ -1,5 +1,6 @@ var HttpClient = require('../lib/http-client'), - should = require('should'); + should = require('should'), + helpers = require('./test_helper'); var db, events = [], listener, bucket; @@ -13,6 +14,10 @@ describe('http-client-search-tests', function() { done(); }); + after(function (done) { + helpers.cleanupBucket(bucket, done); + }); + it('Save the properties of a bucket', function(done) { db.saveBucket(bucket, { search: true }, function(err) { should.not.exist(err); @@ -48,7 +53,7 @@ describe('http-client-search-tests', function() { }); }); - it('Map/Reduce with search', function(done) { + xit('Map/Reduce with search', function(done) { db.mapreduce.search(bucket, 'email:test-search@gmail.com') .map('Riak.mapValuesJson') .run(function(err, data) { @@ -61,7 +66,7 @@ describe('http-client-search-tests', function() { }); }); - it('Searching via Solr interface', function(done) { + xit('Searching via Solr interface', function(done) { db.search.find(bucket, 'email:test-search@gmail.com', function(err, data) { should.not.exist(err); should.exist(data); @@ -73,7 +78,7 @@ describe('http-client-search-tests', function() { }); }); - it('Add a document', function(done) { + xit('Add a document', function(done) { db.search.add(bucket, {id: 'test-add-search@gmail.com', name: 'Sean Cribbs'}, function(err) { should.not.exist(err); @@ -82,7 +87,7 @@ describe('http-client-search-tests', function() { }); }); - it('Find added document', function(done) { + xit('Find added document', function(done) { db.search.find(bucket, 'name:"Sean Cribbs"', function(err, data) { should.not.exist(err); should.exist(data); @@ -94,7 +99,7 @@ describe('http-client-search-tests', function() { }); }); - it('Remove the added document', function(done) { + xit('Remove the added document', function(done) { db.search.remove(bucket, {id: 'test-add-search@gmail.com'}, function(err) { should.not.exist(err); @@ -108,11 +113,4 @@ describe('http-client-search-tests', function() { }); }); }); - - after(function(done) { - db.remove(bucket, 'test-search@gmail.com'); - db.remove(bucket, 'test-add-search@gmail.com'); - - done(); - }); }); diff --git a/test/http-client-test.js b/test/http-client-test.js index 5072c5f..4326c2c 100644 --- a/test/http-client-test.js +++ b/test/http-client-test.js @@ -2,6 +2,7 @@ var HttpClient = require('../lib/http-client'), HttpMeta = require('../lib/http-meta'), async = require('async'), util = require('util'), + helpers = require('./test_helper'), should = require('should'); var db, db2, many = [], bucket; @@ -21,6 +22,12 @@ describe('http-client-tests', function() { done(); }); + after(function (done) { + helpers.cleanupBucket(bucket + '-keys', function () { + helpers.cleanupBucket(bucket, done); + }); + }); + it('Save with returnbody', function(done) { db.save(bucket, 'test-returnbody@gmail.com', { email: 'test@gmail.com', diff --git a/test/http-mapreduce-client-test.js b/test/http-mapreduce-client-test.js index 9439a84..2fa9dcb 100644 --- a/test/http-mapreduce-client-test.js +++ b/test/http-mapreduce-client-test.js @@ -1,7 +1,8 @@ var HttpClient = require('../lib/http-client'), HttpMeta = require('../lib/http-meta'), util = require('util'), - should = require('should'); + should = require('should'), + helpers = require('./test_helper'); var db, bucket; @@ -23,6 +24,10 @@ describe('http-mapreduce-client-tests', function() { }); }); + after(function (done) { + helpers.cleanupBucket(bucket, done); + }); + it('Map to an array of JSON objects', function(done) { db.mapreduce.add(bucket).map('Riak.mapValuesJson').run(function(err, data) { should.not.exist(err); diff --git a/test/protocol-buffers-client-test.js b/test/protocol-buffers-client-test.js index 7a4533b..a5bdffb 100644 --- a/test/protocol-buffers-client-test.js +++ b/test/protocol-buffers-client-test.js @@ -1,7 +1,9 @@ var ProtocolBuffersClient = require('../lib/protocol-buffers-client'), util = require('util'), + helpers = require('./test_helper'), should = require('should'); + var db; describe('protocol-buffers-client-tests', function() { @@ -11,8 +13,12 @@ describe('protocol-buffers-client-tests', function() { }); afterEach(function(done) { - db.end(); - done(); + helpers.cleanupBucket('pb-users', function () { + helpers.cleanupBucket('users', function () { + db.end(); + done(); + }); + }); }); it("Saves an object", function(done) { @@ -22,10 +28,12 @@ describe('protocol-buffers-client-tests', function() { }); it('Gets an object', function(done) { - db.get('pb-users', 'user@gmail.com', function(err, data, meta) { - should.not.exist(err); - data.name.should.equal('Joe Example'); - done(); + db.save('pb-users', 'user2@gmail.com', {name: 'Joe Example'}, {content_type: "application/json"}, function(data) { + db.get('pb-users', 'user2@gmail.com', function(err, data, meta) { + should.not.exist(err); + data.name.should.equal('Joe Example'); + done(); + }); }); }); diff --git a/test/protocol-buffers-mapreduce-test.js b/test/protocol-buffers-mapreduce-test.js index 567b644..1f33e82 100644 --- a/test/protocol-buffers-mapreduce-test.js +++ b/test/protocol-buffers-mapreduce-test.js @@ -1,11 +1,12 @@ var ProtocolBuffersClient = require('../lib/protocol-buffers-client'), - should = require('should'); + should = require('should'), + helpers = require('./test_helper'); var db, bucket; describe('protocol-buffers-mapreduce-client', function() { beforeEach(function(done) { - db = new ProtocolBuffersClient(); + db = new ProtocolBuffersClient(); bucket = 'map-pb-users-riak-js-tests'; db.save(bucket, 'test@gmail.com', {name: "Sean Cribbs"}, function(err, data, meta) { @@ -16,8 +17,10 @@ describe('protocol-buffers-mapreduce-client', function() { }); afterEach(function(done) { - db.end(); - done(); + helpers.cleanupBucket(bucket, function () { + db.end(); + done(); + }); }); it('Map to an array of JSON objects', function(done) { @@ -67,7 +70,7 @@ describe('protocol-buffers-mapreduce-client', function() { }).reduce('Riak.reduceLimit', 2) .run(function(err, data) { should.exist(data); - data[1].should.have.length(1) + data[1].should.have.length(1); done(); }); }); diff --git a/test/protocol-buffers-search-test.js b/test/protocol-buffers-search-test.js index 9062983..4aea24e 100644 --- a/test/protocol-buffers-search-test.js +++ b/test/protocol-buffers-search-test.js @@ -4,7 +4,7 @@ var ProtocolBuffersClient = require('../lib/protocol-buffers-client'), var db, http; -describe('protocol-buffers-search-client', function() { +xdescribe('protocol-buffers-search-client', function() { beforeEach(function(done) { db = new ProtocolBuffersClient(); http = new HttpClient(); @@ -12,18 +12,21 @@ describe('protocol-buffers-search-client', function() { db.save('pb-search', 'roidrage', {name: "Mathias Meyer"}, {content_type: "application/json"}, function(error, data) { done(); }); - }) + }); }); afterEach(function(done) { - db.end(); - done(); + helpers.cleanupBucket('pb-search', function() { + db.end(); + done(); + }); }); + it('Finds documents via search', function(done) { db.search.find('pb-search', 'name:Mathias*', function(error, data) { - data.docs[0].fields.should.include({name: 'Mathias Meyer'}) - data.docs[0].fields.should.include({id: 'roidrage'}) + data.docs[0].fields.should.include({name: 'Mathias Meyer'}); + data.docs[0].fields.should.include({id: 'roidrage'}); done(); }); }); diff --git a/test/test_helper.js b/test/test_helper.js new file mode 100644 index 0000000..ef195ce --- /dev/null +++ b/test/test_helper.js @@ -0,0 +1,30 @@ +var async = require('async'); +var HttpClient = require('../lib/http-client'); + +var helpers = module.exports = { + cleanupBucket: function (bucket, done) { + var db = new HttpClient({ port: 8098 }); + var allKeys = []; + var onKeys = function (keys) { + allKeys = allKeys.concat(keys); + }; + var onEnd = function () { + async.each( + allKeys, + function (key, cb) { + db.remove(bucket, key, function (err) { + cb(); + }); + }, + function (err) { + if (err) throw err; + done(); + } + ); + }; + db.keys(bucket) + .on('keys', onKeys) + .on('end', onEnd) + .start(); + } +}; From 35ceeff12d3f954e316a5a5aab8087336861f6a5 Mon Sep 17 00:00:00 2001 From: Philipp Fehre Date: Sat, 30 Nov 2013 12:39:21 +0100 Subject: [PATCH 10/27] some style fixes --- test/protocol-buffers-client-test.js | 10 +++++----- 1 file changed, 5 insertions(+), 5 deletions(-) diff --git a/test/protocol-buffers-client-test.js b/test/protocol-buffers-client-test.js index a5bdffb..2bfe515 100644 --- a/test/protocol-buffers-client-test.js +++ b/test/protocol-buffers-client-test.js @@ -71,7 +71,7 @@ describe('protocol-buffers-client-tests', function() { db.getBucket('users', function(err, properties, meta) { should.exist(properties); properties.n_val.should.equal(3); - should.exist(properties.allow_mult) + should.exist(properties.allow_mult); done(); }); }); @@ -84,11 +84,11 @@ describe('protocol-buffers-client-tests', function() { done(); }); }); - }) + }); it("Pings", function(done) { db.ping(function(err, pong) { - should.exist(pong) + should.exist(pong); done(); }); }); @@ -97,7 +97,7 @@ describe('protocol-buffers-client-tests', function() { db.ping(function(err, pong) { should.not.exist(err); done(); - }) + }); }); it("Doesn't set notFound on save", function(done) { @@ -113,7 +113,7 @@ describe('protocol-buffers-client-tests', function() { db.save('pb-users', 'user3@gmail.com', {name: 'Joe Example'}, {content_type: "application/json"}, function(data) { db.save('pb-users', 'user4@gmail.com', {name: 'Joe Example'}, {content_type: "application/json"}, function(data) { var keys = db.keys('pb-users', {keys: 'stream'}); - var result = [] + var result = []; keys.on('keys', function(keys) { result = result.concat(keys); }).on('end', function(data) { From a8bb95cf282607ae6f1d46e71cc95f9815db230c Mon Sep 17 00:00:00 2001 From: Philipp Fehre Date: Tue, 3 Dec 2013 11:56:56 +0100 Subject: [PATCH 11/27] remove the luwak tests --- test/http-client-luwak-test.js | 106 --------------------------------- 1 file changed, 106 deletions(-) delete mode 100644 test/http-client-luwak-test.js diff --git a/test/http-client-luwak-test.js b/test/http-client-luwak-test.js deleted file mode 100644 index 6668e83..0000000 --- a/test/http-client-luwak-test.js +++ /dev/null @@ -1,106 +0,0 @@ -var HttpClient = require('../lib/http-client'), - fs = require('fs'), - should = require('should'); - -var db, events = [], listener, - filename = __dirname + '/fixtures/cat.jpg', - filename2 = __dirname + '/fixtures/cat2.jpg', - filename3 = __dirname + '/fixtures/cat3.jpg', - image; - -//describe('http-client-luwak-tests', function() { -// before(function(done) { -// db = new HttpClient({ port: 8098 }); -// -// image = fs.readFileSync(filename); -// -// // Ensure unit tests don't collide with pre-existing buckets -// bucketName = 'users-riak-js-tests'; -// -// done(); -// }); -// -// it('Save a file from a buffer', function(done) { -// db.saveFile('cat2', image, -// { contentType: 'image/jpeg' }, -// function(err, data, meta) { -// should.not.exist(err); -// should.not.exist(data); -// should.exist(meta); -// meta.key.should.equal('cat2'); -// meta.statusCode.should.equal(204); -// -// // race condition - wait for riak -// setTimeout(function() { -// db.getFile('cat2', function(data) { -// data.should.be.an.instanceof(Buffer); -// -// // TODO compare data to image -// fs.writeFileSync(filename2, data); -// -// done(); -// }); -// }, 500); -// }); -// }); -// -// it('Remove a file', function(done) { -// db.removeFile('cat2', function(data) { -// // TODO what are we supposed to be checking here? -// done(); -// }); -// }); -// -// it('Save a file from a stream', function(done) { -// db.saveFile('cat3', -// fs.createReadStream(filename), -// { contentType: 'image/jpeg' }, -// function(err, data, meta) { -// should.not.exist(err); -// should.not.exist(data); -// should.exist(meta); -// meta.statusCode.should.equal(204); -// -// // race condition - wait for riak -// setTimeout(function() { -// db.getFile('cat3', { stream: true }, function(stream) { -// -// should.exist(stream); -// var out = fs.createWriteStream(filename3); -// stream.pipe(out); -// -// out.on('close', function() { -// done(); -// }); -// }); -// }, 500); -// }); -// }); -// -// // TODO test luwak with returnbody=true -// -// it('Remove the file stream', function(done) { -// db.removeFile('cat3', function(data) { -// // TODO what are we supposed to be checking here? -// done(); -// }); -// }); -// -// it('Buffers are equal', function(done) { -// var buf2 = fs.readFileSync(filename2), -// buf3 = fs.readFileSync(filename3); -// -// should.exist(buf2); -// should.exist(buf3); -// -// buf2.length.should.equal(buf3.length); -// -// // TODO should there be a better comparison here? -// -// // cleanup -// fs.unlinkSync(filename2); -// fs.unlinkSync(filename3); -// -// done(); -// }); -//}); From 4b7d486ff0cb5f623cbc976378f6c2fa8c2a513b Mon Sep 17 00:00:00 2001 From: Philipp Fehre Date: Tue, 3 Dec 2013 13:54:17 +0100 Subject: [PATCH 12/27] move the 1.4 specific tests to their own directory --- test/{ => 1.4-specific}/http-client-search-test.js | 0 test/{ => 1.4-specific}/protocol-buffers-search-test.js | 0 2 files changed, 0 insertions(+), 0 deletions(-) rename test/{ => 1.4-specific}/http-client-search-test.js (100%) rename test/{ => 1.4-specific}/protocol-buffers-search-test.js (100%) diff --git a/test/http-client-search-test.js b/test/1.4-specific/http-client-search-test.js similarity index 100% rename from test/http-client-search-test.js rename to test/1.4-specific/http-client-search-test.js diff --git a/test/protocol-buffers-search-test.js b/test/1.4-specific/protocol-buffers-search-test.js similarity index 100% rename from test/protocol-buffers-search-test.js rename to test/1.4-specific/protocol-buffers-search-test.js From 6b8c0d1b95617a8007169ac48fb5579a140a8bea Mon Sep 17 00:00:00 2001 From: Philipp Fehre Date: Tue, 3 Dec 2013 14:32:33 +0100 Subject: [PATCH 13/27] prepare version specific search tests --- Makefile | 19 +++++-- test/1.4-specific/http-client-search-test.js | 50 ++++++++++--------- .../protocol-buffers-search-test.js | 20 ++++---- test/2.0-specific/http-client-solr-test.js | 18 +++++++ 4 files changed, 72 insertions(+), 35 deletions(-) create mode 100644 test/2.0-specific/http-client-solr-test.js diff --git a/Makefile b/Makefile index 8b4a40f..515bdd0 100644 --- a/Makefile +++ b/Makefile @@ -1,13 +1,26 @@ -MOCHA_OPTS=-t 20000 test/*-test.js +MOCHA_OPTS=-t 20000 +TEST_SHARED=test/*-test.js +TEST_1_4=test/1.4-specific/*-test.js +TEST_2_0=test/2.0-specific/*-test.js REPORTER = spec check: test test: test-unit +test_2_0: test-unit-2.0 test-unit: @NODE_ENV=test ./node_modules/.bin/mocha \ --reporter $(REPORTER) \ - $(MOCHA_OPTS) + $(MOCHA_OPTS) \ + $(TEST_SHARED) \ + $(TEST_1_4) -.PHONY: test \ No newline at end of file +test-unit-2.0: + @NODE_ENV=test ./node_modules/.bin/mocha \ + --reporter $(REPORTER) \ + $(MOCHA_OPTS) \ + $(TEST_SHARED) \ + $(TEST_2_0) + +.PHONY: test diff --git a/test/1.4-specific/http-client-search-test.js b/test/1.4-specific/http-client-search-test.js index 399f79e..b42d5bb 100644 --- a/test/1.4-specific/http-client-search-test.js +++ b/test/1.4-specific/http-client-search-test.js @@ -1,6 +1,6 @@ -var HttpClient = require('../lib/http-client'), - should = require('should'), - helpers = require('./test_helper'); +var HttpClient = require('../../lib/http-client'), +should = require('should'), +helpers = require('./../test_helper'); var db, events = [], listener, bucket; @@ -10,8 +10,12 @@ describe('http-client-search-tests', function() { // Ensure unit tests don't collide with pre-existing buckets bucket = 'users-riak-js-tests'; - - done(); + var obj = { email: 'test-search@gmail.com', name: 'Testy Test for Riak Search' }; + db.saveBucket(bucket, {search: true}, function(error) { + db.save(bucket, 'test-search@gmail.com', obj, function(err, data, meta) { + done(); + }); + }); }); after(function (done) { @@ -37,36 +41,36 @@ describe('http-client-search-tests', function() { }); it('Save a document to a search enabled bucket', function(done) { - db.save(bucket, 'test-search@gmail.com', + db.save(bucket, 'test-search2@gmail.com', { - email: 'test-search@gmail.com', - name: 'Testy Test for Riak Search' + email: 'test-search2@gmail.com', + name: 'Testy Test 2 for Riak Search' }, function(err, data, meta) { should.not.exist(err); should.not.exist(data); should.exist(meta); - meta.key.should.equal('test-search@gmail.com'); + meta.key.should.equal('test-search2@gmail.com'); meta.statusCode.should.equal(204); done(); }); }); - xit('Map/Reduce with search', function(done) { + it('Map/Reduce with search', function(done) { db.mapreduce.search(bucket, 'email:test-search@gmail.com') - .map('Riak.mapValuesJson') - .run(function(err, data) { - should.not.exist(err); - should.exist(data); - data.length.should.equal(1); - data[0].email.should.equal('test-search@gmail.com'); + .map('Riak.mapValuesJson') + .run(function(err, data) { + should.not.exist(err); + should.exist(data); + data.length.should.equal(1); + data[0].email.should.equal('test-search@gmail.com'); - done(); - }); + done(); + }); }); - xit('Searching via Solr interface', function(done) { + it('Searching via Solr interface', function(done) { db.search.find(bucket, 'email:test-search@gmail.com', function(err, data) { should.not.exist(err); should.exist(data); @@ -78,16 +82,16 @@ describe('http-client-search-tests', function() { }); }); - xit('Add a document', function(done) { + it('Add a document', function(done) { db.search.add(bucket, {id: 'test-add-search@gmail.com', name: 'Sean Cribbs'}, function(err) { should.not.exist(err); done(); - }); + }); }); - xit('Find added document', function(done) { + it('Find added document', function(done) { db.search.find(bucket, 'name:"Sean Cribbs"', function(err, data) { should.not.exist(err); should.exist(data); @@ -99,7 +103,7 @@ describe('http-client-search-tests', function() { }); }); - xit('Remove the added document', function(done) { + it('Remove the added document', function(done) { db.search.remove(bucket, {id: 'test-add-search@gmail.com'}, function(err) { should.not.exist(err); diff --git a/test/1.4-specific/protocol-buffers-search-test.js b/test/1.4-specific/protocol-buffers-search-test.js index 4aea24e..e2b6ae8 100644 --- a/test/1.4-specific/protocol-buffers-search-test.js +++ b/test/1.4-specific/protocol-buffers-search-test.js @@ -1,22 +1,24 @@ -var ProtocolBuffersClient = require('../lib/protocol-buffers-client'), - HttpClient = require('../lib/http-client'), - should = require('should'); +var ProtocolBuffersClient = require('../../lib/protocol-buffers-client'), + HttpClient = require('../../lib/http-client'), + should = require('should'), + helpers = require('./../test_helper'); -var db, http; +var db, http, bucket; -xdescribe('protocol-buffers-search-client', function() { +describe('protocol-buffers-search-client', function() { beforeEach(function(done) { - db = new ProtocolBuffersClient(); + db = new ProtocolBuffersClient(); http = new HttpClient(); - http.saveBucket('pb-search', {search: true}, function(error) { - db.save('pb-search', 'roidrage', {name: "Mathias Meyer"}, {content_type: "application/json"}, function(error, data) { + bucket = 'pb-search'; + http.saveBucket(bucket, {search: true}, function(error) { + db.save(bucket, 'roidrage', {name: "Mathias Meyer"}, {content_type: "application/json"}, function(error, data) { done(); }); }); }); afterEach(function(done) { - helpers.cleanupBucket('pb-search', function() { + helpers.cleanupBucket(bucket, function() { db.end(); done(); }); diff --git a/test/2.0-specific/http-client-solr-test.js b/test/2.0-specific/http-client-solr-test.js new file mode 100644 index 0000000..6b279f8 --- /dev/null +++ b/test/2.0-specific/http-client-solr-test.js @@ -0,0 +1,18 @@ +var should = require('should'), + helpers = require('./../test_helper'); + +var db, bucket; + +describe('http-client-solr-tests', function() { + before(function(done) { + db = new HttpClient({ port: 8098 }); + + // Ensure unit tests don't collide with pre-existing buckets + bucket = 'users-riak-js-tests-solr'; + done(); + }); + + after(function (done) { + helpers.cleanupBucket(bucket, done); + }); +}); From 63a1214dc6149450b8758388b0544ea94886636e Mon Sep 17 00:00:00 2001 From: Philipp Fehre Date: Wed, 4 Dec 2013 18:28:56 +0100 Subject: [PATCH 14/27] first basic search via yokozuna passing --- lib/http-client.js | 2 + lib/http-search-meta.js | 8 +- lib/http-yokozuna-client.js | 103 ++++++++++++++++++ test/2.0-specific/http-client-solr-test.js | 18 --- .../2.0-specific/http-yokozuna-client-test.js | 66 +++++++++++ 5 files changed, 178 insertions(+), 19 deletions(-) create mode 100644 lib/http-yokozuna-client.js delete mode 100644 test/2.0-specific/http-client-solr-test.js create mode 100644 test/2.0-specific/http-yokozuna-client-test.js diff --git a/lib/http-client.js b/lib/http-client.js index a7c6fa5..82040cb 100644 --- a/lib/http-client.js +++ b/lib/http-client.js @@ -10,6 +10,7 @@ var http = require('http'), HttpMeta = require('./http-meta'), HttpRequest = require('./http-request'), HttpSearchClient = require('./http-search-client'), + HttpYokozunaClient = require('./http-yokozuna-client'), Mapper = require('./mapper'), HttpMapReduceClient = require('./http-mapreduce-client'), Pool = require('poolee'); @@ -29,6 +30,7 @@ var HttpClient = function HttpClient(options) { this._execute(meta) }.bind(this); this.search = new HttpSearchClient(this._defaults, execute); + this.yokozuna = new HttpYokozunaClient(this._defaults, execute); this.mapreduce = new HttpMapReduceClient(this._defaults, execute); var pool = this._defaults.pool; if (pool) { diff --git a/lib/http-search-meta.js b/lib/http-search-meta.js index 1538566..e6bafee 100644 --- a/lib/http-search-meta.js +++ b/lib/http-search-meta.js @@ -53,10 +53,16 @@ Object.defineProperty(HttpSearchMeta.prototype, 'path', { }.bind(this)); var queryString = querystring.stringify(queryParameters), - operation = "/" + this.operation, index = "/" + this.index, + operation, qs = queryString ? "?" + queryString : ''; + if(this.operation) { + operation = "/" + this.operation; + } else { + operation = ""; + } + return "/" + this.resource + index + operation + qs; }, enumerable: true diff --git a/lib/http-yokozuna-client.js b/lib/http-yokozuna-client.js new file mode 100644 index 0000000..2085160 --- /dev/null +++ b/lib/http-yokozuna-client.js @@ -0,0 +1,103 @@ +var HttpSearchMeta = require('./http-search-meta'); + +/** + * Initialize a Riak Yokozuna Search client. + * Expects only a callback that's call to execute a search query, e.g. + * in the HttpClient, therefore doesn't have any coupling to the + * underlying transport. + * + * @param {Object} options + * @param {Function} callback(meta) + */ +var HttpYokozunaClient = function HttpYokozunaClient(options, callback) { + this._execute = callback; + this._defaults = {resource: 'search', method: 'get'}; + + // need to add the options to the defaults object so as to not overwrite them + // for the calling http-client + + if (options.host) { + this._defaults.host = options.host; + } + + if (options.port) { + this._defaults.port = options.port; + } + + if (options.client) { + this._defaults.client = options.client; + } +}; + +/** + * Add a search index to yokozuna + * + * @param {String} index + * @param {String} schema + * @param {Function} callback(err) + * + * @api public + */ +HttpYokozunaClient.prototype.createIndex = function(index, schema, callback) { + if (typeof schema === 'function') { + callback = schema; + schema = null; + } + + var meta = new HttpSearchMeta(this._defaults, {resource: 'yz'}, {method: 'put', callback: callback, index: 'index', operation: index}); + + this._execute(meta); +}; + +/** + * Remove a search index from yokozuna + * + * @param {String} index + * @param {String} schema + * @param {Function} callback(err) + * + * @api public + */ +HttpYokozunaClient.prototype.getIndex = function(index, callback) { + var meta = new HttpSearchMeta(this._defaults, {resource: 'yz'}, {callback: callback, index: 'index', operation: index}); + + this._execute(meta); +}; + +/** + * Find a set of documents in Riak Yokozuna in the specified index. + * The query must be a valid query string as described in the Riak Yokozuna + * documentation. + * + * The options can include all query options supported by Riak Yokozuna. + * + * @param {String} index + * @param {String} query + * @param {Object} options + * @param {Function} callback(err, data, meta + */ +HttpYokozunaClient.prototype.find = function(index, query, options, callback) { + var meta = new HttpSearchMeta(this._defaults, options, {callback: callback, index: index, q: query, wt: 'json', }); + this._run(meta); +}; + + +HttpYokozunaClient.prototype._run = function(meta) { + var callback = meta.callback; + meta.callback = function(err, data, meta) { + if (err) { + return callback(err, data, meta); + } + + try { + data = JSON.parse(data.toString()); + } catch (e) { + return callback(e, e, meta); + } + + callback(err, data.response, meta); + }; + this._execute(meta); +}; + +module.exports = HttpYokozunaClient; diff --git a/test/2.0-specific/http-client-solr-test.js b/test/2.0-specific/http-client-solr-test.js deleted file mode 100644 index 6b279f8..0000000 --- a/test/2.0-specific/http-client-solr-test.js +++ /dev/null @@ -1,18 +0,0 @@ -var should = require('should'), - helpers = require('./../test_helper'); - -var db, bucket; - -describe('http-client-solr-tests', function() { - before(function(done) { - db = new HttpClient({ port: 8098 }); - - // Ensure unit tests don't collide with pre-existing buckets - bucket = 'users-riak-js-tests-solr'; - done(); - }); - - after(function (done) { - helpers.cleanupBucket(bucket, done); - }); -}); diff --git a/test/2.0-specific/http-yokozuna-client-test.js b/test/2.0-specific/http-yokozuna-client-test.js new file mode 100644 index 0000000..b207c37 --- /dev/null +++ b/test/2.0-specific/http-yokozuna-client-test.js @@ -0,0 +1,66 @@ +var HttpClient = require('../../lib/http-client'), + should = require('should'), + helpers = require('./../test_helper'); + +var db, bucket, yzIndex; + +describe('http-client-solr-tests', function() { + before(function(done) { + db = new HttpClient({ port: 8098 }); + + // Ensure unit tests don't collide with pre-existing buckets + bucket = 'users-riak-js-tests-solr'; + yzIndex = 'riak-js-yz-index-test'; + var obj = { + email_s: 'test-search@gmail.com', + name: 'Testy Test for Riak Search' + }; + db.saveBucket(bucket, { yz_index: yzIndex }, function (err) { + db.save(bucket, 'test-search@gmail.com', obj, function(err, data, meta) { + done(); + }); + }); + }); + + after(function (done) { + helpers.cleanupBucket(bucket, done); + }); + + it('creates an index', function (done) { + db.yokozuna.createIndex(yzIndex, function(err) { + db.yokozuna.getIndex(yzIndex, function (err, data) { + should.not.exist(err); + data.name.should.equal(yzIndex); + data.schema.should.equal('_yz_default'); + done(); + }); + }); + }); + + it('saves the index bucket property', function (done) { + db.saveBucket(bucket, { yz_index: yzIndex }, function (err) { + should.not.exist(err); + + done(); + }); + }); + + it('gets the index bucket property', function (done) { + db.getBucket(bucket, function (err, props) { + props.yz_index.should.equal(yzIndex); + done(); + }); + }); + + + it('Searching via Solr interface', function (done) { + db.yokozuna.find(yzIndex, 'email_s:test-search@gmail.com', function(err, data) { + should.not.exist(err); + should.exist(data); + + data.docs[0].email_s.should.equal('test-search@gmail.com'); + + done(); + }); + }); +}); From c6902191c718f075145dd8ff8d40048ca92b326a Mon Sep 17 00:00:00 2001 From: Mathias Meyer Date: Sun, 15 Dec 2013 16:37:02 +0100 Subject: [PATCH 15/27] Add matrix entry for Riak 2.0.0pre5. --- .travis.yml | 6 +++++- install_riak.sh | 16 ++++++++++++++++ 2 files changed, 21 insertions(+), 1 deletion(-) create mode 100755 install_riak.sh diff --git a/.travis.yml b/.travis.yml index c9d3322..ad19cc5 100644 --- a/.travis.yml +++ b/.travis.yml @@ -1,6 +1,10 @@ language: node_js +env: + - RIAK_VERSION=default + - RIAK_VERSION=2.0.0pre5 RIAK_RELEASE=2.0 +before_install: + - ./install_riak.sh node_js: - - "0.8" - "0.10" services: - riak diff --git a/install_riak.sh b/install_riak.sh new file mode 100755 index 0000000..f0ef320 --- /dev/null +++ b/install_riak.sh @@ -0,0 +1,16 @@ +#!/bin/sh + +if $RIAK_VERSION != "default" +then + echo "Stopping Riak service" + sudo service riak stop + sudo apt-get remove riak + riak_file="riak_$RIAK_VERSION-1_amd64.deb" + cd /tmp + echo "Downloading Riak $RIAK_VERSION" + wget http://s3.amazonaws.com/downloads.basho.com/riak/$RIAK_RELEASE/$RIAK_VERSION/ubuntu/precise/$riak_file + echo "Installing Riak $RIAK_VERSION" + sudo dpkg -i $riak_file + echo "Starting Riak..." + sudo service riak start +fi From b6c47fd0e8c2d48dd7f2cf46a0e37135a6f2a33c Mon Sep 17 00:00:00 2001 From: Mathias Meyer Date: Sun, 15 Dec 2013 16:41:39 +0100 Subject: [PATCH 16/27] Remove master exclusion for now. --- .travis.yml | 4 +--- 1 file changed, 1 insertion(+), 3 deletions(-) diff --git a/.travis.yml b/.travis.yml index ad19cc5..cb9e2e3 100644 --- a/.travis.yml +++ b/.travis.yml @@ -10,6 +10,4 @@ services: - riak script: - make test -branches: - only: - - master + From eb18bd999c59b74143088a93d37c95176636c480 Mon Sep 17 00:00:00 2001 From: Mathias Meyer Date: Sun, 15 Dec 2013 16:44:08 +0100 Subject: [PATCH 17/27] Escape if clause. --- install_riak.sh | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/install_riak.sh b/install_riak.sh index f0ef320..b39a366 100755 --- a/install_riak.sh +++ b/install_riak.sh @@ -1,6 +1,6 @@ #!/bin/sh -if $RIAK_VERSION != "default" +if [ $RIAK_VERSION != "default" ] then echo "Stopping Riak service" sudo service riak stop From 95b64d4ab949a68dae9afbe697aa8cbd4c0bdb54 Mon Sep 17 00:00:00 2001 From: Philipp Fehre Date: Tue, 17 Dec 2013 13:03:38 +0100 Subject: [PATCH 18/27] fix test for protocol buffer key fetch this should only return the keys added (4) since we clean the bucket between tests --- test/protocol-buffers-client-test.js | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/test/protocol-buffers-client-test.js b/test/protocol-buffers-client-test.js index 2bfe515..b5f8a79 100644 --- a/test/protocol-buffers-client-test.js +++ b/test/protocol-buffers-client-test.js @@ -117,7 +117,7 @@ describe('protocol-buffers-client-tests', function() { keys.on('keys', function(keys) { result = result.concat(keys); }).on('end', function(data) { - result.should.have.length(5); + result.should.have.length(4); done(); }); }); From 207dfe14850eb8733cfca75bacf17426166d0ada Mon Sep 17 00:00:00 2001 From: Philipp Fehre Date: Tue, 17 Dec 2013 13:14:34 +0100 Subject: [PATCH 19/27] it's enough to clean after all tests are done --- test/1.4-specific/protocol-buffers-search-test.js | 4 ++-- test/protocol-buffers-client-test.js | 4 ++-- test/protocol-buffers-mapreduce-test.js | 10 +++++----- 3 files changed, 9 insertions(+), 9 deletions(-) diff --git a/test/1.4-specific/protocol-buffers-search-test.js b/test/1.4-specific/protocol-buffers-search-test.js index e2b6ae8..8adf24d 100644 --- a/test/1.4-specific/protocol-buffers-search-test.js +++ b/test/1.4-specific/protocol-buffers-search-test.js @@ -6,7 +6,7 @@ var ProtocolBuffersClient = require('../../lib/protocol-buffers-client'), var db, http, bucket; describe('protocol-buffers-search-client', function() { - beforeEach(function(done) { + before(function(done) { db = new ProtocolBuffersClient(); http = new HttpClient(); bucket = 'pb-search'; @@ -17,7 +17,7 @@ describe('protocol-buffers-search-client', function() { }); }); - afterEach(function(done) { + after(function(done) { helpers.cleanupBucket(bucket, function() { db.end(); done(); diff --git a/test/protocol-buffers-client-test.js b/test/protocol-buffers-client-test.js index b5f8a79..5c8fef1 100644 --- a/test/protocol-buffers-client-test.js +++ b/test/protocol-buffers-client-test.js @@ -7,12 +7,12 @@ var ProtocolBuffersClient = require('../lib/protocol-buffers-client'), var db; describe('protocol-buffers-client-tests', function() { - beforeEach(function(done) { + before(function(done) { db = new ProtocolBuffersClient(); done(); }); - afterEach(function(done) { + after(function(done) { helpers.cleanupBucket('pb-users', function () { helpers.cleanupBucket('users', function () { db.end(); diff --git a/test/protocol-buffers-mapreduce-test.js b/test/protocol-buffers-mapreduce-test.js index 1f33e82..daf9493 100644 --- a/test/protocol-buffers-mapreduce-test.js +++ b/test/protocol-buffers-mapreduce-test.js @@ -5,7 +5,7 @@ var ProtocolBuffersClient = require('../lib/protocol-buffers-client'), var db, bucket; describe('protocol-buffers-mapreduce-client', function() { - beforeEach(function(done) { + before(function(done) { db = new ProtocolBuffersClient(); bucket = 'map-pb-users-riak-js-tests'; @@ -16,7 +16,7 @@ describe('protocol-buffers-mapreduce-client', function() { }); }); - afterEach(function(done) { + after(function(done) { helpers.cleanupBucket(bucket, function () { db.end(); done(); @@ -92,17 +92,17 @@ describe('protocol-buffers-mapreduce-client', function() { should.exist(err.message); should.exist(err.statusCode); done(); - }) + }); }); it('Supports chunked map/reduce', function(done) { var job = db.mapreduce.add(bucket).map('Riak.mapValuesJson').run({chunked: true}); var result = []; job.on('data', function(data) { - result.push(data) + result.push(data); }).on('end', function(data) { result.should.have.length(2); done(); }); - }) + }); }); From f3c51ea07906d7ee5543cfd42aa5f3646104925d Mon Sep 17 00:00:00 2001 From: Philipp Fehre Date: Tue, 17 Dec 2013 13:48:42 +0100 Subject: [PATCH 20/27] run 2.0 or 1.4 tests depending on the environment environment is set on Travis --- Makefile | 22 ++++++++-------------- 1 file changed, 8 insertions(+), 14 deletions(-) diff --git a/Makefile b/Makefile index 515bdd0..6040d9f 100644 --- a/Makefile +++ b/Makefile @@ -1,26 +1,20 @@ -MOCHA_OPTS=-t 20000 -TEST_SHARED=test/*-test.js -TEST_1_4=test/1.4-specific/*-test.js -TEST_2_0=test/2.0-specific/*-test.js +MOCHA_OPTS = -t 20000 REPORTER = spec +ifeq ($(RIAK_VERSION),default) +TESTS = test/*-test.js test/1.4-specific/*-test.js +else +TESTS = test/*-test.js test/2.0-specific/*-test.js +endif + check: test test: test-unit -test_2_0: test-unit-2.0 test-unit: @NODE_ENV=test ./node_modules/.bin/mocha \ --reporter $(REPORTER) \ $(MOCHA_OPTS) \ - $(TEST_SHARED) \ - $(TEST_1_4) - -test-unit-2.0: - @NODE_ENV=test ./node_modules/.bin/mocha \ - --reporter $(REPORTER) \ - $(MOCHA_OPTS) \ - $(TEST_SHARED) \ - $(TEST_2_0) + $(TESTS) .PHONY: test From 1cdfc745a2b228492d084268c160593bdc0c93ae Mon Sep 17 00:00:00 2001 From: Philipp Fehre Date: Tue, 17 Dec 2013 14:15:26 +0100 Subject: [PATCH 21/27] make sure yokozuna is enabled for 2.0 --- install_riak.sh | 2 ++ test/protocol-buffers-client-test.js | 4 ++-- 2 files changed, 4 insertions(+), 2 deletions(-) diff --git a/install_riak.sh b/install_riak.sh index b39a366..3fa167e 100755 --- a/install_riak.sh +++ b/install_riak.sh @@ -11,6 +11,8 @@ then wget http://s3.amazonaws.com/downloads.basho.com/riak/$RIAK_RELEASE/$RIAK_VERSION/ubuntu/precise/$riak_file echo "Installing Riak $RIAK_VERSION" sudo dpkg -i $riak_file + echo "Enabling yokozuna" + sudo sed -i'' 's/yokozuna = off/yokozuna = on/' /etc/riak/riak.conf echo "Starting Riak..." sudo service riak start fi diff --git a/test/protocol-buffers-client-test.js b/test/protocol-buffers-client-test.js index 5c8fef1..b5f8a79 100644 --- a/test/protocol-buffers-client-test.js +++ b/test/protocol-buffers-client-test.js @@ -7,12 +7,12 @@ var ProtocolBuffersClient = require('../lib/protocol-buffers-client'), var db; describe('protocol-buffers-client-tests', function() { - before(function(done) { + beforeEach(function(done) { db = new ProtocolBuffersClient(); done(); }); - after(function(done) { + afterEach(function(done) { helpers.cleanupBucket('pb-users', function () { helpers.cleanupBucket('users', function () { db.end(); From b68301b1539cbe614858793fb87b7a281ac75410 Mon Sep 17 00:00:00 2001 From: Philipp Fehre Date: Tue, 17 Dec 2013 14:28:58 +0100 Subject: [PATCH 22/27] make sure the key stream tests does not interfer only expect keys this tests explicitly saves --- test/protocol-buffers-client-test.js | 28 +++++++++++++++------------- 1 file changed, 15 insertions(+), 13 deletions(-) diff --git a/test/protocol-buffers-client-test.js b/test/protocol-buffers-client-test.js index b5f8a79..13b7448 100644 --- a/test/protocol-buffers-client-test.js +++ b/test/protocol-buffers-client-test.js @@ -7,12 +7,12 @@ var ProtocolBuffersClient = require('../lib/protocol-buffers-client'), var db; describe('protocol-buffers-client-tests', function() { - beforeEach(function(done) { + before(function(done) { db = new ProtocolBuffersClient(); done(); }); - afterEach(function(done) { + after(function(done) { helpers.cleanupBucket('pb-users', function () { helpers.cleanupBucket('users', function () { db.end(); @@ -108,17 +108,19 @@ describe('protocol-buffers-client-tests', function() { }); it("Fetches keys", function(done) { - db.save('pb-users', 'user1@gmail.com', {name: 'Joe Example'}, {content_type: "application/json"}, function(data) { - db.save('pb-users', 'user2@gmail.com', {name: 'Joe Example'}, {content_type: "application/json"}, function(data) { - db.save('pb-users', 'user3@gmail.com', {name: 'Joe Example'}, {content_type: "application/json"}, function(data) { - db.save('pb-users', 'user4@gmail.com', {name: 'Joe Example'}, {content_type: "application/json"}, function(data) { - var keys = db.keys('pb-users', {keys: 'stream'}); - var result = []; - keys.on('keys', function(keys) { - result = result.concat(keys); - }).on('end', function(data) { - result.should.have.length(4); - done(); + db.save('pb-users', 'user@gmail.com', {name: 'Joe Example'}, {content_type: "application/json"}, function(data) { + db.save('pb-users', 'user1@gmail.com', {name: 'Joe Example'}, {content_type: "application/json"}, function(data) { + db.save('pb-users', 'user2@gmail.com', {name: 'Joe Example'}, {content_type: "application/json"}, function(data) { + db.save('pb-users', 'user3@gmail.com', {name: 'Joe Example'}, {content_type: "application/json"}, function(data) { + db.save('pb-users', 'user4@gmail.com', {name: 'Joe Example'}, {content_type: "application/json"}, function(data) { + var keys = db.keys('pb-users', {keys: 'stream'}); + var result = []; + keys.on('keys', function(keys) { + result = result.concat(keys); + }).on('end', function(data) { + result.should.have.length(5); + done(); + }); }); }); }); From 73ed95baa0ef7e0ecbe5ee537d72a6e5ad02d711 Mon Sep 17 00:00:00 2001 From: Philipp Fehre Date: Tue, 17 Dec 2013 16:17:49 +0100 Subject: [PATCH 23/27] make sure the config files from riak are removed otherwise riak.conf will be ignored due to a present app.config --- install_riak.sh | 10 +++++++--- 1 file changed, 7 insertions(+), 3 deletions(-) diff --git a/install_riak.sh b/install_riak.sh index 3fa167e..6520eb3 100755 --- a/install_riak.sh +++ b/install_riak.sh @@ -4,15 +4,19 @@ if [ $RIAK_VERSION != "default" ] then echo "Stopping Riak service" sudo service riak stop - sudo apt-get remove riak + sudo apt-get purge riak riak_file="riak_$RIAK_VERSION-1_amd64.deb" cd /tmp echo "Downloading Riak $RIAK_VERSION" wget http://s3.amazonaws.com/downloads.basho.com/riak/$RIAK_RELEASE/$RIAK_VERSION/ubuntu/precise/$riak_file echo "Installing Riak $RIAK_VERSION" sudo dpkg -i $riak_file - echo "Enabling yokozuna" - sudo sed -i'' 's/yokozuna = off/yokozuna = on/' /etc/riak/riak.conf + echo "Enabling Yokozuna" + sudo sed -i 's/yokozuna = off/yokozuna = on/' /etc/riak/riak.conf + echo "Use leveldb as the backend" + sudo sed -i 's/storage_backend = bitcask/storage_backend = leveldb/' /etc/riak/riak.conf + echo "Set ulimit" + sudo sh -c 'echo "ulimit -n 65536" > /etc/default/riak' echo "Starting Riak..." sudo service riak start fi From 0ac9d6c1bec1ce18592df16ce749228467451a67 Mon Sep 17 00:00:00 2001 From: Philipp Fehre Date: Tue, 17 Dec 2013 16:18:10 +0100 Subject: [PATCH 24/27] wait for yokozuna index to propagate --- .../2.0-specific/http-yokozuna-client-test.js | 24 +++++++++++-------- 1 file changed, 14 insertions(+), 10 deletions(-) diff --git a/test/2.0-specific/http-yokozuna-client-test.js b/test/2.0-specific/http-yokozuna-client-test.js index b207c37..de8a40f 100644 --- a/test/2.0-specific/http-yokozuna-client-test.js +++ b/test/2.0-specific/http-yokozuna-client-test.js @@ -15,9 +15,15 @@ describe('http-client-solr-tests', function() { email_s: 'test-search@gmail.com', name: 'Testy Test for Riak Search' }; - db.saveBucket(bucket, { yz_index: yzIndex }, function (err) { - db.save(bucket, 'test-search@gmail.com', obj, function(err, data, meta) { - done(); + db.yokozuna.createIndex(yzIndex, function(err) { + db.saveBucket(bucket, { yz_index: yzIndex }, function (err) { + setTimeout(function () { + db.save(bucket, 'test-search@gmail.com', obj, function(err, data, meta) { + setTimeout(function () { + done(); + }, 1000); + }); + }, 1000); }); }); }); @@ -27,13 +33,11 @@ describe('http-client-solr-tests', function() { }); it('creates an index', function (done) { - db.yokozuna.createIndex(yzIndex, function(err) { - db.yokozuna.getIndex(yzIndex, function (err, data) { - should.not.exist(err); - data.name.should.equal(yzIndex); - data.schema.should.equal('_yz_default'); - done(); - }); + db.yokozuna.getIndex(yzIndex, function (err, data) { + should.not.exist(err); + data.name.should.equal(yzIndex); + data.schema.should.equal('_yz_default'); + done(); }); }); From 7683743ae6b5712a103e7673dd409fffbfbf0c24 Mon Sep 17 00:00:00 2001 From: Philipp Fehre Date: Tue, 17 Dec 2013 16:25:01 +0100 Subject: [PATCH 25/27] is the travis vm that much slower? locally 1 sec is more than enough wait for indexes to propagate but on travis this seems to not be the case let's see... --- test/2.0-specific/http-yokozuna-client-test.js | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/test/2.0-specific/http-yokozuna-client-test.js b/test/2.0-specific/http-yokozuna-client-test.js index de8a40f..52d375d 100644 --- a/test/2.0-specific/http-yokozuna-client-test.js +++ b/test/2.0-specific/http-yokozuna-client-test.js @@ -21,9 +21,9 @@ describe('http-client-solr-tests', function() { db.save(bucket, 'test-search@gmail.com', obj, function(err, data, meta) { setTimeout(function () { done(); - }, 1000); + }, 4000); }); - }, 1000); + }, 4000); }); }); }); From 7f0efbd3826a4758e75eeae6006786ff8215f724 Mon Sep 17 00:00:00 2001 From: Alexander Popov Date: Wed, 24 Sep 2014 15:54:13 +0400 Subject: [PATCH 26/27] fix error handling --- lib/http-request.js | 8 +++++++- 1 file changed, 7 insertions(+), 1 deletion(-) diff --git a/lib/http-request.js b/lib/http-request.js index a5239a7..fa7e92e 100644 --- a/lib/http-request.js +++ b/lib/http-request.js @@ -236,7 +236,13 @@ HttpRequest.prototype.handleRequestEnd = function(buffer) { // deal with errors if (this.meta.statusCode >= 400) { var err = new Error(); - err.message = buffer && JSON.stringify( buffer ).trim(); + if ( buffer ){ + if ( Array.isArray( buffer ) || buffer.length && Object.prototype.toString.call(buffer[0]) == '[object Number]' ) + for ( var i = 0; i< buffer.length; i++ ) + err.message += String.fromCharCode(buffer[i]) + else + err.message = JSON.stringify( buffer ).trim(); + } err.statusCode = this.meta.statusCode; buffer = err; } From 02742bf2525de36816dfc61b81d96863f185d3df Mon Sep 17 00:00:00 2001 From: Alexander Popov Date: Thu, 20 Nov 2014 13:07:33 +0300 Subject: [PATCH 27/27] support for Yokozuna, and some Solr features --- lib/http-client.js | 2 +- lib/http-request.js | 4 +++- lib/http-search-client.js | 4 ++-- lib/http-search-meta.js | 15 ++++++++++++++- lib/http-yokozuna-client.js | 12 +++++++++--- lib/meta.js | 16 ++++++++++++++-- 6 files changed, 43 insertions(+), 10 deletions(-) diff --git a/lib/http-client.js b/lib/http-client.js index e25c4fd..99a7d32 100644 --- a/lib/http-client.js +++ b/lib/http-client.js @@ -367,7 +367,7 @@ HttpClient.prototype.query = function(bucket, query, options, callback) { if (end) key += "/" + encodeURIComponent(end); this.get(bucket, key, meta, function(err, data, _meta) { - var continuation = data.continuation; + var continuation = data && data.continuation; data = data ? data.keys : undefined; if ( continuation && data ) data.continuation = continuation; diff --git a/lib/http-request.js b/lib/http-request.js index fa7e92e..2375e90 100644 --- a/lib/http-request.js +++ b/lib/http-request.js @@ -60,6 +60,8 @@ HttpRequest.prototype.execute = function() { this.client.emit('riak.request.start', this.event) + + var request = httpClient.request(meta, function(err, response) { var normalstatus = [200,201,204,300,304]; if(self.event.method == 'delete') normalstatus.push(404); @@ -241,7 +243,7 @@ HttpRequest.prototype.handleRequestEnd = function(buffer) { for ( var i = 0; i< buffer.length; i++ ) err.message += String.fromCharCode(buffer[i]) else - err.message = JSON.stringify( buffer ).trim(); + err.message = ( typeof buffer == "string" ? buffer : JSON.stringify( buffer ) ).trim(); } err.statusCode = this.meta.statusCode; buffer = err; diff --git a/lib/http-search-client.js b/lib/http-search-client.js index 1c4ca9b..a0b54fe 100644 --- a/lib/http-search-client.js +++ b/lib/http-search-client.js @@ -12,7 +12,7 @@ var XML = require('xml'), */ var HttpSearchClient = function HttpSearchClient(options, callback) { this._execute = callback; - this._defaults = {resource: 'solr', method: 'get'}; + this._defaults = {resource: 'search/query', method: 'get'}; // need to add the options to the defaults object so as to not overwrite them // for the calling http-client @@ -78,7 +78,7 @@ HttpSearchClient.prototype.remove = function(index, documents, callback) { * @param {Function} callback(err, data, meta */ HttpSearchClient.prototype.find = function(index, query, options, callback) { - var meta = new HttpSearchMeta(this._defaults, options, {callback: callback, index: index, operation: 'select', q: query, wt: 'json', }); + var meta = new HttpSearchMeta(this._defaults, options, {callback: callback, index: index, operation: '', q: query, wt: 'json' }); this._run(meta); } diff --git a/lib/http-search-meta.js b/lib/http-search-meta.js index e6bafee..236f80e 100644 --- a/lib/http-search-meta.js +++ b/lib/http-search-meta.js @@ -90,13 +90,26 @@ var responseMappings = { HttpSearchMeta.queryProperties = [ 'q', + 'fq', 'start', 'rows', 'wt', 'sort', 'presort', 'filter', - 'fl' + 'fl', + 'df', + 'facet', //TODO: SOLR have huge ammount of options. nightmare to specify it all + 'facet.field', + 'facet.query', + 'stats', + 'stats.field', + 'group', + 'group.field', + 'group.main', + 'group.sort', + 'group.limit' + ]; module.exports = HttpSearchMeta; diff --git a/lib/http-yokozuna-client.js b/lib/http-yokozuna-client.js index 2085160..905d6fe 100644 --- a/lib/http-yokozuna-client.js +++ b/lib/http-yokozuna-client.js @@ -11,7 +11,7 @@ var HttpSearchMeta = require('./http-search-meta'); */ var HttpYokozunaClient = function HttpYokozunaClient(options, callback) { this._execute = callback; - this._defaults = {resource: 'search', method: 'get'}; + this._defaults = {resource: 'search/query', method: 'get'}; // need to add the options to the defaults object so as to not overwrite them // for the calling http-client @@ -42,6 +42,7 @@ HttpYokozunaClient.prototype.createIndex = function(index, schema, callback) { if (typeof schema === 'function') { callback = schema; schema = null; + schema = null; } var meta = new HttpSearchMeta(this._defaults, {resource: 'yz'}, {method: 'put', callback: callback, index: 'index', operation: index}); @@ -77,7 +78,7 @@ HttpYokozunaClient.prototype.getIndex = function(index, callback) { * @param {Function} callback(err, data, meta */ HttpYokozunaClient.prototype.find = function(index, query, options, callback) { - var meta = new HttpSearchMeta(this._defaults, options, {callback: callback, index: index, q: query, wt: 'json', }); + var meta = new HttpSearchMeta(this._defaults, options, {callback: callback, index: index, q: query, wt: 'json' }); this._run(meta); }; @@ -94,8 +95,13 @@ HttpYokozunaClient.prototype._run = function(meta) { } catch (e) { return callback(e, e, meta); } + if ( !data.response) data.response = {}; + + data.response.facet_counts = data.facet_counts; + data.response.stats = data.stats; + data.response.grouped = data.grouped; - callback(err, data.response, meta); + callback(err, data.response, meta); }; this._execute(meta); }; diff --git a/lib/meta.js b/lib/meta.js index 974fc22..70eb4d1 100644 --- a/lib/meta.js +++ b/lib/meta.js @@ -108,8 +108,9 @@ Meta.prototype.serialize = function(data) { * * @api public */ + Meta.prototype.loadData = function(data) { - if (data) data = this.serialize(data); + if (data!=null) data = this.serialize(data); // cannot check like if(data), Riak 2.0 accept numeric body for counters, and its failed with counter value 0 this.data = data; } @@ -204,6 +205,7 @@ Meta.keywords = [ 'callback', 'method', 'q', + 'fq', 'start', 'rows', 'wt', @@ -211,7 +213,17 @@ Meta.keywords = [ 'presort', 'filter', 'fl', - 'noError404', + 'df', + 'facet', //TODO: SOLR have huge ammount of options. nightmare to specify it all + 'facet.field', + 'facet.query', + 'group', + 'group.field', + 'group.main', + 'group.sort', + 'group.limit', + 'stats', + 'stats.field', 'date', 'operation', 'pool',