From 5dd48e267ce7604446aa364a003c86f0ebd3f3d3 Mon Sep 17 00:00:00 2001 From: Bharath Reddy Kontham Date: Sun, 3 May 2020 20:11:45 +0530 Subject: [PATCH] release(2.1.0): search_after support added (#134) * task(Readme): readme file update * fix(eslint): all lint issues, remove unused files * fix(eslint): constructor, prototype issues for ESConnector class * fix(lint): lint issues fixed in all.js and buildOrder.js * feature(Client): all client APIs updated * fix(replaceOrCreate): response fix * task(README, examples): update README and example server * Release: Version 2.0.0 * task(filter): added _source filter support for fields * release(2.0.1): version update * task(search_after): add elasticsearch search after support in filter * task(search-after): updated readme * release(2.1.0): search_after support added --- README.md | 58 +++++++++++++++++++++++++- examples/common/models/user-model.json | 6 ++- lib/buildFilter.js | 5 +++ lib/create.js | 5 +++ lib/esConnector.js | 8 +++- lib/find.js | 4 +- lib/replaceById.js | 5 +++ lib/replaceOrCreate.js | 5 +++ lib/save.js | 5 +++ lib/updateAll.js | 4 +- lib/updateAttributes.js | 4 +- lib/updateOrCreate.js | 5 +++ package.json | 2 +- 13 files changed, 107 insertions(+), 9 deletions(-) diff --git a/README.md b/README.md index 5e793ac..55f3997 100644 --- a/README.md +++ b/README.md @@ -14,7 +14,9 @@ Elasticsearch(versions 6.x and 7.x) datasource connector for [Loopback 3.x](http - [Recommended properties](#recommended) - [Optional properties](#optional) - [Sample for copy paste](#sample) -- [How to achieve Instant search](#how-to-achieve-instant-search) + +- [Elasticsearch SearchAfter Support](#elasticsearch-searchafter-support) +- [Example](#about-the-example-app) - [Troubleshooting](#troubleshooting) - [Contributing](#contributing) - [Frequently Asked Questions](#faqs) @@ -154,6 +156,60 @@ npm install loopback-connector-esv6 --save --save-exact 2.You can peek at `/examples/server/datasources.json` for more hints. +## Elasticsearch SearchAfter Support + +- ```search_after``` feature of elasticsearch is supported in loopback filter. +- For this, you need to create a property in model called ```_search_after``` with loopback type ```["any"]```. This field cannot be updated using in connector. +- Elasticsearch ```sort``` value will return in this field. +- You need pass ```_search_after``` value in ```searchafter``` key of loopback filter. +- Example filter query for ```find```. + +```json +{ + "where": { + "username": "hello" + }, + "order": "created DESC", + "searchafter": [ + 1580902552905 + ], + "limit": 4 +} +``` + +- Example result. + +```json +[ + { + "id": "1bb2dd63-c7b9-588e-a942-15ca4f891a80", + "username": "test", + "_search_after": [ + 1580902552905 + ], + "created": "2020-02-05T11:35:52.905Z" + }, + { + "id": "fd5ea4df-f159-5816-9104-22147f2a740f", + "username": "test3", + "_search_after": [ + 1580902552901 + ], + "created": "2020-02-05T11:35:52.901Z" + }, + { + "id": "047c0adb-13ea-5f80-a772-3d2a4691d47a", + "username": "test4", + "_search_after": [ + 1580902552897 + ], + "created": "2020-02-05T11:35:52.897Z" + } +] +``` + +- This is useful for pagination. To go to previous page, change sorting order. + ## About the example app 1. The `examples` directory contains a loopback app which uses this connector. diff --git a/examples/common/models/user-model.json b/examples/common/models/user-model.json index ef04362..7d81387 100644 --- a/examples/common/models/user-model.json +++ b/examples/common/models/user-model.json @@ -2,7 +2,11 @@ "name": "UserModel", "base": "User", "idInjection": true, - "properties": {}, + "properties": { + "_search_after": { + "type": ["any"] + } + }, "validations": [], "relations": { "globalConfigModels": { diff --git a/lib/buildFilter.js b/lib/buildFilter.js index a28bbf2..fb8f29f 100644 --- a/lib/buildFilter.js +++ b/lib/buildFilter.js @@ -44,6 +44,11 @@ function buildFilter(modelName, idName, criteria = {}, size = null, offset = nul }; } } + if (criteria.searchafter && Array.isArray(criteria.searchafter) + && criteria.searchafter.length) { + filter.body.search_after = criteria.searchafter; + filter.from = undefined; + } if (criteria.order) { log('ESConnector.prototype.buildFilter', 'will delegate sorting to buildOrder()'); filter.body.sort = self.buildOrder(modelName, idName, criteria.order); diff --git a/lib/create.js b/lib/create.js index 43f4644..2f5ec71 100644 --- a/lib/create.js +++ b/lib/create.js @@ -1,5 +1,7 @@ const _ = require('lodash'); const log = require('debug')('loopback:connector:elasticsearch'); +// CONSTANTS +const SEARCHAFTERKEY = '_search_after'; function create(model, data, done) { const self = this; @@ -26,6 +28,9 @@ function create(model, data, done) { method = 'index'; // if there is no/empty id field, we must use the index method to create it (API 5.0) } document.body.docType = model; + if (document.body[SEARCHAFTERKEY]) { + document.body[SEARCHAFTERKEY] = undefined; + } self.db[method]( document ).then( diff --git a/lib/esConnector.js b/lib/esConnector.js index 107ee44..08f30db 100755 --- a/lib/esConnector.js +++ b/lib/esConnector.js @@ -35,6 +35,9 @@ const { updateAll } = require('./updateAll'); const { updateAttributes } = require('./updateAttributes'); const { updateOrCreate } = require('./updateOrCreate'); +// CONSTANTS +const SEARCHAFTERKEY = '_search_after'; + /** * Connector constructor * @param {object} datasource settings @@ -212,7 +215,7 @@ ESConnector.prototype.getValueFromProperty = function (property, value) { * @param {Object} data from DB * @returns {object} modeled document */ -ESConnector.prototype.matchDataToModel = function (modelName, data, esId, idName) { +ESConnector.prototype.matchDataToModel = function (modelName, data, esId, idName, sort) { /* log('ESConnector.prototype.matchDataToModel', 'modelName', modelName, 'data', JSON.stringify(data,null,0)); @@ -239,6 +242,7 @@ ESConnector.prototype.matchDataToModel = function (modelName, data, esId, idName ); } }); + document[SEARCHAFTERKEY] = sort; log('ESConnector.prototype.matchDataToModel', 'document', JSON.stringify(document, null, 0)); return document; } catch (err) { @@ -258,7 +262,7 @@ ESConnector.prototype.dataSourceToModel = function (modelName, data, idName) { // return data._source; // TODO: super-simplify? // eslint-disable-next-line no-underscore-dangle - return this.matchDataToModel(modelName, data._source, data._id, idName); + return this.matchDataToModel(modelName, data._source, data._id, idName, data.sort || []); }; /** diff --git a/lib/find.js b/lib/find.js index 614a8df..a6b0433 100644 --- a/lib/find.js +++ b/lib/find.js @@ -8,12 +8,12 @@ function find(modelName, id, done) { if (id === undefined || id === null) { throw new Error('id not set!'); } - + const idName = self.idName(modelName); const defaults = self.addDefaults(modelName, 'find'); self.db.get(_.defaults({ id: self.getDocumentId(id) }, defaults)).then(({ body }) => { - done(null, self.dataSourceToModel(modelName, body)); + done(null, self.dataSourceToModel(modelName, body, idName)); }).catch((error) => { log('ESConnector.prototype.find', error.message); done(error); diff --git a/lib/replaceById.js b/lib/replaceById.js index 5a55c69..ba65e99 100644 --- a/lib/replaceById.js +++ b/lib/replaceById.js @@ -1,5 +1,7 @@ const _ = require('lodash'); const log = require('debug')('loopback:connector:elasticsearch'); +// CONSTANTS +const SEARCHAFTERKEY = '_search_after'; function replaceById(modelName, id, data, options, callback) { const self = this; @@ -21,6 +23,9 @@ function replaceById(modelName, id, data, options, callback) { if (Object.prototype.hasOwnProperty.call(modelProperties, idName)) { document.body[idName] = id; } + if (document.body[SEARCHAFTERKEY]) { + document.body[SEARCHAFTERKEY] = undefined; + } log('ESConnector.prototype.replaceById', 'document', document); self.db.index( document diff --git a/lib/replaceOrCreate.js b/lib/replaceOrCreate.js index d025c12..fd71c15 100644 --- a/lib/replaceOrCreate.js +++ b/lib/replaceOrCreate.js @@ -1,5 +1,7 @@ const _ = require('lodash'); const log = require('debug')('loopback:connector:elasticsearch'); +// CONSTANTS +const SEARCHAFTERKEY = '_search_after'; function replaceOrCreate(modelName, data, callback) { const self = this; @@ -16,6 +18,9 @@ function replaceOrCreate(modelName, data, callback) { document.body = {}; _.assign(document.body, data); document.body.docType = modelName; + if (document.body[SEARCHAFTERKEY]) { + document.body[SEARCHAFTERKEY] = undefined; + } log('ESConnector.prototype.replaceOrCreate', 'document', document); self.db.index( document diff --git a/lib/save.js b/lib/save.js index 22b5740..85d8051 100644 --- a/lib/save.js +++ b/lib/save.js @@ -1,5 +1,7 @@ const _ = require('lodash'); const log = require('debug')('loopback:connector:elasticsearch'); +// CONSTANTS +const SEARCHAFTERKEY = '_search_after'; // eslint-disable-next-line consistent-return function save(model, data, done) { @@ -16,6 +18,9 @@ function save(model, data, done) { return done('Document id not setted!', null); } data.docType = model; + if (data[SEARCHAFTERKEY]) { + data[SEARCHAFTERKEY] = undefined; + } self.db.update(_.defaults({ id, body: { diff --git a/lib/updateAll.js b/lib/updateAll.js index 332df7d..1accb1c 100644 --- a/lib/updateAll.js +++ b/lib/updateAll.js @@ -1,5 +1,7 @@ const _ = require('lodash'); const log = require('debug')('loopback:connector:elasticsearch'); +// CONSTANTS +const SEARCHAFTERKEY = '_search_after'; function updateAll(model, where, data, options, cb) { const self = this; @@ -20,7 +22,7 @@ function updateAll(model, where, data, options, cb) { params: {} }; _.forEach(data, (value, key) => { - if (key !== '_id' || key !== idName) { + if (key !== '_id' && key !== idName && key !== SEARCHAFTERKEY) { // default language for inline scripts is painless if ES 5, so this needs the extra params. reqBody.script.inline += `ctx._source.${key}=params.${key};`; reqBody.script.params[key] = value; diff --git a/lib/updateAttributes.js b/lib/updateAttributes.js index bdf5b53..c1b577e 100644 --- a/lib/updateAttributes.js +++ b/lib/updateAttributes.js @@ -1,5 +1,7 @@ const _ = require('lodash'); const log = require('debug')('loopback:connector:elasticsearch'); +// CONSTANTS +const SEARCHAFTERKEY = '_search_after'; function updateAttributes(modelName, id, data, callback) { const self = this; @@ -22,7 +24,7 @@ function updateAttributes(modelName, id, data, callback) { params: {} }; _.forEach(data, (value, key) => { - if (key !== '_id' || key !== idName) { + if (key !== '_id' && key !== idName && key !== SEARCHAFTERKEY) { // default language for inline scripts is painless if ES 5, so this needs the extra params. reqBody.script.inline += `ctx._source.${key}=params.${key};`; reqBody.script.params[key] = value; diff --git a/lib/updateOrCreate.js b/lib/updateOrCreate.js index 8eb477a..f67f1e3 100644 --- a/lib/updateOrCreate.js +++ b/lib/updateOrCreate.js @@ -1,5 +1,7 @@ const _ = require('lodash'); const log = require('debug')('loopback:connector:elasticsearch'); +// CONSTANTS +const SEARCHAFTERKEY = '_search_after'; function updateOrCreate(modelName, data, callback) { const self = this; @@ -13,6 +15,9 @@ function updateOrCreate(modelName, data, callback) { const defaults = self.addDefaults(modelName, 'updateOrCreate'); data.docType = modelName; + if (data[SEARCHAFTERKEY]) { + data[SEARCHAFTERKEY] = undefined; + } self.db.update(_.defaults({ id, body: { diff --git a/package.json b/package.json index ce9ae20..cddb630 100644 --- a/package.json +++ b/package.json @@ -1,6 +1,6 @@ { "name": "loopback-connector-esv6", - "version": "2.0.1", + "version": "2.1.0", "description": "LoopBack Connector for Elasticsearch 6.x and 7.x", "main": "index.js", "scripts": {