release(2.1.0): search_after support added (#134)

* task(Readme): readme file update * fix(eslint): all lint issues, remove unused files * fix(eslint): constructor, prototype issues for ESConnector class * fix(lint): lint issues fixed in all.js and buildOrder.js * feature(Client): all client APIs updated * fix(replaceOrCreate): response fix * task(README, examples): update README and example server * Release: Version 2.0.0 * task(filter): added _source filter support for fields * release(2.0.1): version update * task(search_after): add elasticsearch search after support in filter * task(search-after): updated readme * release(2.1.0): search_after support added
loopbackio · May 3, 2020 · 5dd48e2 · 5dd48e2
1 parent e8140ca
commit 5dd48e2
Show file tree

Hide file tree

Showing 13 changed files with 107 additions and 9 deletions.
diff --git a/README.md b/README.md
@@ -14,7 +14,9 @@ Elasticsearch(versions 6.x and 7.x) datasource connector for [Loopback 3.x](http
   - [Recommended properties](#recommended)
   - [Optional properties](#optional)
   - [Sample for copy paste](#sample)
-- [How to achieve Instant search](#how-to-achieve-instant-search)
+
+- [Elasticsearch SearchAfter Support](#elasticsearch-searchafter-support)
+- [Example](#about-the-example-app)
 - [Troubleshooting](#troubleshooting)
 - [Contributing](#contributing)
 - [Frequently Asked Questions](#faqs)
@@ -154,6 +156,60 @@ npm install loopback-connector-esv6 --save --save-exact
 
 2.You can peek at `/examples/server/datasources.json` for more hints.
 
+## Elasticsearch SearchAfter Support
+
+- ```search_after``` feature of elasticsearch is supported in loopback filter.
+- For this, you need to create a property in model called ```_search_after``` with loopback type ```["any"]```. This field cannot be updated using in connector.
+- Elasticsearch ```sort``` value will return in this field.
+- You need pass ```_search_after``` value in ```searchafter``` key of loopback filter.
+- Example filter query for ```find```.
+
+```json
+{
+  "where": {
+    "username": "hello"
+  },
+  "order": "created DESC",
+  "searchafter": [
+    1580902552905
+  ],
+  "limit": 4
+}
+```
+
+- Example result.
+
+```json
+[
+  {
+    "id": "1bb2dd63-c7b9-588e-a942-15ca4f891a80",
+    "username": "test",
+    "_search_after": [
+      1580902552905
+    ],
+    "created": "2020-02-05T11:35:52.905Z"
+  },
+  {
+    "id": "fd5ea4df-f159-5816-9104-22147f2a740f",
+    "username": "test3",
+    "_search_after": [
+      1580902552901
+    ],
+    "created": "2020-02-05T11:35:52.901Z"
+  },
+  {
+    "id": "047c0adb-13ea-5f80-a772-3d2a4691d47a",
+    "username": "test4",
+    "_search_after": [
+      1580902552897
+    ],
+    "created": "2020-02-05T11:35:52.897Z"
+  }
+]
+```
+
+- This is useful for pagination. To go to previous page, change sorting order.
+
 ## About the example app
 
 1. The `examples` directory contains a loopback app which uses this connector.

diff --git a/examples/common/models/user-model.json b/examples/common/models/user-model.json
@@ -2,7 +2,11 @@
   "name": "UserModel",
   "base": "User",
   "idInjection": true,
-  "properties": {},
+  "properties": {
+    "_search_after": {
+      "type": ["any"]
+    }
+  },
   "validations": [],
   "relations": {
     "globalConfigModels": {

diff --git a/lib/buildFilter.js b/lib/buildFilter.js
@@ -44,6 +44,11 @@ function buildFilter(modelName, idName, criteria = {}, size = null, offset = nul
         };
       }
     }
+    if (criteria.searchafter && Array.isArray(criteria.searchafter)
+    && criteria.searchafter.length) {
+      filter.body.search_after = criteria.searchafter;
+      filter.from = undefined;
+    }
     if (criteria.order) {
       log('ESConnector.prototype.buildFilter', 'will delegate sorting to buildOrder()');
       filter.body.sort = self.buildOrder(modelName, idName, criteria.order);

diff --git a/lib/create.js b/lib/create.js
@@ -1,5 +1,7 @@
 const _ = require('lodash');
 const log = require('debug')('loopback:connector:elasticsearch');
+// CONSTANTS
+const SEARCHAFTERKEY = '_search_after';
 
 function create(model, data, done) {
   const self = this;
@@ -26,6 +28,9 @@ function create(model, data, done) {
     method = 'index'; // if there is no/empty id field, we must use the index method to create it (API 5.0)
   }
   document.body.docType = model;
+  if (document.body[SEARCHAFTERKEY]) {
+    document.body[SEARCHAFTERKEY] = undefined;
+  }
   self.db[method](
     document
   ).then(

diff --git a/lib/esConnector.js b/lib/esConnector.js
@@ -35,6 +35,9 @@ const { updateAll } = require('./updateAll');
 const { updateAttributes } = require('./updateAttributes');
 const { updateOrCreate } = require('./updateOrCreate');
 
+// CONSTANTS
+const SEARCHAFTERKEY = '_search_after';
+
 /**
  * Connector constructor
  * @param {object} datasource settings
@@ -212,7 +215,7 @@ ESConnector.prototype.getValueFromProperty = function (property, value) {
  * @param {Object} data from DB
  * @returns {object} modeled document
  */
-ESConnector.prototype.matchDataToModel = function (modelName, data, esId, idName) {
+ESConnector.prototype.matchDataToModel = function (modelName, data, esId, idName, sort) {
   /*
   log('ESConnector.prototype.matchDataToModel', 'modelName',
     modelName, 'data', JSON.stringify(data,null,0));
@@ -239,6 +242,7 @@ ESConnector.prototype.matchDataToModel = function (modelName, data, esId, idName
         );
       }
     });
+    document[SEARCHAFTERKEY] = sort;
     log('ESConnector.prototype.matchDataToModel', 'document', JSON.stringify(document, null, 0));
     return document;
   } catch (err) {
@@ -258,7 +262,7 @@ ESConnector.prototype.dataSourceToModel = function (modelName, data, idName) {
 
   // return data._source; // TODO: super-simplify?
   // eslint-disable-next-line no-underscore-dangle
-  return this.matchDataToModel(modelName, data._source, data._id, idName);
+  return this.matchDataToModel(modelName, data._source, data._id, idName, data.sort || []);
 };
 
 /**

diff --git a/lib/find.js b/lib/find.js
@@ -8,12 +8,12 @@ function find(modelName, id, done) {
   if (id === undefined || id === null) {
     throw new Error('id not set!');
   }
-
+  const idName = self.idName(modelName);
   const defaults = self.addDefaults(modelName, 'find');
   self.db.get(_.defaults({
     id: self.getDocumentId(id)
   }, defaults)).then(({ body }) => {
-    done(null, self.dataSourceToModel(modelName, body));
+    done(null, self.dataSourceToModel(modelName, body, idName));
   }).catch((error) => {
     log('ESConnector.prototype.find', error.message);
     done(error);

diff --git a/lib/replaceById.js b/lib/replaceById.js
@@ -1,5 +1,7 @@
 const _ = require('lodash');
 const log = require('debug')('loopback:connector:elasticsearch');
+// CONSTANTS
+const SEARCHAFTERKEY = '_search_after';
 
 function replaceById(modelName, id, data, options, callback) {
   const self = this;
@@ -21,6 +23,9 @@ function replaceById(modelName, id, data, options, callback) {
   if (Object.prototype.hasOwnProperty.call(modelProperties, idName)) {
     document.body[idName] = id;
   }
+  if (document.body[SEARCHAFTERKEY]) {
+    document.body[SEARCHAFTERKEY] = undefined;
+  }
   log('ESConnector.prototype.replaceById', 'document', document);
   self.db.index(
     document

diff --git a/lib/replaceOrCreate.js b/lib/replaceOrCreate.js
@@ -1,5 +1,7 @@
 const _ = require('lodash');
 const log = require('debug')('loopback:connector:elasticsearch');
+// CONSTANTS
+const SEARCHAFTERKEY = '_search_after';
 
 function replaceOrCreate(modelName, data, callback) {
   const self = this;
@@ -16,6 +18,9 @@ function replaceOrCreate(modelName, data, callback) {
   document.body = {};
   _.assign(document.body, data);
   document.body.docType = modelName;
+  if (document.body[SEARCHAFTERKEY]) {
+    document.body[SEARCHAFTERKEY] = undefined;
+  }
   log('ESConnector.prototype.replaceOrCreate', 'document', document);
   self.db.index(
     document

diff --git a/lib/save.js b/lib/save.js
@@ -1,5 +1,7 @@
 const _ = require('lodash');
 const log = require('debug')('loopback:connector:elasticsearch');
+// CONSTANTS
+const SEARCHAFTERKEY = '_search_after';
 
 // eslint-disable-next-line consistent-return
 function save(model, data, done) {
@@ -16,6 +18,9 @@ function save(model, data, done) {
     return done('Document id not setted!', null);
   }
   data.docType = model;
+  if (data[SEARCHAFTERKEY]) {
+    data[SEARCHAFTERKEY] = undefined;
+  }
   self.db.update(_.defaults({
     id,
     body: {

diff --git a/lib/updateAll.js b/lib/updateAll.js
@@ -1,5 +1,7 @@
 const _ = require('lodash');
 const log = require('debug')('loopback:connector:elasticsearch');
+// CONSTANTS
+const SEARCHAFTERKEY = '_search_after';
 
 function updateAll(model, where, data, options, cb) {
   const self = this;
@@ -20,7 +22,7 @@ function updateAll(model, where, data, options, cb) {
     params: {}
   };
   _.forEach(data, (value, key) => {
-    if (key !== '_id' || key !== idName) {
+    if (key !== '_id' && key !== idName && key !== SEARCHAFTERKEY) {
       // default language for inline scripts is painless if ES 5, so this needs the extra params.
       reqBody.script.inline += `ctx._source.${key}=params.${key};`;
       reqBody.script.params[key] = value;

diff --git a/lib/updateAttributes.js b/lib/updateAttributes.js
@@ -1,5 +1,7 @@
 const _ = require('lodash');
 const log = require('debug')('loopback:connector:elasticsearch');
+// CONSTANTS
+const SEARCHAFTERKEY = '_search_after';
 
 function updateAttributes(modelName, id, data, callback) {
   const self = this;
@@ -22,7 +24,7 @@ function updateAttributes(modelName, id, data, callback) {
     params: {}
   };
   _.forEach(data, (value, key) => {
-    if (key !== '_id' || key !== idName) {
+    if (key !== '_id' && key !== idName && key !== SEARCHAFTERKEY) {
       // default language for inline scripts is painless if ES 5, so this needs the extra params.
       reqBody.script.inline += `ctx._source.${key}=params.${key};`;
       reqBody.script.params[key] = value;

diff --git a/lib/updateOrCreate.js b/lib/updateOrCreate.js
@@ -1,5 +1,7 @@
 const _ = require('lodash');
 const log = require('debug')('loopback:connector:elasticsearch');
+// CONSTANTS
+const SEARCHAFTERKEY = '_search_after';
 
 function updateOrCreate(modelName, data, callback) {
   const self = this;
@@ -13,6 +15,9 @@ function updateOrCreate(modelName, data, callback) {
 
   const defaults = self.addDefaults(modelName, 'updateOrCreate');
   data.docType = modelName;
+  if (data[SEARCHAFTERKEY]) {
+    data[SEARCHAFTERKEY] = undefined;
+  }
   self.db.update(_.defaults({
     id,
     body: {

diff --git a/package.json b/package.json
@@ -1,6 +1,6 @@
 {
   "name": "loopback-connector-esv6",
-  "version": "2.0.1",
+  "version": "2.1.0",
   "description": "LoopBack Connector for Elasticsearch 6.x and 7.x",
   "main": "index.js",
   "scripts": {