Full text search (fts5)

CHANGELOG-Unreleased.md

@@ -11,6 +11,19 @@
 
 ### New features
 
+- `db.write(writer => { ... writer.batch() })` - you can now call batch on the interface passed to a writer block
+- **Fetching record IDs and unsafe raws.** You can now optimize fetching of queries that only require IDs, not full cached records:
+  - `await query.fetchIds()` will return an array of record ids
+  - `await query.unsafeFetchRaw()` will return an array of unsanitized, unsafe raw objects (use alongside `Q.unsafeSqlQuery` to exclude unnecessary or include extra columns)
+  - advanced `adapter.queryIds()`, `adapter.unsafeQueryRaw` are also available
+- **Raw SQL queries**. New syntax for running unsafe raw SQL queries:
+  - `collection.query(Q.unsafeSqlQuery("select * from tasks where foo = ?", ['bar'])).fetch()`
+  - You can now also run `.fetchCount()`, `.fetchIds()` on SQL queries
+  - You can now safely pass values for SQL placeholders by passing an array
+  - You can also observe an unsafe raw SQL query -- with some caveats! refer to documentation for more details
+- [SQLiteAdapter] Added support for Full Text Search for SQLite adapter:
+  Add `isFTS` boolean flag to schema column descriptor for creating Full Text Search-able columns
+  Add `Q.ftsMatch(value)` that compiles to `match 'value'` SQL for performing Full Text Search using SQLite adpater
 - **LocalStorage**. `database.localStorage` is now available
 - **sortBy, skip, take** are now available in LokiJSAdapter as well
 - **Disposable records**. Read-only records that cannot be saved in the database, updated, or deleted and only exist for as long as you keep a reference to them in memory can now be created using `collection.disposableFromDirtyRaw()`. This is useful when you're adding online-only features to an otherwise offline-first app.

docs-master/Query.md

@@ -256,6 +256,11 @@ tasksCollection.query(
 )
 ```
 
+
+#### Full Text Search with `Q.ftsMatch`
+
+If you are using `SQLite` and used `isFTS` in one or more of your text columns, you can use `Q.where(fieldName, Q.ftsMatch(searchText))`. If you have more than one column with `isFTS`, you can either use `tableName` instead of `fieldName` to search in all fields, or specify `fieldName` to focus the search in only one field.
+
 ## Advanced Queries
 
 ### Advanced observing

src/QueryDescription/index.d.ts

@@ -16,6 +16,7 @@ declare module '@nozbe/watermelondb/QueryDescription' {
     | 'oneOf'
     | 'notIn'
     | 'between'
+    | 'match'
 
   export interface ColumnDescription {
     column: ColumnName
@@ -111,6 +112,7 @@ declare module '@nozbe/watermelondb/QueryDescription' {
   export function where(left: ColumnName, valueOrComparison: Value | Comparison): WhereDescription
   export function and(...conditions: Condition[]): And
   export function or(...conditions: Condition[]): Or
+  export function ftsMatch(value: string): Comparison
   export function like(value: string): Comparison
   export function notLike(value: string): Comparison
   export function sortBy(sortColumn: ColumnName, sortOrder?: SortOrder): SortBy

src/QueryDescription/index.js

@@ -30,6 +30,7 @@ export type Operator =
   | 'between'
   | 'like'
   | 'notLike'
+  | 'ftsMatch'
 
 export type ColumnDescription = $RE<{ column: ColumnName, type?: symbol }>
 export type ComparisonRight =
@@ -248,6 +249,11 @@ export function sanitizeLikeString(value: string): string {
   return value.replace(nonLikeSafeRegexp, '_')
 }
 
+export function ftsMatch(value: string): Comparison {
+  invariant(typeof value === 'string', 'Value passed to Q.ftsMatch() is not a string')
+  return { operator: 'ftsMatch', right: { value }, type: comparisonSymbol }
+}
+
 export function column(name: ColumnName): ColumnDescription {
   invariant(typeof name === 'string', 'Name passed to Q.column() is not a string')
   return { column: checkName(name), type: columnSymbol }

src/QueryDescription/test.js

@@ -516,6 +516,26 @@ describe('buildQueryDescription', () => {
       process.env.NODE_ENV = env
     }
   })
+  it('supports ftsMatch as fts join', () => {
+    const query = Q.buildQueryDescription([Q.where('searchable', Q.ftsMatch('hello world'))])
+    expect(query).toEqual({
+      where: [
+        {
+          type: 'where',
+          left: 'searchable',
+          comparison: {
+            operator: 'ftsMatch',
+            right: {
+              value: 'hello world',
+            },
+          },
+        },
+      ],
+      joinTables: [],
+      nestedJoinTables: [],
+      sortBy: [],
+    })
+  })
   it('catches bad types', () => {
     expect(() => Q.eq({})).toThrow('Invalid value passed to query')
     // TODO: oneOf/notIn values?
@@ -526,6 +546,7 @@ describe('buildQueryDescription', () => {
     expect(() => Q.notLike(null)).toThrow('not a string')
     expect(() => Q.notLike({})).toThrow('not a string')
     expect(() => Q.sanitizeLikeString(null)).toThrow('not a string')
+    expect(() => Q.ftsMatch(null)).toThrow('not a string')
     expect(() => Q.column({})).toThrow('not a string')
     expect(() => Q.take('0')).toThrow('not a number')
     expect(() => Q.skip('0')).toThrow('not a number')

src/Schema/index.d.ts

@@ -17,6 +17,7 @@ declare module '@nozbe/watermelondb/Schema' {
     type: ColumnType
     isOptional?: boolean
     isIndexed?: boolean
+    isFTS?: boolean
   }
 
   interface ColumnMap {

src/Schema/index.js

@@ -15,6 +15,7 @@ export type ColumnSchema = $RE<{
   type: ColumnType,
   isOptional?: boolean,
   isIndexed?: boolean,
+  isFTS?: boolean,
 }>
 
 export type ColumnMap = { [name: ColumnName]: ColumnSchema }

src/__tests__/databaseTests.js

@@ -1267,7 +1267,6 @@ joinTest({
       { id: 'p4', num1: 5, num2: undefined },
       { id: 'p5', num1: 0 },
       { id: 'p6', num1: 0, num2: null },
-
       { id: 'badp1' },
       { id: 'badp2', num1: null },
       { id: 'badp3', num2: null },
@@ -1466,4 +1465,53 @@ joinTest({
   skipSqlite: true,
 })
 
-export { matchTests, joinTests }
+const ftsMatchTests = [
+  {
+    name: 'Can ftsMatch - text1',
+    query: [Q.where('text1', Q.ftsMatch('bar'))],
+    matching: [
+      { id: 'fts_foo_bar', text1: 'foo bar' },
+      { id: 'fts_bar', text1: 'bar' },
+      { id: 'fts_bar_baz', text1: 'bar baz' },
+    ],
+    nonMatching: [
+      { id: 'fts_foo', text1: 'foo', text2: 'bar baz' },
+      { id: 'fts_foo_baz', text1: 'foo baz', text2: 'bar' },
+      { id: 'fts_baz', text1: 'baz', text2: 'foo bar' },
+      { id: 'fts_foo_bar_baz', text1: 'foo bar baz', _status: 'deleted' },
+    ],
+    skipLoki: true,
+  },
+  {
+    name: 'Can ftsMatch - text2',
+    query: [Q.where('text2', Q.ftsMatch('bar'))],
+    matching: [
+      { id: 'fts_foo', text1: 'foo', text2: 'bar baz' },
+      { id: 'fts_foo_baz', text1: 'foo baz', text2: 'bar' },
+      { id: 'fts_baz', text1: 'baz', text2: 'foo bar' },
+    ],
+    nonMatching: [
+      { id: 'fts_foo_bar', text1: 'foo bar' },
+      { id: 'fts_bar', text1: 'bar' },
+      { id: 'fts_bar_baz', text1: 'bar baz' },
+      { id: 'fts_foo_bar_baz', text1: 'foo bar baz', _status: 'deleted' },
+    ],
+    skipLoki: true,
+  },
+  {
+    name: 'Can ftsMatch - text1 and text2',
+    query: [Q.where('tasks', Q.ftsMatch('bar'))],
+    matching: [
+      { id: 'fts_foo_bar', text1: 'foo bar' },
+      { id: 'fts_bar', text1: 'bar' },
+      { id: 'fts_bar_baz', text1: 'bar baz' },
+      { id: 'fts_foo', text1: 'foo', text2: 'bar baz' },
+      { id: 'fts_foo_baz', text1: 'foo baz', text2: 'bar' },
+      { id: 'fts_baz', text1: 'baz', text2: 'foo bar' },
+    ],
+    nonMatching: [{ id: 'fts_foo_bar_baz', text1: 'foo bar baz', _status: 'deleted' }],
+    skipLoki: true,
+  },
+]
+
+export { matchTests, joinTests, ftsMatchTests }

src/adapters/__tests__/commonTests.js

@@ -12,14 +12,20 @@ import * as Q from '../../QueryDescription'
 import { appSchema, tableSchema } from '../../Schema'
 import { schemaMigrations, createTable, addColumns } from '../../Schema/migrations'
 
-import { matchTests, naughtyMatchTests, joinTests } from '../../__tests__/databaseTests'
+import {
+  matchTests,
+  naughtyMatchTests,
+  joinTests,
+  ftsMatchTests,
+} from '../../__tests__/databaseTests'
 import DatabaseAdapterCompat from '../compat'
 import {
   testSchema,
   taskQuery,
   mockTaskRaw,
   performMatchTest,
   performJoinTest,
+  performFtsMatchTest,
   expectSortedEqual,
   MockTask,
   MockSyncTestRecord,
@@ -1245,7 +1251,21 @@ export default () => {
         await performMatchTest(adapter, testCase)
       }
     }
-  })
+  }),
+  ftsMatchTests.forEach((testCase) => [
+    `[shared ftsMatch test] ${testCase.name}`,
+    async (adapter, AdapterClass) => {
+      const perform = () => performFtsMatchTest(adapter, testCase)
+      const shouldSkip =
+        (AdapterClass.name === 'LokiJSAdapter' && testCase.skipLoki) ||
+        (AdapterClass.name === 'SQLiteAdapter' && testCase.skipSqlite)
+      if (shouldSkip) {
+        await expect(perform()).rejects.toBeInstanceOf(Error)
+      } else {
+        await perform()
+      }
+    },
+  ]),
   it('can store and retrieve large numbers (regression test)', async (_adapter) => {
     // NOTE: matcher test didn't catch it because both insert and query has the same bug
     let adapter = _adapter

src/adapters/__tests__/helpers.js

@@ -59,8 +59,8 @@ export const testSchema = appSchema({
         { name: 'num3', type: 'number' },
         { name: 'float1', type: 'number' }, // TODO: Remove me?
         { name: 'float2', type: 'number' },
-        { name: 'text1', type: 'string' },
-        { name: 'text2', type: 'string' },
+        { name: 'text1', type: 'string', isFTS: true },
+        { name: 'text2', type: 'string', isFTS: true },
         { name: 'bool1', type: 'boolean' },
         { name: 'bool2', type: 'boolean' },
         { name: 'order', type: 'number' },
@@ -207,3 +207,9 @@ export const performJoinTest = async (adapter, testCase) => {
   await allPromises(([table, records]) => insertAll(adapter, table, records), pairs)
   await performMatchTest(adapter, testCase)
 }
+
+export const performFtsMatchTest = async (adapter, testCase) => {
+  const pairs = toPairs(testCase.extraRecords)
+  await allPromises(([table, records]) => insertAll(adapter, table, records), pairs)
+  await performMatchTest(adapter, testCase)
+}

src/adapters/lokijs/worker/DatabaseDriver.js

@@ -10,7 +10,7 @@ import type {
   BatchOperation,
   UnsafeExecuteOperations,
 } from '../../type'
-import type { TableName, AppSchema, SchemaVersion, TableSchema } from '../../../Schema'
+import type { TableName, AppSchema, SchemaVersion, TableSchema, ColumnSchema } from '../../../Schema'
 import type {
   SchemaMigrations,
   CreateTableMigrationStep,
@@ -318,6 +318,8 @@ export default class DatabaseDriver {
       [],
     )
 
+    this._warnAboutLackingFTSSupport(columnArray)
+
     this.loki.addCollection(name, {
       unique: ['id'],
       indices: ['_status', ...indexedColumns],
@@ -421,6 +423,8 @@ export default class DatabaseDriver {
         collection.ensureIndex(column.name)
       }
     })
+
+    this._warnAboutLackingFTSSupport(columns)
   }
 
   // Maps records to their IDs if the record is already cached on JS side
@@ -476,4 +480,13 @@ export default class DatabaseDriver {
     // Rethrow error
     throw error
   }
+
+  _warnAboutLackingFTSSupport(columns: Array<ColumnSchema>): void {
+    if (columns.some((column) => column.isFTS)) {
+      // Warn the user about missing FTS support for the LokiJS adapter
+      // Please contribute! Here are some pointers:
+      // https://github.com/LokiJS-Forge/LokiDB/blob/master/packages/full-text-search/spec/generic/full_text_search.spec.ts
+      logger.warn('[DB][Worker] LokiJS support for FTS is still to be implemented')
+    }
+  }
 }

src/adapters/sqlite/encodeQuery/index.js

@@ -52,6 +52,7 @@ const operators: { [Operator]: string } = {
   between: 'between',
   like: 'like',
   notLike: 'not like',
+  ftsMatch: 'match',
 }
 
 const encodeComparison = (table: TableName<any>, comparison: Comparison) => {
@@ -111,6 +112,22 @@ const encodeWhereCondition = (
     )
   }
 
+
+  if (comparison.operator === 'ftsMatch') {
+    const srcTable = `"${table}"`
+    const ftsTable = `"_fts_${table}"`
+    const rowid = '"rowid"'
+    const ftsColumn = `"${left}"`
+    const matchValue = getComparisonRight(table, comparison.right)
+    const ftsTableColumn = table === left ? `${ftsTable}` : `${ftsTable}.${ftsColumn}`
+    return (
+      `${srcTable}.${rowid} in (` +
+      `select ${ftsTable}.${rowid} from ${ftsTable} ` +
+      `where ${ftsTableColumn} match ${matchValue}` +
+      `)`
+    )
+  }
+
   return `"${table}"."${left}" ${encodeComparison(table, comparison)}`
 }
 

src/adapters/sqlite/encodeQuery/test.js

@@ -71,7 +71,7 @@ describe('SQLite encodeQuery', () => {
         Q.where('col5', Q.lte(5)),
         Q.where('col6', Q.notEq(null)),
         Q.where('col7', Q.oneOf([1, 2, 3])),
-        Q.where('col8', Q.notIn(['"a"', "'b'", 'c'])),
+        Q.where('col8', Q.notIn(['"a"', "'b'", 'c'])), // eslint-disable-line quotes
         Q.where('col9', Q.between(10, 11)),
         Q.where('col10', Q.like('%abc')),
         Q.where('col11', Q.notLike('def%')),
@@ -171,6 +171,57 @@ describe('SQLite encodeQuery', () => {
         ` and "tasks"."_status" is not 'deleted'`,
     )
   })
+  it('encodes ftsMatch', () => {
+    expect(encoded([Q.where('searchable', Q.ftsMatch('hello world'))])).toBe(
+      `select "tasks".* from "tasks" ` +
+        `where "tasks"."rowid" in (` +
+        `select "_fts_tasks"."rowid" from "_fts_tasks" ` +
+        `where "_fts_tasks"."searchable" match 'hello world'` +
+        `) and "tasks"."_status" is not 'deleted'`,
+    )
+    expect(encoded([Q.where('tasks', Q.ftsMatch('hello world'))])).toBe(
+      `select "tasks".* from "tasks" ` +
+        `where "tasks"."rowid" in (` +
+        `select "_fts_tasks"."rowid" from "_fts_tasks" ` +
+        `where "_fts_tasks" match 'hello world'` +
+        `) and "tasks"."_status" is not 'deleted'`,
+    )
+  })
+  it('encodes ftsMatch with other joins', () => {
+    const query = [
+      Q.on('projects', 'team_id', 'abcdef'),
+      Q.on('projects', 'is_active', true),
+      Q.on('projects', 'left_column', Q.lte(Q.column('right_column'))),
+      Q.on('projects', 'left2', Q.weakGt(Q.column('right2'))),
+      Q.where('left_column', 'right_value'),
+      Q.on('tag_assignments', 'tag_id', Q.oneOf(['a', 'b', 'c'])),
+      Q.where('searchable', Q.ftsMatch('hello world')),
+    ]
+    const expectedQuery =
+      `join "projects" on "projects"."id" = "tasks"."project_id"` +
+      ` join "tag_assignments" on "tag_assignments"."task_id" = "tasks"."id"` +
+      ` where ("projects"."team_id" is 'abcdef'` +
+      ` and "projects"."_status" is not 'deleted')` +
+      ` and ("projects"."is_active" is 1` +
+      ` and "projects"."_status" is not 'deleted')` +
+      ` and ("projects"."left_column" <= "projects"."right_column"` +
+      ` and "projects"."_status" is not 'deleted')` +
+      ` and (("projects"."left2" > "projects"."right2"` +
+      ` or ("projects"."left2" is not null` +
+      ` and "projects"."right2" is null))` +
+      ` and "projects"."_status" is not 'deleted')` +
+      ` and "tasks"."left_column" is 'right_value'` +
+      ` and ("tag_assignments"."tag_id" in ('a', 'b', 'c')` +
+      ` and "tag_assignments"."_status" is not 'deleted')` +
+      ` and "tasks"."rowid" in` +
+      ` (select "_fts_tasks"."rowid" from "_fts_tasks" where` +
+      ` "_fts_tasks"."searchable" match 'hello world')` +
+      ` and "tasks"."_status" is not 'deleted'`
+    expect(encoded(query)).toBe(`select distinct "tasks".* from "tasks" ${expectedQuery}`)
+    expect(encoded(query, true)).toBe(
+      `select count(distinct "tasks"."id") as "count" from "tasks" ${expectedQuery}`,
+    )
+  })
   it(`encodes on nested in and/or`, () => {
     expect(
       encoded([