Merge branch 'fts4gen'

Author: ajungg

core/src/androidTest/assets/demoDb_1_6_no_fts.sqlite3

@@ -54,13 +54,32 @@ public void testBoostedPromotionsQuery() throws Throwable {
     }
 
     @Test
-    public void testTextSearch() {
+    public void testTextSearch() throws IOException, SnabbleSdk.SnabbleException {
+        setupSdkWithDb("demoDb_1_6.sqlite3");
+
         ProductDatabase productDatabase = snabbleSdk.getProductDatabase();
-        Cursor cursor = productDatabase.searchByFoldedName("muller", null);
+        Cursor cursor = productDatabase.searchByFoldedName("gold", null);
         cursor.moveToFirst();
         Product product = productDatabase.productAtCursor(cursor);
-        assertEquals(product.getSku(), "1");
-        assertEquals(product.getName(), "Müllermilch Banane 0,4l");
+        assertEquals(product.getSku(), "31");
+        assertEquals(product.getName(), "Goldbären 200g");
+        cursor.close();
+
+        cursor = productDatabase.searchByFoldedName("foo", null);
+        assertEquals(0, cursor.getCount());
+        cursor.close();
+    }
+
+    @Test
+    public void testTextSearchNoFTS() throws IOException, SnabbleSdk.SnabbleException {
+        setupSdkWithDb("demoDb_1_6_no_fts.sqlite3");
+
+        ProductDatabase productDatabase = snabbleSdk.getProductDatabase();
+        Cursor cursor = productDatabase.searchByFoldedName("gold", null);
+        cursor.moveToFirst();
+        Product product = productDatabase.productAtCursor(cursor);
+        assertEquals(product.getSku(), "31");
+        assertEquals(product.getName(), "Goldbären 200g");
         cursor.close();
 
         cursor = productDatabase.searchByFoldedName("foo", null);

core/src/androidTest/java/io/snabble/sdk/ProductDatabaseTest.java

@@ -55,7 +55,7 @@ public static void setupMockWebServer() throws Exception {
 
         final Dispatcher dispatcher = new Dispatcher() {
             @Override
-            public MockResponse dispatch(RecordedRequest request) throws InterruptedException {
+            public MockResponse dispatch(RecordedRequest request) {
                 if (request.getPath().equals(metadataUrl)) {
                     return new MockResponse()
                             .addHeader("Content-Type", "application/json; charset=utf-8")
@@ -116,18 +116,19 @@ public void setupSdkWithDb(String testDbName) throws IOException, SnabbleSdk.Sna
         config.metadataUrl = metadataUrl;
         config.endpointBaseUrl = "http://" + mockWebServer.getHostName() + ":" + mockWebServer.getPort();
         config.productDbName = testDbName;
+        config.generateSearchIndex = true;
 
         snabbleSdk = SnabbleSdk.setupBlocking((Application) context.getApplicationContext(), config);
     }
 
     @After
-    public void teardownSdk() throws IOException {
+    public void teardownSdk() {
         FileUtils.deleteQuietly(context.getFilesDir());
         FileUtils.deleteQuietly(new File(context.getFilesDir().getParentFile(), "/databases/"));
     }
 
     @Test
-    public void testSdkInitialization() throws IOException {
+    public void testSdkInitialization() {
         String endpointBaseUrl = snabbleSdk.getEndpointBaseUrl();
         String metadataUrl = snabbleSdk.getMetadataUrl();
         String appDbUrl = snabbleSdk.getAppDbUrl();

core/src/androidTest/java/io/snabble/sdk/SnabbleSdkTest.java

@@ -46,6 +46,7 @@ public static class Config {
         int bundledSchemaVersionMajor;
         int bundledSchemaVersionMinor;
         boolean autoUpdateIfMissing;
+        boolean generateSearchIndex;
     }
 
     private SQLiteDatabase db;
@@ -64,6 +65,7 @@ public static class Config {
     private Date lastUpdateDate;
     private int schemaVersionMajor;
     private int schemaVersionMinor;
+    private boolean generateSearchIndex;
 
     private SnabbleSdk sdk;
     private Application application;
@@ -88,6 +90,7 @@ public static class Config {
         this.bundledRevisionId = config.bundledRevisionId;
         this.bundledSchemaVersionMajor = config.bundledSchemaVersionMajor;
         this.bundledSchemaVersionMinor = config.bundledSchemaVersionMinor;
+        this.generateSearchIndex = config.generateSearchIndex;
 
         this.productDatabaseDownloader = new ProductDatabaseDownloader(sdk, this);
         this.productApi = new ProductApi(sdk);
@@ -207,6 +210,7 @@ private boolean open(boolean allowCopyFromBundle) {
                             Formatter.formatFileSize(application, size()));
                 }
 
+                createFTSIndexIfNecessary();
                 parseLastUpdateTimestamp();
                 return true;
             } catch (Exception e) {
@@ -438,6 +442,79 @@ synchronized void applyFullUpdate(InputStream inputStream) throws IOException {
         Logger.d("Full update took %d ms", time2);
     }
 
+    private void createFTSIndexIfNecessary() {
+        if(generateSearchIndex) {
+            long time = SystemClock.elapsedRealtime();
+
+            synchronized (dbLock) {
+                Cursor cursor;
+                cursor = rawQuery("SELECT name FROM sqlite_master WHERE type='table' AND name='searchByName'", null, null);
+                boolean hasFTS = cursor != null && cursor.getCount() == 1;
+                if(cursor != null){
+                    cursor.close();
+                }
+
+                if(!hasFTS) {
+                    db.beginTransaction();
+
+                    exec("DROP TABLE IF EXISTS searchByName");
+                    exec("CREATE VIRTUAL TABLE searchByName USING fts4(sku TEXT, foldedName TEXT)");
+
+                    ContentValues contentValues = new ContentValues(2);
+                    cursor = rawQuery("SELECT sku, name FROM products", null, null);
+                    if (cursor != null) {
+                        while (cursor.moveToNext()) {
+                            contentValues.clear();
+                            contentValues.put("sku", cursor.getString(0));
+                            contentValues.put("foldedName", StringNormalizer.normalize(cursor.getString(1)));
+                            db.insert("searchByName", null, contentValues);
+                        }
+                        cursor.close();
+                    } else {
+                        Logger.d("Could not create FTS4 index, no products");
+                    }
+
+                    db.setTransactionSuccessful();
+                    db.endTransaction();
+
+                    Logger.d("Created FTS4 index in " + (SystemClock.elapsedRealtime() - time) + " ms");
+                } else {
+                    Logger.d("Already has FTS4 index");
+                }
+            }
+        }
+    }
+
+    private Cursor rawQuery(String sql, String[] args, CancellationSignal cancellationSignal) {
+        if (db == null) {
+            return null;
+        }
+
+        long time = SystemClock.elapsedRealtime();
+        Cursor cursor;
+
+        synchronized (dbLock) {
+            try {
+                cursor = db.rawQuery(sql, args, cancellationSignal);
+            } catch (Exception e) {
+                // query could not be executed
+                Logger.e(e.toString());
+                return null;
+            }
+        }
+
+        // query executes when we call the first function that needs data, not on db.rawQuery
+        int count = cursor.getCount();
+
+        long time2 = SystemClock.elapsedRealtime() - time;
+        if (time2 > 16) {
+            Logger.d("Query performance warning (%d ms, %d rows) for SQL: %s",
+                    time2, count, bindArgs(sql, args));
+        }
+
+        return cursor;
+    }
+
     /**
      * Updates the current database in the background by requesting new data from the backend.
      * <p>
@@ -788,33 +865,35 @@ private Cursor productQuery(String appendSql, String[] args, boolean distinct) {
         return productQuery(appendSql, args, distinct, null);
     }
 
-    private Cursor rawQuery(String sql, String[] args, CancellationSignal cancellationSignal) {
-        if (db == null) {
+    /**
+     * This function is deprecated and will be removed from the SDK in the future. There will be no
+     * alternative function to find products by name.
+     *
+     * Find a product by its name. Matching is normalized, so "Apple" finds also "apple".
+     *
+     * @param name The name of the product.
+     * @return The first product matching the name, otherwise null if no product was found.
+     */
+    public Product findByName(String name) {
+        if (name == null || name.length() == 0) {
             return null;
         }
 
-        long time = SystemClock.elapsedRealtime();
-        Cursor cursor;
+        StringNormalizer.normalize(name);
 
-        synchronized (dbLock) {
-            try {
-                cursor = db.rawQuery(sql, args, cancellationSignal);
-            } catch (Exception e) {
-                // query could not be executed
-                return null;
-            }
-        }
+        Cursor cursor = productQuery("JOIN searchByName ns ON ns.sku = p.sku " +
+                "WHERE ns.foldedName MATCH ? LIMIT 1", new String[]{
+                name
+        }, false);
 
-        // query executes when we call the first function that needs data, not on db.rawQuery
-        int count = cursor.getCount();
+        return getFirstProductAndClose(cursor);
+    }
 
-        long time2 = SystemClock.elapsedRealtime() - time;
-        if (time2 > 16) {
-            Logger.d("Query performance warning (%d ms, %d rows) for SQL: %s",
-                    time2, count, bindArgs(sql, args));
+    private void exec(String sql){
+        Cursor cursor = rawQuery(sql, null, null);
+        if(cursor != null){
+            cursor.close();
         }
-
-        return cursor;
     }
 
     private String bindArgs(String sql, String[] args) {
@@ -1074,31 +1153,6 @@ public void findByWeighItemIdOnline(String weighItemId,
         }
     }
 
-    /**
-     * This function is deprecated and will be removed from the SDK in the future. There will be no
-     * alternative function to find products by name.
-     *
-     * Find a product by its name. Matching is normalized, so "Apple" finds also "apple".
-     *
-     * @param name The name of the product.
-     * @return The first product matching the name, otherwise null if no product was found.
-     */
-    @Deprecated
-    public Product findByName(String name) {
-        if (name == null || name.length() == 0) {
-            return null;
-        }
-
-        StringNormalizer.normalize(name);
-
-        Cursor cursor = productQuery("JOIN searchByName s ON " + getSearchIndexColumn() + " = p.sku " +
-                "WHERE s.foldedName MATCH ? LIMIT 1", new String[]{
-                name
-        }, false);
-
-        return getFirstProductAndClose(cursor);
-    }
-
     /**
      * This function is deprecated and will be removed from the SDK in the future. There will be no
      * alternative function to search for products.
@@ -1111,31 +1165,16 @@ public Product findByName(String name) {
      *
      * @param cancellationSignal Calls can be cancelled with a {@link CancellationSignal}. Can be null.
      */
-    @Deprecated
     public Cursor searchByFoldedName(String searchString, CancellationSignal cancellationSignal) {
-        return productQuery("JOIN searchByName s ON " + getSearchIndexColumn() + " = p.sku " +
-                "WHERE s.foldedName MATCH ? " +
+        return productQuery("JOIN searchByName ns ON ns.sku = p.sku " +
+                "WHERE ns.foldedName MATCH ? " +
                 "AND p.weighing != " + Product.Type.PreWeighed.getDatabaseValue() + " " +
                 "AND p.isDeposit = 0 " +
                 "LIMIT 100", new String[]{
                 searchString + "*"
         }, true, cancellationSignal);
     }
 
-    private String getSearchIndexColumn() {
-        // older schema version used the docid field as a primary key
-        // we changed skus to be TEXT instead of INTEGER keys, so we cant use docid anymore
-        // to still support older version we set the field name here
-        String indexColumn;
-
-        if(schemaVersionMajor == 1 && schemaVersionMinor <= 4){
-            indexColumn = "s.docid";
-        } else {
-            indexColumn = "s.sku";
-        }
-        return indexColumn;
-    }
-
     /**
      * Returns a {@link Cursor} which can be iterated for items containing the given scannable code.
      * <p>

core/src/main/java/io/snabble/sdk/ProductDatabase.java

@@ -123,6 +123,15 @@ public static class Config {
          */
         public boolean productDbDownloadIfMissing = true;
 
+        /**
+         * If set to true, creates an full text index to support searching in the product database
+         * using findByName or searchByName.
+         *
+         * Note that this increases setup time of the SDK and it is highly recommended to use
+         * the non-blocking initialization function.
+         */
+        public boolean generateSearchIndex = false;
+
         /**
          * Optional SSLSocketFactory that gets used for HTTP requests.
          *
@@ -384,6 +393,7 @@ private void setupSdk(Config config, final SetupCompletionListener setupCompleti
         dbConfig.bundledSchemaVersionMajor = config.productDbBundledSchemaVersionMajor;
         dbConfig.bundledSchemaVersionMinor = config.productDbBundledSchemaVersionMinor;
         dbConfig.autoUpdateIfMissing = config.productDbDownloadIfMissing;
+        dbConfig.generateSearchIndex = config.generateSearchIndex;
 
         productDatabase = new ProductDatabase(this,
                 config.productDbName,

core/src/main/java/io/snabble/sdk/SnabbleSdk.java

@@ -11,7 +11,6 @@
 import java.io.IOException;
 import java.nio.charset.Charset;
 
-import io.snabble.sdk.PaymentMethod;
 import io.snabble.sdk.SnabbleSdk;
 import io.snabble.sdk.ui.SnabbleUI;
 import io.snabble.sdk.ui.telemetry.Telemetry;