From 599d4f9bf88447884a10cd5f040d3cb231a131c7 Mon Sep 17 00:00:00 2001
From: Bob den Os <bob.den.os@sap.com>
Date: Wed, 27 Mar 2024 14:24:57 +0100
Subject: [PATCH] Delay expand materialization for SQLite

---
 db-service/lib/cqn2sql.js   | 36 +++++++++++++++++++++++++++++++-----
 sqlite/lib/SQLiteService.js | 31 ++++++++++++++++++++++++++++---
 2 files changed, 59 insertions(+), 8 deletions(-)

diff --git a/db-service/lib/cqn2sql.js b/db-service/lib/cqn2sql.js
index c1d035878..5a894585d 100644
--- a/db-service/lib/cqn2sql.js
+++ b/db-service/lib/cqn2sql.js
@@ -209,7 +209,7 @@ class CQN2SQLRenderer {
       q.SELECT
     // REVISIT: When selecting from an entity that is not in the model the from.where are not normalized (as cqn4sql is skipped)
     if (!where && from?.ref?.length === 1 && from.ref[0]?.where) where = from.ref[0]?.where
-    let columns = this.SELECT_columns(q)
+    const columns = this.SELECT_columns(q)
     let sql = `SELECT`
     if (distinct) sql += ` DISTINCT`
     if (!_empty(columns)) sql += ` ${columns}`
@@ -236,7 +236,14 @@ class CQN2SQLRenderer {
    * @returns {string} SQL
    */
   SELECT_columns(q) {
-    return (q.SELECT.columns ?? ['*']).map(x => this.column_expr(x, q))
+    const foreignKeys = {}
+    return (q.SELECT.columns ?? ['*']).map(x => {
+      // Return foreign keys for expands where possible
+      if (x.elements) {
+        return this.column_expand(x, q, foreignKeys)
+      }
+      return this.column_expr(x, q)
+    }).filter(a => a)
   }
 
   /**
@@ -251,9 +258,11 @@ class CQN2SQLRenderer {
     const SELECT = q.SELECT
     if (!SELECT.columns) return sql
 
-    let cols = SELECT.columns.map(x => {
+    const values = this.values
+    this.values = values ? [] : undefined
+    const cols = SELECT.columns.map(x => {
       const name = this.column_name(x)
-      let col = `'$."${name}"',${this.output_converter4(x.element, this.quote(name))}`
+      const col = `'$."${name}"',${this.output_converter4(x.element, x._delayed_expand ? this.expr(x) : this.quote(name))}`
       if (x.SELECT?.count) {
         // Return both the sub select and the count for @odata.count
         const qc = cds.ql.clone(x, { columns: [{ func: 'count' }], one: 1, limit: 0, orderBy: 0 })
@@ -261,6 +270,10 @@ class CQN2SQLRenderer {
       }
       return col
     }).flat()
+    if (values) {
+      // prefix value from the expand columns to retain correct values order
+      this.values = this.values.concat(values)
+    }
 
     const isRoot = SELECT.expand === 'root'
 
@@ -269,7 +282,11 @@ class CQN2SQLRenderer {
     for (let i = 0; i < cols.length; i += 48) {
       obj = `jsonb_insert(${obj},${cols.slice(i, i + 48)})`
     }
-    return `SELECT ${isRoot || SELECT.one ? obj.replace('jsonb', 'json') : `jsonb_group_array(${obj})`} as _json_ FROM (${sql})`
+    if (!SELECT.one && !isRoot) {
+      obj = `jsonb_group_array(${obj})`
+    }
+    const alias = q.SELECT.from.args?.[0]?.as || q.SELECT.from.as
+    return `SELECT ${isRoot ? `json(${obj})` : obj} as _json_ FROM (${sql})${alias ? ` as ${this.quote(alias)}` : ''}`
   }
 
   /**
@@ -291,6 +308,15 @@ class CQN2SQLRenderer {
     return sql
   }
 
+  /**
+   * Renders a SELECT column expression into generic SQL
+   * @param {import('./infer/cqn').col} x
+   * @returns {string} SQL
+   */
+  column_expand(x, q, foreignKeys = {}) {
+    return this.column_expr(x, q)
+  }
+
   /**
    * Extracts the column alias from a SELECT column expression
    * @param {import('./infer/cqn').col} x
diff --git a/sqlite/lib/SQLiteService.js b/sqlite/lib/SQLiteService.js
index 669ce8c90..4373457d6 100644
--- a/sqlite/lib/SQLiteService.js
+++ b/sqlite/lib/SQLiteService.js
@@ -151,6 +151,31 @@ class SQLiteService extends SQLService {
   }
 
   static CQN2SQL = class CQN2SQLite extends SQLService.CQN2SQL {
+    // Move as many expand columns outside of the native query
+    // When using ORDER BY or HAVING this materializes all expand columns
+    // Before applying LIMIT which increases query processing time
+    column_expand(x, q, foreignKeys = {}) {
+      const parentAlias = q.SELECT.from.args?.[0]?.as || q.SELECT.from.as
+
+      if (x.element.parent !== q.target || !parentAlias) return this.column_expr(x, q)
+      const fkeys = []
+      const invalid = x.element._foreignKeys.find(k => {
+        const element = k.parentElement
+        const name = element.name
+        if (foreignKeys[name]) return
+        if (!q.elements[name]) {
+          foreignKeys[name] = true
+          fkeys.push(this.expr({ ref: [parentAlias, k.parentElement.name] }))
+        } else if (q.elements[name].parent !== element.parent || q.elements[name].name !== element.name) {
+          return true // Invalid foreignkey inclusion detected
+        }
+      })
+
+      if (invalid) return this.column_expr(x, q)
+      x._delayed_expand = true
+      return fkeys.length ? `${fkeys}` : ''
+    }
+
     column_alias4(x, q) {
       let alias = super.column_alias4(x, q)
       if (alias) return alias
@@ -206,9 +231,9 @@ class SQLiteService extends SQLService {
 
       // Structs and arrays are stored as JSON strings; the ->'$' unwraps them.
       // Otherwise they would be added as strings to json_objects.
-      Association: expr => `${expr}->'$'`,
-      struct: expr => `${expr}->'$'`,
-      array: expr => `${expr}->'$'`,
+      Association: expr => `jsonb(${expr})`,
+      struct: expr => `jsonb(${expr})`,
+      array: expr => `jsonb(${expr})`,
 
       // SQLite has no booleans so we need to convert 0 and 1
       boolean: expr => `CASE ${expr} when 1 then 'true' when 0 then 'false' END ->'$'`,