parked: fix(sqlite): Reduce expand materialization for SQLite

db-service/lib/cqn2sql.js

@@ -215,7 +215,7 @@
 
     // REVISIT: When selecting from an entity that is not in the model the from.where are not normalized (as cqn4sql is skipped)
     if (!where && from?.ref?.length === 1 && from.ref[0]?.where) where = from.ref[0]?.where
-    let columns = this.SELECT_columns(q)
+    const columns = this.SELECT_columns(q)
     let sql = `SELECT`
     if (distinct) sql += ` DISTINCT`
     if (!_empty(columns)) sql += ` ${columns}`
@@ -242,7 +242,14 @@
    * @returns {string} SQL
    */
   SELECT_columns(q) {
-    return (q.SELECT.columns ?? ['*']).map(x => this.column_expr(x, q))
+    const foreignKeys = {}
+    return (q.SELECT.columns ?? ['*']).map(x => {
+      // Return foreign keys for expands where possible
+      if (x.elements) {
+        return this.column_expand(x, q, foreignKeys)
+      }
+      return this.column_expr(x, q)
+    }).filter(a => a)
   }
 
   /**
@@ -267,6 +274,8 @@
         q.elements[e].items // Array types require to be inlined with a json result
       )
 
+    const values = this.values
+    this.values = values ? [] : undefined
     let cols = SELECT.columns.map(isSimple
       ? x => {
         const name = this.column_name(x)
@@ -282,7 +291,7 @@
       : x => {
         const name = this.column_name(x)
         const escaped = `${name.replace(/"/g, '""')}`
-        let col = `'$."${escaped}"',${this.output_converter4(x.element, this.quote(name))}`
+        let col = `'$."${escaped}"',${this.output_converter4(x.element, x._delayed_expand ? this.expr(x) : this.quote(name))}`
         if (x.SELECT?.count) {
           // Return both the sub select and the count for @odata.count
           const qc = cds.ql.clone(x, { columns: [{ func: 'count' }], one: 1, limit: 0, orderBy: 0 })
@@ -292,13 +301,21 @@
       }).flat()
 
     if (isSimple) return `SELECT ${cols} FROM (${sql})`
+    if (values) {
+      // prefix value from the expand columns to retain correct values order
+      this.values = this.values.concat(values)
+    }
 
     // Prevent SQLite from hitting function argument limit of 100
     let obj = "'{}'"
     for (let i = 0; i < cols.length; i += 48) {
       obj = `jsonb_insert(${obj},${cols.slice(i, i + 48)})`
     }
-    return `SELECT ${isRoot || SELECT.one ? obj.replace('jsonb', 'json') : `jsonb_group_array(${obj})`} as _json_ FROM (${sql})`
+    if (!SELECT.one && !isRoot) {
+      obj = `jsonb_group_array(${obj})`
+    }
+    const alias = q.SELECT.from.args?.[0]?.as || q.SELECT.from.as
+    return `SELECT ${isRoot ? `json(${obj})` : obj} as _json_ FROM (${sql})${alias ? ` as ${this.quote(alias)}` : ''}`
   }
 
   /**
@@ -320,6 +337,15 @@
     return sql
   }
 
+  /**
+   * Renders a SELECT column expression into generic SQL
+   * @param {import('./infer/cqn').col} x
+   * @returns {string} SQL
+   */
+  column_expand(x, q, foreignKeys = {}) {
+    return this.column_expr(x, q)
+  }
+
   /**
    * Extracts the column alias from a SELECT column expression
    * @param {import('./infer/cqn').col} x

sqlite/lib/SQLiteService.js

@@ -151,6 +151,31 @@ class SQLiteService extends SQLService {
   }
 
   static CQN2SQL = class CQN2SQLite extends SQLService.CQN2SQL {
+    // Move as many expand columns outside of the native query
+    // When using ORDER BY or HAVING this materializes all expand columns
+    // Before applying LIMIT which increases query processing time
+    column_expand(x, q, foreignKeys = {}) {
+      const parentAlias = q.SELECT.from.args?.[0]?.as || q.SELECT.from.as
+
+      if (x.element.parent !== q.target || !parentAlias) return this.column_expr(x, q)
+      const fkeys = []
+      const invalid = x.element._foreignKeys.find(k => {
+        const element = k.parentElement
+        const name = element.name
+        if (foreignKeys[name]) return
+        if (!q.elements[name]) {
+          foreignKeys[name] = true
+          fkeys.push(this.expr({ ref: [parentAlias, k.parentElement.name] }))
+        } else if (q.elements[name].parent !== element.parent || q.elements[name].name !== element.name) {
+          return true // Invalid foreignkey inclusion detected
+        }
+      })
+
+      if (invalid) return this.column_expr(x, q)
+      x._delayed_expand = true
+      return fkeys.length ? `${fkeys}` : ''
+    }
+
     column_alias4(x, q) {
       let alias = super.column_alias4(x, q)
       if (alias) return alias
@@ -204,9 +229,9 @@ class SQLiteService extends SQLService {
       ...super.OutputConverters,
       // Structs and arrays are stored as JSON strings; the ->'$' unwraps them.
       // Otherwise they would be added as strings to json_objects.
-      Association: expr => `${expr}->'$'`,
-      struct: expr => `${expr}->'$'`,
-      array: expr => `${expr}->'$'`,
+      Association: expr => `jsonb(${expr})`,
+      struct: expr => `jsonb(${expr})`,
+      array: expr => `jsonb(${expr})`,
       // SQLite has no booleans so we need to convert 0 and 1
       boolean: expr => `CASE ${expr} when 1 then 'true' when 0 then 'false' END ->'$'`,
       // DateTimes are returned without ms added by InputConverters