[IMP] pivot: use NA() for a missing value

Author: LucasLefevre

src/functions/module_lookup.ts

@@ -698,7 +698,7 @@ export const PIVOT_VALUE = {
     arg("pivot_id (number,string)", _t("ID of the pivot.")),
     arg("measure_name (string)", _t("Name of the measure.")),
     arg("domain_field_name (string,optional,repeating)", _t("Field name.")),
-    arg("domain_value (number,string,boolean,optional,repeating)", _t("Value.")),
+    arg("domain_value (number,string,boolean,any,optional,repeating)", _t("Value.")),
   ],
   compute: function (
     formulaId: Maybe<FunctionResultObject>,
@@ -746,7 +746,7 @@ export const PIVOT_HEADER = {
   args: [
     arg("pivot_id (number,string)", _t("ID of the pivot.")),
     arg("domain_field_name (string,optional,repeating)", _t("Field name.")),
-    arg("domain_value (number,string,value,optional,repeating)", _t("Value.")),
+    arg("domain_value (number,string,any,optional,repeating)", _t("Value.")),
   ],
   compute: function (
     pivotId: Maybe<FunctionResultObject>,

src/helpers/pivot/pivot_helpers.ts

@@ -12,7 +12,7 @@ import {
   Matrix,
   Pivot,
 } from "../../types";
-import { EvaluationError } from "../../types/errors";
+import { CellErrorType, EvaluationError } from "../../types/errors";
 import {
   Granularity,
   PivotCoreDimension,
@@ -195,16 +195,18 @@ export function toNormalizedPivotValue(
   dimension: Pick<PivotDimension, "type" | "displayName" | "granularity">,
   groupValue
 ) {
-  if (groupValue === null || groupValue === "null") {
+  if (
+    groupValue === null ||
+    groupValue?.value === null ||
+    groupValue === CellErrorType.NotAvailable ||
+    groupValue?.value === CellErrorType.NotAvailable
+  ) {
     return null;
   }
   const groupValueString =
     typeof groupValue === "boolean"
       ? toString(groupValue).toLocaleLowerCase()
       : toString(groupValue);
-  if (groupValueString === "null") {
-    return null;
-  }
   if (!pivotNormalizationValueRegistry.contains(dimension.type)) {
     throw new EvaluationError(
       _t("Field %(field)s is not supported because of its type (%(type)s)", {
@@ -233,7 +235,7 @@ export function toFunctionPivotValue(
   dimension: Pick<PivotDimension, "type" | "granularity">
 ) {
   if (value === null) {
-    return `"null"`;
+    return "NA()";
   }
   if (!pivotToFunctionValueRegistry.contains(dimension.type)) {
     return `"${value}"`;

src/helpers/pivot/spreadsheet_pivot/data_entry_spreadsheet_pivot.ts

@@ -1,4 +1,5 @@
 import { CellValue, EvaluatedCell } from "../../../types";
+import { CellErrorType } from "../../../types/errors";
 import {
   DimensionTree,
   PivotDimension,
@@ -220,7 +221,7 @@ export function groupPivotDataEntriesBy(dataEntries: DataEntries, dimension: Piv
 function keySelector(dimension: PivotDimension): (item: DataEntry, index: number) => string {
   const name = dimension.nameWithGranularity;
   return (item) => {
-    return `${item[name]?.value ?? null}`;
+    return `${item[name]?.value ?? CellErrorType.NotAvailable}`;
   };
 }
 
@@ -245,10 +246,10 @@ export function orderDataEntriesKeys(
  * Used to order two values
  */
 function compareDimensionValues(dimension: PivotDimension, a: string, b: string): number {
-  if (a === "null") {
+  if (a === CellErrorType.NotAvailable) {
     return dimension.order === "asc" ? 1 : -1;
   }
-  if (b === "null") {
+  if (b === CellErrorType.NotAvailable) {
     return dimension.order === "asc" ? -1 : 1;
   }
   if (dimension.type === "integer" || dimension.type === "datetime") {

src/migrations/data.ts

@@ -19,7 +19,7 @@ import { migrationStepRegistry } from "./migration_steps";
  * a breaking change is made in the way the state is handled, and an upgrade
  * function should be defined
  */
-export const CURRENT_VERSION = 24;
+export const CURRENT_VERSION = 25;
 const INITIAL_SHEET_ID = "Sheet1";
 
 /**

src/migrations/migration_steps.ts

@@ -446,6 +446,25 @@ migrationStepRegistry
       }
       return data;
     },
+  })
+  .add("migration_24", {
+    // convert "null" in pivot formulas to NA()
+    versionFrom: "24",
+    migrate(data: WorkbookData): any {
+      for (const sheet of data.sheets || []) {
+        for (const xc in sheet.cells || []) {
+          const cellLowerCased = sheet.cells[xc]?.toLowerCase();
+          if (
+            cellLowerCased?.startsWith("=") &&
+            cellLowerCased.includes("pivot") &&
+            cellLowerCased.includes('"null"')
+          ) {
+            sheet.cells[xc] = sheet.cells[xc]?.replace(/"null"/g, "NA()");
+          }
+        }
+      }
+      return data;
+    },
   });
 
 function fixOverlappingFilters(data: any): any {

tests/clipboard/clipboard_plugin.test.ts

@@ -2114,7 +2114,7 @@ describe("clipboard", () => {
       ["",                                      "=PIVOT.HEADER(1)"],
       ["",                                      '=PIVOT.HEADER(1,"measure","Price:sum")'],
       ['=PIVOT.HEADER(1,"Customer","Alice")',   '=PIVOT.VALUE(1,"Price:sum","Customer","Alice")'],
-      ['=PIVOT.HEADER(1,"Customer","null")',    '=PIVOT.VALUE(1,"Price:sum","Customer","null")'],
+      ['=PIVOT.HEADER(1,"Customer",NA())',    '=PIVOT.VALUE(1,"Price:sum","Customer",NA())'],
     ]);
   });
 

tests/model/model_import_export.test.ts

@@ -586,6 +586,19 @@ describe("Migrations", () => {
     });
     expect(getCellContent(model, "A1")).toBe("Hello");
   });
+
+  test("migrate version 25: flatten cell object", () => {
+    const model = new Model({
+      version: 24,
+      sheets: [
+        {
+          id: "1",
+          cells: { A1: '=PIVOT.VALUE(1, "revenue", "customer", "null")' },
+        },
+      ],
+    });
+    expect(getCell(model, "A1")?.content).toBe('=PIVOT.VALUE(1, "revenue", "customer", NA())');
+  });
 });
 
 describe("Import", () => {

tests/pivots/pivot_menu_items.test.ts

@@ -216,7 +216,7 @@ describe("Pivot fix formula menu item", () => {
     model.dispatch("SET_FORMULA_VISIBILITY", { show: true });
     expect(getEvaluatedGrid(model, "C3:D4")).toEqual([
       [`=PIVOT.HEADER(1,"Customer","Alice")`, `=PIVOT.VALUE(1,"Amount:sum","Customer","Alice")`],
-      [`=PIVOT.HEADER(1,"Customer","null")`, `=PIVOT.VALUE(1,"Amount:sum","Customer","null")`],
+      [`=PIVOT.HEADER(1,"Customer",NA())`, `=PIVOT.VALUE(1,"Amount:sum","Customer",NA())`],
     ]);
   });