Add four-symbol shape fallback

README.md

@@ -101,10 +101,11 @@ for await (const row of parquetFind({
 ```
 
 Literal-free regexes can also prune on fixed structural patterns such as dates,
-phone numbers, SSNs, and UUIDs. Broad or variable patterns such as `/./`,
-`/\d+/`, and `/\w+/` still fall back to a full scan. The per-row regex filter
-always applies, so results remain exact when the index produces false-positive
-candidate blocks.
+phone numbers, SSNs, and UUIDs. The index prefers selective 8-symbol shapes and
+falls back to 4-symbol shapes for shorter structures. Broad or variable
+patterns such as `/./`, `/\d+/`, and `/\w+/` still fall back to a full scan.
+The per-row regex filter always applies, so results remain exact when the index
+produces false-positive candidate blocks.
 
 If you want full control over the row predicate (e.g. a custom JS function), pass `rowFilter`. The string `query` is still used for index pruning while the callback decides which rows to keep:
 

benchmark/build-wildchat-default.js

@@ -13,11 +13,13 @@ import { asyncBufferFromFile } from 'hyparquet'
 import { fileWriter } from 'hyparquet-writer'
 import {
   defaultBlockSize,
+  defaultHexShapeNgramLength,
   defaultIndexRowGroupSize,
   defaultNgramChars,
   defaultNgramLength,
   defaultShapeChars,
   defaultShapeNgramLength,
+  defaultShortShapeNgramLength,
   hypGrepVersion,
   maximumIndexRowGroupSize,
   targetIndexRowGroups,
@@ -53,6 +55,8 @@ console.log(JSON.stringify({
     ngramLength: defaultNgramLength,
     ngramChars: defaultNgramChars,
     shapeNgramLength: defaultShapeNgramLength,
+    shortShapeNgramLength: defaultShortShapeNgramLength,
+    hexShapeNgramLength: defaultHexShapeNgramLength,
     shapeChars: defaultShapeChars,
   },
 }))

src/constants.js

@@ -63,12 +63,17 @@ export const defaultNgramLength = 5
 // empty set, i.e. plain alphanumeric, and keeps working unchanged.
 export const defaultNgramChars = '"{}:'
 
-// Length of shape (character-class skeleton) n-grams. Shape n-grams collapse
-// letters to L and digits to D over a tiny alphabet, so a longer window captures
-// enough digit-group structure to distinguish phones, SSNs and dates from the
-// timestamps and identifiers common in trace logs.
+// Primary length of shape (character-class skeleton) n-grams. Shape n-grams
+// collapse letters to L and digits to D over a tiny alphabet, so a longer
+// window captures enough digit-group structure to distinguish phones, SSNs and
+// dates from the timestamps and identifiers common in trace logs.
 export const defaultShapeNgramLength = 8
 
+// A shorter ordinary-shape layer preserves pruning for structures that cannot
+// produce an 8-symbol window. Queries prefer the primary layer and use this
+// only as a fallback.
+export const defaultShortShapeNgramLength = 4
+
 // Structural punctuation kept verbatim inside shape runs (everything else that
 // is not a letter or digit is a shape boundary). These are the separators that
 // give numeric patterns their selectivity: dots, dashes, slashes, colons,

src/createIndex.js

@@ -1,6 +1,6 @@
 import { parquetMetadataAsync, parquetReadObjects } from 'hyparquet'
 import { ParquetWriter, schemaFromColumnData } from 'hyparquet-writer'
-import { chooseIndexRowGroupSize, defaultBlockSize, defaultHexShapeNgramLength, defaultIndexPageSize, defaultNgramChars, defaultNgramLength, defaultShapeChars, defaultShapeNgramLength, hypGrepVersion } from './constants.js'
+import { chooseIndexRowGroupSize, defaultBlockSize, defaultHexShapeNgramLength, defaultIndexPageSize, defaultNgramChars, defaultNgramLength, defaultShapeChars, defaultShapeNgramLength, defaultShortShapeNgramLength, hypGrepVersion } from './constants.js'
 import { extractNgrams } from './ngrams.js'
 import { extractStructuralNgrams } from './shape.js'
 import { assertNonNegativeSafeInteger, assertPositiveSafeInteger, getTextColumnsFromSchema } from './utils.js'
@@ -39,6 +39,7 @@ export async function createIndex({
   // tokenization of the same text. Its settings are recorded in kv metadata so
   // queries tokenize identically and so an index built without it is detected.
   const shapeNgramLength = defaultShapeNgramLength
+  const shortShapeNgramLength = defaultShortShapeNgramLength
   const shapeChars = defaultShapeChars
   const hexShapeNgramLength = defaultHexShapeNgramLength
 
@@ -124,6 +125,7 @@ export async function createIndex({
         ngramLength,
         ngramChars,
         shapeNgramLength,
+        shortShapeNgramLength,
         shapeChars,
         hexShapeNgramLength,
         blockNgrams
@@ -148,6 +150,7 @@ export async function createIndex({
     // Shape (character-class skeleton) layer. Absence of these keys means the
     // index has no shape tokens, so queryIndex must not require any.
     { key: 'hypgrep.shape_ngram_length', value: String(shapeNgramLength) },
+    { key: 'hypgrep.short_shape_ngram_length', value: String(shortShapeNgramLength) },
     { key: 'hypgrep.shape_chars', value: shapeChars },
     { key: 'hypgrep.hex_shape_ngram_length', value: String(hexShapeNgramLength) },
     { key: 'hypgrep.text_columns', value: textColumns.join(',') },
@@ -240,7 +243,8 @@ export async function createIndex({
  * @param {string[]} textColumns
  * @param {number} n
  * @param {string} chars extra characters kept inside n-gram runs
- * @param {number} shapeN shape n-gram length
+ * @param {number} shapeN primary shape n-gram length
+ * @param {number} shortShapeN fallback shape n-gram length
  * @param {string} shapeChars structural punctuation kept verbatim in shapes
  * @param {number} hexShapeN hexadecimal shape n-gram length
  * @param {Set<string>} ngrams block accumulator to add into
@@ -251,6 +255,7 @@ function collectRowNgrams(
   n,
   chars,
   shapeN,
+  shortShapeN,
   shapeChars,
   hexShapeN,
   ngrams
@@ -262,8 +267,14 @@ function collectRowNgrams(
     if (value.length >= n) {
       for (const g of extractNgrams(value, n, chars)) ngrams.add(g)
     }
-    // Both structural projections share one source traversal and stream
-    // directly into the block accumulator.
-    extractStructuralNgrams(value, shapeN, hexShapeN, shapeChars, ngrams)
+    // All structural layers share one source traversal and stream directly
+    // into the block accumulator.
+    extractStructuralNgrams(
+      value,
+      [shapeN, shortShapeN],
+      hexShapeN,
+      shapeChars,
+      ngrams
+    )
   }
 }

src/queryIndex.js

@@ -37,6 +37,7 @@ export async function queryIndex({ query, indexFile, indexMetadata }) {
     ngramLength,
     ngramChars,
     shapeNgramLength,
+    shortShapeNgramLength,
     shapeChars,
     hexShapeNgramLength,
     textColumns,
@@ -51,23 +52,27 @@ export async function queryIndex({ query, indexFile, indexMetadata }) {
   // - RegExp query: one branch per top-level alternation arm, plus any mandatory
   //   shape n-grams (character-class skeletons) ANDed in. Shape tokens let a
   //   literal-free pattern (a phone number, SSN, IP) prune by its structure.
-  //   Gated on shapeNgramLength so an index with no shape layer is never asked
-  //   for shape tokens (which would be a false negative).
+  //   Gated on recorded shape lengths so an index without a given layer is
+  //   never asked for its tokens (which would be a false negative).
   /** @type {string[][]} */
   let branches
   if (query instanceof RegExp) {
     const shapeGrams = shapeNgramLength ? extractRegexShapes(query, shapeNgramLength, shapeChars) : []
+    const shortShapeGrams = shortShapeNgramLength
+      ? extractRegexShapes(query, shortShapeNgramLength, shapeChars)
+      : []
     const hexShapeGrams = hexShapeNgramLength
       ? extractRegexHexShapes(query, hexShapeNgramLength, shapeChars)
       : []
     branches = extractRegexLiterals(query).map(lits => {
       const grams = literalsToNgrams(lits, ngramLength, ngramChars)
-      // Prefer the narrowest cheap proof. Ordinary literals normally win;
-      // the hex projection is only needed when the ordinary D/L projection
-      // cannot resolve a mixed hexadecimal class.
+      // Prefer the narrowest cheap proof. Ordinary literals normally win,
+      // followed by either selective 8-symbol projection. The shorter ordinary
+      // layer is only a fallback when neither long projection is available.
       if (grams.length) return grams
       if (shapeGrams.length) return shapeGrams
-      return hexShapeGrams
+      if (hexShapeGrams.length) return hexShapeGrams
+      return shortShapeGrams
     })
   } else {
     branches = [queryNgrams(query, ngramLength, ngramChars)]
@@ -221,6 +226,8 @@ export function parseKvMetadata(kvMetadata) {
   // without a shape layer, which signals queryIndex not to require shape tokens.
   /** @type {number | undefined} */
   let shapeNgramLength
+  /** @type {number | undefined} */
+  let shortShapeNgramLength
   let shapeChars = ''
   /** @type {number | undefined} */
   let hexShapeNgramLength
@@ -248,6 +255,10 @@ export function parseKvMetadata(kvMetadata) {
       const n = Number(value)
       if (Number.isSafeInteger(n) && n > 0) shapeNgramLength = n
     }
+    if (key === 'hypgrep.short_shape_ngram_length') {
+      const n = Number(value)
+      if (Number.isSafeInteger(n) && n > 0) shortShapeNgramLength = n
+    }
     if (key === 'hypgrep.shape_chars' && typeof value === 'string') {
       shapeChars = value
     }
@@ -299,6 +310,7 @@ export function parseKvMetadata(kvMetadata) {
     ngramLength,
     ngramChars,
     shapeNgramLength,
+    shortShapeNgramLength,
     shapeChars,
     hexShapeNgramLength,
     textColumns,

src/shape.js

@@ -12,10 +12,10 @@
  * n-grams. They are prefixed with `SHAPE_PREFIX` (a control byte that never
  * occurs in lowercased text n-grams) so the two namespaces can never collide.
  *
- * Index and query MUST use the same shape length and kept-character set, or the
- * tokens won't line up; createIndex records both in kv metadata and queryIndex
- * reads them back. An index built without a shape layer simply has no shape
- * tokens, and queryIndex must not require any (it gates on the metadata).
+ * Index and query MUST use the same shape lengths and kept-character set, or
+ * the tokens won't line up; createIndex records them in kv metadata and
+ * queryIndex reads them back. An index built without a shape layer simply has
+ * no shape tokens, and queryIndex must not require any.
  */
 
 // Control byte that namespaces shape tokens away from text n-grams.
@@ -87,10 +87,11 @@ export function extractHexShapeNgrams(text, shapeN, shapeChars, out = new Set())
 }
 
 /**
- * Extract ordinary and hexadecimal shape n-grams in one source traversal.
+ * Extract one or more ordinary shape lengths and hexadecimal shape n-grams in
+ * one source traversal.
  *
  * @param {string} text
- * @param {number} shapeN shape n-gram length
+ * @param {number | number[]} shapeN ordinary shape n-gram length(s)
  * @param {number} hexShapeN hexadecimal shape n-gram length
  * @param {string} shapeChars structural punctuation kept verbatim in shapes
  * @param {Set<string>} [out] accumulator to add grams into (default: fresh set)
@@ -112,24 +113,36 @@ export function extractStructuralNgrams(
  * both layers and pays for one traversal.
  *
  * @param {string} text
- * @param {number | undefined} shapeN ordinary shape n-gram length
+ * @param {number | number[] | undefined} shapeN ordinary shape n-gram length(s)
  * @param {number | undefined} hexShapeN hexadecimal shape n-gram length
  * @param {string} shapeChars structural punctuation kept verbatim in shapes
  * @param {Set<string>} out accumulator to add grams into
  * @returns {Set<string>}
  */
 function extractStructuralLayers(text, shapeN, hexShapeN, shapeChars, out) {
   if (typeof text !== 'string') return out
-  const shapeLength = shapeN ?? 0
+  const shapeLengths = shapeN === undefined
+    ? []
+    : Array.isArray(shapeN) ? shapeN : [shapeN]
   const hexShapeLength = hexShapeN ?? 0
-  const useShape = shapeLength > 0 && text.length >= shapeLength
+  /** @type {{length: number, informative: number}[]} */
+  const shapeLayers = []
+  let shapeWindowLength = 0
+  for (const length of shapeLengths) {
+    if (length > 0 && text.length >= length) {
+      shapeLayers.push({
+        length,
+        informative: 0,
+      })
+      shapeWindowLength = Math.max(shapeWindowLength, length)
+    }
+  }
+  const useShape = shapeLayers.length > 0
+  const shapeWindow = useShape ? new Array(shapeWindowLength) : []
+  let shapeCount = 0
   const useHex = hexShapeLength > 0 && text.length >= hexShapeLength
   if (!useShape && !useHex) return out
 
-  const shapeWindow = useShape ? new Array(shapeLength) : []
-  let shapeCount = 0
-  let shapeInformative = 0
-
   const hexWindow = useHex ? new Array(hexShapeLength) : []
   let hexCount = 0
   let hexNonUniform = 0
@@ -150,22 +163,34 @@ function extractStructuralLayers(text, shapeN, hexShapeN, shapeChars, out) {
       const symbol = digit ? 'D' : letter ? 'L' : kept ? char : null
       if (symbol === null) {
         shapeCount = 0
-        shapeInformative = 0
+        for (const layer of shapeLayers) layer.informative = 0
       } else {
-        const slot = shapeCount % shapeLength
-        if (shapeCount >= shapeLength && shapeWindow[slot] !== 'L') {
-          shapeInformative -= 1
+        // Every ordinary length uses the same projected symbol stream. Keep one
+        // ring sized for the longest layer and update each layer's informative
+        // count before overwriting the oldest symbol.
+        for (const layer of shapeLayers) {
+          if (shapeCount >= layer.length) {
+            const outgoing =
+              shapeWindow[(shapeCount - layer.length) % shapeWindowLength]
+            if (outgoing !== 'L') layer.informative -= 1
+          }
+        }
+        shapeWindow[shapeCount % shapeWindowLength] = symbol
+        if (symbol !== 'L') {
+          for (const layer of shapeLayers) layer.informative += 1
         }
-        shapeWindow[slot] = symbol
-        if (symbol !== 'L') shapeInformative += 1
         shapeCount += 1
-        if (shapeCount >= shapeLength && shapeInformative > 0) {
-          let gram = SHAPE_PREFIX
-          for (let k = shapeCount % shapeLength, n = 0; n < shapeLength; n += 1) {
-            gram += shapeWindow[k]
-            k = k + 1 === shapeLength ? 0 : k + 1
+
+        for (const layer of shapeLayers) {
+          if (shapeCount >= layer.length && layer.informative > 0) {
+            let gram = SHAPE_PREFIX
+            let k = (shapeCount - layer.length) % shapeWindowLength
+            for (let n = 0; n < layer.length; n += 1) {
+              gram += shapeWindow[k]
+              k = k + 1 === shapeWindowLength ? 0 : k + 1
+            }
+            out.add(gram)
           }
-          out.add(gram)
         }
       }
     }

src/types.d.ts

@@ -64,7 +64,8 @@ export interface HypGrepMetadata {
   blockSize: number // number of rows per logical block
   ngramLength: number // n-gram size used to build the index
   ngramChars: string // extra characters kept inside n-gram runs ('' for pre-structural indexes)
-  shapeNgramLength?: number // shape n-gram size, or undefined if the index has no shape layer
+  shapeNgramLength?: number // primary shape n-gram size, or undefined if the index has no shape layer
+  shortShapeNgramLength?: number // fallback shape n-gram size, or undefined if absent
   shapeChars: string // structural punctuation kept verbatim in shapes ('' if no shape layer)
   hexShapeNgramLength?: number // hexadecimal shape n-gram size, or undefined if absent
   textColumns: string[] // list of indexed text columns

test/createIndex.test.js

@@ -27,22 +27,23 @@ describe('createIndex', () => {
 
     expect(existsSync(TEST_INDEX)).toBe(true)
     const indexBuffer = await asyncBufferFromFile(TEST_INDEX)
-    expect(indexBuffer.byteLength).toBe(2620)
+    expect(indexBuffer.byteLength).toBe(2658)
     const indexMetadata = await parquetMetadataAsync(indexBuffer)
     expect(indexMetadata.row_groups.length).toBe(7)
     expect(indexMetadata.num_rows).toBe(676n)
-    expect(indexMetadata.key_value_metadata?.length).toBe(10)
+    expect(indexMetadata.key_value_metadata?.length).toBe(11)
     const kv = indexMetadata.key_value_metadata
     expect(kv?.[0]).toEqual({ key: 'hypgrep.version', value: '0' })
     expect(kv?.[1]).toEqual({ key: 'hypgrep.block_size', value: '200' })
     expect(kv?.[2]).toEqual({ key: 'hypgrep.ngram_length', value: '5' })
     expect(kv?.[3]).toEqual({ key: 'hypgrep.ngram_chars', value: '"{}:' })
     expect(kv?.[4]).toEqual({ key: 'hypgrep.shape_ngram_length', value: '8' })
-    expect(kv?.[5]).toEqual({ key: 'hypgrep.shape_chars', value: '@.-_/:' })
-    expect(kv?.[6]).toEqual({ key: 'hypgrep.hex_shape_ngram_length', value: '8' })
-    expect(kv?.[7]).toEqual({ key: 'hypgrep.text_columns', value: 'id' })
-    expect(kv?.[8]).toEqual({ key: 'hypgrep.source_rows', value: '676' })
-    expect(kv?.[9]).toEqual({ key: 'hypgrep.source_bytelength', value: String(sourceFile.byteLength) })
+    expect(kv?.[5]).toEqual({ key: 'hypgrep.short_shape_ngram_length', value: '4' })
+    expect(kv?.[6]).toEqual({ key: 'hypgrep.shape_chars', value: '@.-_/:' })
+    expect(kv?.[7]).toEqual({ key: 'hypgrep.hex_shape_ngram_length', value: '8' })
+    expect(kv?.[8]).toEqual({ key: 'hypgrep.text_columns', value: 'id' })
+    expect(kv?.[9]).toEqual({ key: 'hypgrep.source_rows', value: '676' })
+    expect(kv?.[10]).toEqual({ key: 'hypgrep.source_bytelength', value: String(sourceFile.byteLength) })
   })
 
   it('should reject invalid sizing options', async () => {

test/queryIndex.test.js

@@ -240,6 +240,21 @@ describe('queryIndex', () => {
     }
   })
 
+  it('treats a malformed short_shape_ngram_length as no fallback shape layer', () => {
+    for (const value of ['', 'null', '0', undefined]) {
+      const kv = [
+        { key: 'hypgrep.version', value: '0' },
+        { key: 'hypgrep.block_size', value: '100' },
+        { key: 'hypgrep.ngram_length', value: '5' },
+        { key: 'hypgrep.short_shape_ngram_length', value },
+        { key: 'hypgrep.text_columns', value: 'id' },
+        { key: 'hypgrep.source_rows', value: '10' },
+        { key: 'hypgrep.source_bytelength', value: '100' },
+      ]
+      expect(parseKvMetadata(kv).shortShapeNgramLength).toBeUndefined()
+    }
+  })
+
   it('treats a malformed hex_shape_ngram_length as no hex shape layer', () => {
     for (const value of ['', 'null', '0', undefined]) {
       const kv = [