Add hex shape n-grams

README.md

@@ -100,7 +100,11 @@ for await (const row of parquetFind({
 })) ...
 ```
 
-If the regex has no extractable literal (e.g. `/./`, `/foo|bar/`), the index can't prune and HypGrep does a full scan. The substring/regex filter still applies — results are correct, just unaccelerated.
+Literal-free regexes can also prune on fixed structural patterns such as dates,
+phone numbers, SSNs, and UUIDs. Broad or variable patterns such as `/./`,
+`/\d+/`, and `/\w+/` still fall back to a full scan. The per-row regex filter
+always applies, so results remain exact when the index produces false-positive
+candidate blocks.
 
 If you want full control over the row predicate (e.g. a custom JS function), pass `rowFilter`. The string `query` is still used for index pruning while the callback decides which rows to keep:
 

src/constants.js

@@ -64,14 +64,17 @@ export const defaultNgramLength = 5
 export const defaultNgramChars = '"{}:'
 
 // Length of shape (character-class skeleton) n-grams. Shape n-grams collapse
-// letters to L and digits to D over a tiny alphabet, so a short window stays
-// selective for structured patterns (a phone shape DDD-DDDD is rare even though
-// every individual character class is common). 4 captures the digit-group
-// structure of phones, SSNs, IPs and dates.
-export const defaultShapeNgramLength = 4
+// letters to L and digits to D over a tiny alphabet, so a longer window captures
+// enough digit-group structure to distinguish phones, SSNs and dates from the
+// timestamps and identifiers common in trace logs.
+export const defaultShapeNgramLength = 8
 
 // Structural punctuation kept verbatim inside shape runs (everything else that
 // is not a letter or digit is a shape boundary). These are the separators that
 // give numeric patterns their selectivity: dots, dashes, slashes, colons,
 // underscores and the email '@'.
 export const defaultShapeChars = '@.-_/:'
+
+// Hexadecimal shapes keep UUID and hash structure without depending on the
+// particular mix of digits and a-f characters.
+export const defaultHexShapeNgramLength = 8

src/createIndex.js

@@ -1,8 +1,8 @@
 import { parquetMetadataAsync, parquetReadObjects } from 'hyparquet'
 import { ParquetWriter, schemaFromColumnData } from 'hyparquet-writer'
-import { chooseIndexRowGroupSize, defaultBlockSize, defaultIndexPageSize, defaultNgramChars, defaultNgramLength, defaultShapeChars, defaultShapeNgramLength, hypGrepVersion } from './constants.js'
+import { chooseIndexRowGroupSize, defaultBlockSize, defaultHexShapeNgramLength, defaultIndexPageSize, defaultNgramChars, defaultNgramLength, defaultShapeChars, defaultShapeNgramLength, hypGrepVersion } from './constants.js'
 import { extractNgrams } from './ngrams.js'
-import { extractShapeNgrams } from './shape.js'
+import { extractStructuralNgrams } from './shape.js'
 import { assertNonNegativeSafeInteger, assertPositiveSafeInteger, getTextColumnsFromSchema } from './utils.js'
 
 /**
@@ -40,6 +40,7 @@ export async function createIndex({
   // queries tokenize identically and so an index built without it is detected.
   const shapeNgramLength = defaultShapeNgramLength
   const shapeChars = defaultShapeChars
+  const hexShapeNgramLength = defaultHexShapeNgramLength
 
   const metadata = sourceMetadata ?? await parquetMetadataAsync(sourceFile)
   const numRows = Number(metadata.num_rows)
@@ -117,7 +118,16 @@ export async function createIndex({
       columns: textColumns,
     })
     for (const row of rows) {
-      collectRowNgrams(row, textColumns, ngramLength, ngramChars, shapeNgramLength, shapeChars, blockNgrams)
+      collectRowNgrams(
+        row,
+        textColumns,
+        ngramLength,
+        ngramChars,
+        shapeNgramLength,
+        shapeChars,
+        hexShapeNgramLength,
+        blockNgrams
+      )
       if (++rowsInBlock === blockSize) flushBlock()
     }
     groupStart += groupRows
@@ -139,6 +149,7 @@ export async function createIndex({
     // index has no shape tokens, so queryIndex must not require any.
     { key: 'hypgrep.shape_ngram_length', value: String(shapeNgramLength) },
     { key: 'hypgrep.shape_chars', value: shapeChars },
+    { key: 'hypgrep.hex_shape_ngram_length', value: String(hexShapeNgramLength) },
     { key: 'hypgrep.text_columns', value: textColumns.join(',') },
     { key: 'hypgrep.source_rows', value: String(numRows) },
     // Can save network requests on the source file
@@ -231,17 +242,28 @@ export async function createIndex({
  * @param {string} chars extra characters kept inside n-gram runs
  * @param {number} shapeN shape n-gram length
  * @param {string} shapeChars structural punctuation kept verbatim in shapes
+ * @param {number} hexShapeN hexadecimal shape n-gram length
  * @param {Set<string>} ngrams block accumulator to add into
  */
-function collectRowNgrams(row, textColumns, n, chars, shapeN, shapeChars, ngrams) {
+function collectRowNgrams(
+  row,
+  textColumns,
+  n,
+  chars,
+  shapeN,
+  shapeChars,
+  hexShapeN,
+  ngrams
+) {
   if (!row) return
   for (const columnName of textColumns) {
     const value = row[columnName]
     if (typeof value !== 'string') continue
     if (value.length >= n) {
       for (const g of extractNgrams(value, n, chars)) ngrams.add(g)
     }
-    // Shape grams stream straight into the block accumulator (no throwaway set).
-    extractShapeNgrams(value, shapeN, shapeChars, ngrams)
+    // Both structural projections share one source traversal and stream
+    // directly into the block accumulator.
+    extractStructuralNgrams(value, shapeN, hexShapeN, shapeChars, ngrams)
   }
 }

src/queryIndex.js

@@ -1,7 +1,7 @@
 import { parquetMetadataAsync, parquetQuery } from 'hyparquet'
 import { hypGrepVersion } from './constants.js'
 import { literalsToNgrams, queryNgrams } from './ngrams.js'
-import { extractRegexLiterals, extractRegexShapes } from './regex.js'
+import { extractRegexHexShapes, extractRegexLiterals, extractRegexShapes } from './regex.js'
 import { assertNonNegativeSafeInteger, assertPositiveSafeInteger } from './utils.js'
 
 /**
@@ -32,7 +32,17 @@ export async function queryIndex({ query, indexFile, indexMetadata }) {
   // Read index kv metadata
   indexMetadata ??= await parquetMetadataAsync(indexFile)
   const kvMetadata = indexMetadata.key_value_metadata || []
-  const { blockSize, ngramLength, ngramChars, shapeNgramLength, shapeChars, textColumns, sourceByteLength, sourceRows } = parseKvMetadata(kvMetadata)
+  const {
+    blockSize,
+    ngramLength,
+    ngramChars,
+    shapeNgramLength,
+    shapeChars,
+    hexShapeNgramLength,
+    textColumns,
+    sourceByteLength,
+    sourceRows,
+  } = parseKvMetadata(kvMetadata)
 
   // A "branch" is a conjunction of n-grams that ALL must appear in a block.
   // A query matches a block if ANY branch is fully satisfied (DNF).
@@ -47,12 +57,17 @@ export async function queryIndex({ query, indexFile, indexMetadata }) {
   let branches
   if (query instanceof RegExp) {
     const shapeGrams = shapeNgramLength ? extractRegexShapes(query, shapeNgramLength, shapeChars) : []
+    const hexShapeGrams = hexShapeNgramLength
+      ? extractRegexHexShapes(query, hexShapeNgramLength, shapeChars)
+      : []
     branches = extractRegexLiterals(query).map(lits => {
       const grams = literalsToNgrams(lits, ngramLength, ngramChars)
-      // Shape grams are near-universal (any 4-digit run, any hyphenated word),
-      // so their posting lists cost bytes but prune little. Only spend them on a
-      // branch with no literal n-gram to prune on; otherwise the literal wins.
-      return grams.length ? grams : grams.concat(shapeGrams)
+      // Prefer the narrowest cheap proof. Ordinary literals normally win;
+      // the hex projection is only needed when the ordinary D/L projection
+      // cannot resolve a mixed hexadecimal class.
+      if (grams.length) return grams
+      if (shapeGrams.length) return shapeGrams
+      return hexShapeGrams
     })
   } else {
     branches = [queryNgrams(query, ngramLength, ngramChars)]
@@ -84,7 +99,6 @@ export async function queryIndex({ query, indexFile, indexMetadata }) {
     blockSize,
     sourceRows,
   })
-
   return { blocks, textColumns, sourceByteLength }
 }
 
@@ -208,6 +222,8 @@ export function parseKvMetadata(kvMetadata) {
   /** @type {number | undefined} */
   let shapeNgramLength
   let shapeChars = ''
+  /** @type {number | undefined} */
+  let hexShapeNgramLength
   /** @type {string[]} */
   let textColumns = []
   /** @type {number | undefined} */
@@ -235,6 +251,10 @@ export function parseKvMetadata(kvMetadata) {
     if (key === 'hypgrep.shape_chars' && typeof value === 'string') {
       shapeChars = value
     }
+    if (key === 'hypgrep.hex_shape_ngram_length') {
+      const n = Number(value)
+      if (Number.isSafeInteger(n) && n > 0) hexShapeNgramLength = n
+    }
     if (key === 'hypgrep.version') {
       version = Number(value)
       if (version !== hypGrepVersion) {
@@ -274,5 +294,15 @@ export function parseKvMetadata(kvMetadata) {
   assertNonNegativeSafeInteger(sourceRows, 'hypgrep.source_rows')
   assertNonNegativeSafeInteger(sourceByteLength, 'hypgrep.source_bytelength')
 
-  return { blockSize, ngramLength, ngramChars, shapeNgramLength, shapeChars, textColumns, sourceByteLength, sourceRows }
+  return {
+    blockSize,
+    ngramLength,
+    ngramChars,
+    shapeNgramLength,
+    shapeChars,
+    hexShapeNgramLength,
+    textColumns,
+    sourceByteLength,
+    sourceRows,
+  }
 }

src/regex.js

@@ -1,4 +1,4 @@
-import { SHAPE_PREFIX, shapeSymbol } from './shape.js'
+import { HEX_SHAPE_PREFIX, SHAPE_PREFIX, hexShapeSymbol, shapeSymbol } from './shape.js'
 
 /**
  * Extract a disjunctive-normal-form set of mandatory literal substrings from a
@@ -71,6 +71,31 @@ const MAX_DNF = 32
  * @returns {string[]} prefixed shape n-grams (all mandatory; empty if none)
  */
 export function extractRegexShapes(regex, shapeN, shapeChars) {
+  return extractRegexProjectedShapes(regex, shapeN, shapeChars, 'shape')
+}
+
+/**
+ * Extract hexadecimal shape n-grams from a regex.
+ *
+ * @param {RegExp} regex
+ * @param {number} shapeN shape n-gram length
+ * @param {string} shapeChars structural punctuation kept verbatim in shapes
+ * @returns {string[]} prefixed hexadecimal shape n-grams
+ */
+export function extractRegexHexShapes(regex, shapeN, shapeChars) {
+  return extractRegexProjectedShapes(regex, shapeN, shapeChars, 'hex')
+}
+
+/**
+ * Extract projected regex shapes for one structural alphabet.
+ *
+ * @param {RegExp} regex
+ * @param {number} shapeN shape n-gram length
+ * @param {string} shapeChars structural punctuation kept verbatim in shapes
+ * @param {'shape'|'hex'} projection
+ * @returns {string[]} prefixed shape n-grams
+ */
+function extractRegexProjectedShapes(regex, shapeN, shapeChars, projection) {
   // v-mode class syntax desyncs the scanner (see extractRegexLiterals), and
   // i+u case folding lets an ASCII letter atom match non-ASCII characters
   // (U+017F ſ, U+212A K) that the index maps to run boundaries. Either would
@@ -82,7 +107,7 @@ export function extractRegexShapes(regex, shapeN, shapeChars) {
   let branches
   try {
     // No 'i' flag: shape symbols L/D must not be lowercased.
-    branches = extractRegexLiterals(new RegExp(projectToShape(regex.source, shapeChars)))
+    branches = extractRegexLiterals(new RegExp(projectToShape(regex.source, shapeChars, projection)))
   } catch {
     return []
   }
@@ -92,7 +117,9 @@ export function extractRegexShapes(regex, shapeN, shapeChars) {
   for (const lit of branches[0]) {
     for (let i = 0; i + shapeN <= lit.length; i += 1) {
       const gram = lit.slice(i, i + shapeN)
-      if (!/^L+$/.test(gram)) out.add(SHAPE_PREFIX + gram)
+      if (projection === 'hex' || !/^L+$/.test(gram)) {
+        out.add((projection === 'hex' ? HEX_SHAPE_PREFIX : SHAPE_PREFIX) + gram)
+      }
     }
   }
   return [...out]
@@ -108,9 +135,10 @@ export function extractRegexShapes(regex, shapeN, shapeChars) {
  *
  * @param {string} src regex source
  * @param {string} shapeChars structural punctuation kept verbatim in shapes
+ * @param {'shape'|'hex'} projection
  * @returns {string} a projected regex source
  */
-function projectToShape(src, shapeChars) {
+function projectToShape(src, shapeChars, projection) {
   let out = ''
   let i = 0
   while (i < src.length) {
@@ -124,20 +152,20 @@ function projectToShape(src, shapeChars) {
     if (c === '\\') {
       const next = src[i + 1]
       if (next === undefined) { i += 1; continue }
-      if (next === 'd') { sym = 'D'; atomEnd = i + 2 }
+      if (next === 'd') { sym = projection === 'hex' ? 'H' : 'D'; atomEnd = i + 2 }
       else if (/[a-zA-Z0-9]/.test(next)) { sym = null; atomEnd = skipSpecialEscape(src, i) } // \w \s \D backref \p{}
-      else { sym = shapeSymbol(next, shapeChars); atomEnd = i + 2 } // \. \- \/ escaped literal
+      else { sym = projectedShapeSymbol(next, shapeChars, projection); atomEnd = i + 2 } // \. \- \/ escaped literal
     } else if (c === '[') {
       atomEnd = skipClass(src, i)
-      sym = classShapeSymbol(src, i, atomEnd, shapeChars)
+      sym = classShapeSymbol(src, i, atomEnd, shapeChars, projection)
     } else if (c === '(') {
       atomEnd = skipGroup(src, i)
       const group = parseGroupPrefix(src, i)
       if (group.kind === 'consuming') {
         // Project the group body in place and keep the (...) structure so the
         // literal extractor folds it — grouped patterns like (\d{3})-(\d{4})
         // still prune by shape instead of collapsing to a boundary.
-        const body = projectToShape(src.slice(group.innerStart, atomEnd - 1), shapeChars)
+        const body = projectToShape(src.slice(group.innerStart, atomEnd - 1), shapeChars, projection)
         const q = peekQuantifier(src, atomEnd)
         out += src.slice(i, group.innerStart) + body + ')' + src.slice(atomEnd, q.end)
         i = q.end
@@ -149,7 +177,7 @@ function projectToShape(src, shapeChars) {
       sym = null
     } else {
       atomEnd = i + 1
-      sym = shapeSymbol(c, shapeChars)
+      sym = projectedShapeSymbol(c, shapeChars, projection)
     }
 
     // Copy any quantifier on the atom verbatim so the extractor expands it.
@@ -169,7 +197,7 @@ function projectToShape(src, shapeChars) {
  */
 function shapeAtomSource(sym) {
   if (sym === null) return '.'
-  if (sym === 'L' || sym === 'D') return sym
+  if (sym === 'L' || sym === 'D' || sym === 'H') return sym
   return /[.*+?^${}()|[\]\\/]/.test(sym) ? '\\' + sym : sym
 }
 
@@ -183,9 +211,10 @@ function shapeAtomSource(sym) {
  * @param {number} start index of '['
  * @param {number} classEnd index after ']'
  * @param {string} shapeChars
+ * @param {'shape'|'hex'} projection
  * @returns {string | null}
  */
-function classShapeSymbol(src, start, classEnd, shapeChars) {
+function classShapeSymbol(src, start, classEnd, shapeChars, projection) {
   let i = start + 1
   const lastInside = classEnd - 1 // position of ']'
   if (src[i] === '^') return null
@@ -197,15 +226,15 @@ function classShapeSymbol(src, start, classEnd, shapeChars) {
     if (src[i] === '\\') {
       const next = src[i + 1]
       if (next === undefined) return null
-      if (next === 'd') s = 'D'
+      if (next === 'd') s = projection === 'hex' ? 'H' : 'D'
       else if (/[a-zA-Z0-9]/.test(next)) return null // \w \s \x41 etc.
-      else s = shapeSymbol(next, shapeChars)
+      else s = projectedShapeSymbol(next, shapeChars, projection)
       i += 2
     } else if (src[i + 1] === '-' && i + 2 < lastInside) {
-      s = rangeShapeSymbol(src[i], src[i + 2])
+      s = rangeShapeSymbol(src[i], src[i + 2], projection)
       i += 3
     } else {
-      s = shapeSymbol(src[i], shapeChars)
+      s = projectedShapeSymbol(src[i], shapeChars, projection)
       i += 1
     }
     if (s === null) return null
@@ -223,15 +252,36 @@ function classShapeSymbol(src, start, classEnd, shapeChars) {
  *
  * @param {string} lo
  * @param {string} hi
+ * @param {'shape'|'hex'} projection
  * @returns {string | null}
  */
-function rangeShapeSymbol(lo, hi) {
+function rangeShapeSymbol(lo, hi, projection) {
+  if (projection === 'hex') {
+    if (lo >= '0' && lo <= '9' && hi >= '0' && hi <= '9') return 'H'
+    if (lo >= 'a' && lo <= 'f' && hi >= 'a' && hi <= 'f') return 'H'
+    if (lo >= 'A' && lo <= 'F' && hi >= 'A' && hi <= 'F') return 'H'
+    return null
+  }
   if (lo >= '0' && lo <= '9' && hi >= '0' && hi <= '9') return 'D'
   if (lo >= 'a' && lo <= 'z' && hi >= 'a' && hi <= 'z') return 'L'
   if (lo >= 'A' && lo <= 'Z' && hi >= 'A' && hi <= 'Z') return 'L'
   return null
 }
 
+/**
+ * Map a literal character through a structural projection.
+ *
+ * @param {string} char
+ * @param {string} shapeChars
+ * @param {'shape'|'hex'} projection
+ * @returns {string | null}
+ */
+function projectedShapeSymbol(char, shapeChars, projection) {
+  return projection === 'hex'
+    ? hexShapeSymbol(char, shapeChars)
+    : shapeSymbol(char, shapeChars)
+}
+
 /**
  * Expand a (sub-)region of regex source into a DNF of literal branches.
  * Splits on top-level alternation first, then expands each branch.