diff --git a/README.md b/README.md index efd86a4..42036d7 100644 --- a/README.md +++ b/README.md @@ -101,10 +101,11 @@ for await (const row of parquetFind({ ``` Literal-free regexes can also prune on fixed structural patterns such as dates, -phone numbers, SSNs, and UUIDs. Broad or variable patterns such as `/./`, -`/\d+/`, and `/\w+/` still fall back to a full scan. The per-row regex filter -always applies, so results remain exact when the index produces false-positive -candidate blocks. +phone numbers, SSNs, and UUIDs. The index prefers selective 8-symbol shapes and +falls back to 4-symbol shapes for shorter structures. Broad or variable +patterns such as `/./`, `/\d+/`, and `/\w+/` still fall back to a full scan. +The per-row regex filter always applies, so results remain exact when the index +produces false-positive candidate blocks. If you want full control over the row predicate (e.g. a custom JS function), pass `rowFilter`. The string `query` is still used for index pruning while the callback decides which rows to keep: diff --git a/benchmark/build-wildchat-default.js b/benchmark/build-wildchat-default.js index 600e990..2997f35 100644 --- a/benchmark/build-wildchat-default.js +++ b/benchmark/build-wildchat-default.js @@ -13,11 +13,13 @@ import { asyncBufferFromFile } from 'hyparquet' import { fileWriter } from 'hyparquet-writer' import { defaultBlockSize, + defaultHexShapeNgramLength, defaultIndexRowGroupSize, defaultNgramChars, defaultNgramLength, defaultShapeChars, defaultShapeNgramLength, + defaultShortShapeNgramLength, hypGrepVersion, maximumIndexRowGroupSize, targetIndexRowGroups, @@ -53,6 +55,8 @@ console.log(JSON.stringify({ ngramLength: defaultNgramLength, ngramChars: defaultNgramChars, shapeNgramLength: defaultShapeNgramLength, + shortShapeNgramLength: defaultShortShapeNgramLength, + hexShapeNgramLength: defaultHexShapeNgramLength, shapeChars: defaultShapeChars, }, })) diff --git a/src/constants.js b/src/constants.js index 4a22952..ed9909d 100644 --- a/src/constants.js +++ b/src/constants.js @@ -63,12 +63,17 @@ export const defaultNgramLength = 5 // empty set, i.e. plain alphanumeric, and keeps working unchanged. export const defaultNgramChars = '"{}:' -// Length of shape (character-class skeleton) n-grams. Shape n-grams collapse -// letters to L and digits to D over a tiny alphabet, so a longer window captures -// enough digit-group structure to distinguish phones, SSNs and dates from the -// timestamps and identifiers common in trace logs. +// Primary length of shape (character-class skeleton) n-grams. Shape n-grams +// collapse letters to L and digits to D over a tiny alphabet, so a longer +// window captures enough digit-group structure to distinguish phones, SSNs and +// dates from the timestamps and identifiers common in trace logs. export const defaultShapeNgramLength = 8 +// A shorter ordinary-shape layer preserves pruning for structures that cannot +// produce an 8-symbol window. Queries prefer the primary layer and use this +// only as a fallback. +export const defaultShortShapeNgramLength = 4 + // Structural punctuation kept verbatim inside shape runs (everything else that // is not a letter or digit is a shape boundary). These are the separators that // give numeric patterns their selectivity: dots, dashes, slashes, colons, diff --git a/src/createIndex.js b/src/createIndex.js index 80aafad..bbf0757 100644 --- a/src/createIndex.js +++ b/src/createIndex.js @@ -1,6 +1,6 @@ import { parquetMetadataAsync, parquetReadObjects } from 'hyparquet' import { ParquetWriter, schemaFromColumnData } from 'hyparquet-writer' -import { chooseIndexRowGroupSize, defaultBlockSize, defaultHexShapeNgramLength, defaultIndexPageSize, defaultNgramChars, defaultNgramLength, defaultShapeChars, defaultShapeNgramLength, hypGrepVersion } from './constants.js' +import { chooseIndexRowGroupSize, defaultBlockSize, defaultHexShapeNgramLength, defaultIndexPageSize, defaultNgramChars, defaultNgramLength, defaultShapeChars, defaultShapeNgramLength, defaultShortShapeNgramLength, hypGrepVersion } from './constants.js' import { extractNgrams } from './ngrams.js' import { extractStructuralNgrams } from './shape.js' import { assertNonNegativeSafeInteger, assertPositiveSafeInteger, getTextColumnsFromSchema } from './utils.js' @@ -39,6 +39,7 @@ export async function createIndex({ // tokenization of the same text. Its settings are recorded in kv metadata so // queries tokenize identically and so an index built without it is detected. const shapeNgramLength = defaultShapeNgramLength + const shortShapeNgramLength = defaultShortShapeNgramLength const shapeChars = defaultShapeChars const hexShapeNgramLength = defaultHexShapeNgramLength @@ -124,6 +125,7 @@ export async function createIndex({ ngramLength, ngramChars, shapeNgramLength, + shortShapeNgramLength, shapeChars, hexShapeNgramLength, blockNgrams @@ -148,6 +150,7 @@ export async function createIndex({ // Shape (character-class skeleton) layer. Absence of these keys means the // index has no shape tokens, so queryIndex must not require any. { key: 'hypgrep.shape_ngram_length', value: String(shapeNgramLength) }, + { key: 'hypgrep.short_shape_ngram_length', value: String(shortShapeNgramLength) }, { key: 'hypgrep.shape_chars', value: shapeChars }, { key: 'hypgrep.hex_shape_ngram_length', value: String(hexShapeNgramLength) }, { key: 'hypgrep.text_columns', value: textColumns.join(',') }, @@ -240,7 +243,8 @@ export async function createIndex({ * @param {string[]} textColumns * @param {number} n * @param {string} chars extra characters kept inside n-gram runs - * @param {number} shapeN shape n-gram length + * @param {number} shapeN primary shape n-gram length + * @param {number} shortShapeN fallback shape n-gram length * @param {string} shapeChars structural punctuation kept verbatim in shapes * @param {number} hexShapeN hexadecimal shape n-gram length * @param {Set} ngrams block accumulator to add into @@ -251,6 +255,7 @@ function collectRowNgrams( n, chars, shapeN, + shortShapeN, shapeChars, hexShapeN, ngrams @@ -262,8 +267,14 @@ function collectRowNgrams( if (value.length >= n) { for (const g of extractNgrams(value, n, chars)) ngrams.add(g) } - // Both structural projections share one source traversal and stream - // directly into the block accumulator. - extractStructuralNgrams(value, shapeN, hexShapeN, shapeChars, ngrams) + // All structural layers share one source traversal and stream directly + // into the block accumulator. + extractStructuralNgrams( + value, + [shapeN, shortShapeN], + hexShapeN, + shapeChars, + ngrams + ) } } diff --git a/src/queryIndex.js b/src/queryIndex.js index d5ae2f2..429526a 100644 --- a/src/queryIndex.js +++ b/src/queryIndex.js @@ -37,6 +37,7 @@ export async function queryIndex({ query, indexFile, indexMetadata }) { ngramLength, ngramChars, shapeNgramLength, + shortShapeNgramLength, shapeChars, hexShapeNgramLength, textColumns, @@ -51,23 +52,27 @@ export async function queryIndex({ query, indexFile, indexMetadata }) { // - RegExp query: one branch per top-level alternation arm, plus any mandatory // shape n-grams (character-class skeletons) ANDed in. Shape tokens let a // literal-free pattern (a phone number, SSN, IP) prune by its structure. - // Gated on shapeNgramLength so an index with no shape layer is never asked - // for shape tokens (which would be a false negative). + // Gated on recorded shape lengths so an index without a given layer is + // never asked for its tokens (which would be a false negative). /** @type {string[][]} */ let branches if (query instanceof RegExp) { const shapeGrams = shapeNgramLength ? extractRegexShapes(query, shapeNgramLength, shapeChars) : [] + const shortShapeGrams = shortShapeNgramLength + ? extractRegexShapes(query, shortShapeNgramLength, shapeChars) + : [] const hexShapeGrams = hexShapeNgramLength ? extractRegexHexShapes(query, hexShapeNgramLength, shapeChars) : [] branches = extractRegexLiterals(query).map(lits => { const grams = literalsToNgrams(lits, ngramLength, ngramChars) - // Prefer the narrowest cheap proof. Ordinary literals normally win; - // the hex projection is only needed when the ordinary D/L projection - // cannot resolve a mixed hexadecimal class. + // Prefer the narrowest cheap proof. Ordinary literals normally win, + // followed by either selective 8-symbol projection. The shorter ordinary + // layer is only a fallback when neither long projection is available. if (grams.length) return grams if (shapeGrams.length) return shapeGrams - return hexShapeGrams + if (hexShapeGrams.length) return hexShapeGrams + return shortShapeGrams }) } else { branches = [queryNgrams(query, ngramLength, ngramChars)] @@ -221,6 +226,8 @@ export function parseKvMetadata(kvMetadata) { // without a shape layer, which signals queryIndex not to require shape tokens. /** @type {number | undefined} */ let shapeNgramLength + /** @type {number | undefined} */ + let shortShapeNgramLength let shapeChars = '' /** @type {number | undefined} */ let hexShapeNgramLength @@ -248,6 +255,10 @@ export function parseKvMetadata(kvMetadata) { const n = Number(value) if (Number.isSafeInteger(n) && n > 0) shapeNgramLength = n } + if (key === 'hypgrep.short_shape_ngram_length') { + const n = Number(value) + if (Number.isSafeInteger(n) && n > 0) shortShapeNgramLength = n + } if (key === 'hypgrep.shape_chars' && typeof value === 'string') { shapeChars = value } @@ -299,6 +310,7 @@ export function parseKvMetadata(kvMetadata) { ngramLength, ngramChars, shapeNgramLength, + shortShapeNgramLength, shapeChars, hexShapeNgramLength, textColumns, diff --git a/src/shape.js b/src/shape.js index 0b548e6..b8815d3 100644 --- a/src/shape.js +++ b/src/shape.js @@ -12,10 +12,10 @@ * n-grams. They are prefixed with `SHAPE_PREFIX` (a control byte that never * occurs in lowercased text n-grams) so the two namespaces can never collide. * - * Index and query MUST use the same shape length and kept-character set, or the - * tokens won't line up; createIndex records both in kv metadata and queryIndex - * reads them back. An index built without a shape layer simply has no shape - * tokens, and queryIndex must not require any (it gates on the metadata). + * Index and query MUST use the same shape lengths and kept-character set, or + * the tokens won't line up; createIndex records them in kv metadata and + * queryIndex reads them back. An index built without a shape layer simply has + * no shape tokens, and queryIndex must not require any. */ // Control byte that namespaces shape tokens away from text n-grams. @@ -87,10 +87,11 @@ export function extractHexShapeNgrams(text, shapeN, shapeChars, out = new Set()) } /** - * Extract ordinary and hexadecimal shape n-grams in one source traversal. + * Extract one or more ordinary shape lengths and hexadecimal shape n-grams in + * one source traversal. * * @param {string} text - * @param {number} shapeN shape n-gram length + * @param {number | number[]} shapeN ordinary shape n-gram length(s) * @param {number} hexShapeN hexadecimal shape n-gram length * @param {string} shapeChars structural punctuation kept verbatim in shapes * @param {Set} [out] accumulator to add grams into (default: fresh set) @@ -112,7 +113,7 @@ export function extractStructuralNgrams( * both layers and pays for one traversal. * * @param {string} text - * @param {number | undefined} shapeN ordinary shape n-gram length + * @param {number | number[] | undefined} shapeN ordinary shape n-gram length(s) * @param {number | undefined} hexShapeN hexadecimal shape n-gram length * @param {string} shapeChars structural punctuation kept verbatim in shapes * @param {Set} out accumulator to add grams into @@ -120,16 +121,28 @@ export function extractStructuralNgrams( */ function extractStructuralLayers(text, shapeN, hexShapeN, shapeChars, out) { if (typeof text !== 'string') return out - const shapeLength = shapeN ?? 0 + const shapeLengths = shapeN === undefined + ? [] + : Array.isArray(shapeN) ? shapeN : [shapeN] const hexShapeLength = hexShapeN ?? 0 - const useShape = shapeLength > 0 && text.length >= shapeLength + /** @type {{length: number, informative: number}[]} */ + const shapeLayers = [] + let shapeWindowLength = 0 + for (const length of shapeLengths) { + if (length > 0 && text.length >= length) { + shapeLayers.push({ + length, + informative: 0, + }) + shapeWindowLength = Math.max(shapeWindowLength, length) + } + } + const useShape = shapeLayers.length > 0 + const shapeWindow = useShape ? new Array(shapeWindowLength) : [] + let shapeCount = 0 const useHex = hexShapeLength > 0 && text.length >= hexShapeLength if (!useShape && !useHex) return out - const shapeWindow = useShape ? new Array(shapeLength) : [] - let shapeCount = 0 - let shapeInformative = 0 - const hexWindow = useHex ? new Array(hexShapeLength) : [] let hexCount = 0 let hexNonUniform = 0 @@ -150,22 +163,34 @@ function extractStructuralLayers(text, shapeN, hexShapeN, shapeChars, out) { const symbol = digit ? 'D' : letter ? 'L' : kept ? char : null if (symbol === null) { shapeCount = 0 - shapeInformative = 0 + for (const layer of shapeLayers) layer.informative = 0 } else { - const slot = shapeCount % shapeLength - if (shapeCount >= shapeLength && shapeWindow[slot] !== 'L') { - shapeInformative -= 1 + // Every ordinary length uses the same projected symbol stream. Keep one + // ring sized for the longest layer and update each layer's informative + // count before overwriting the oldest symbol. + for (const layer of shapeLayers) { + if (shapeCount >= layer.length) { + const outgoing = + shapeWindow[(shapeCount - layer.length) % shapeWindowLength] + if (outgoing !== 'L') layer.informative -= 1 + } + } + shapeWindow[shapeCount % shapeWindowLength] = symbol + if (symbol !== 'L') { + for (const layer of shapeLayers) layer.informative += 1 } - shapeWindow[slot] = symbol - if (symbol !== 'L') shapeInformative += 1 shapeCount += 1 - if (shapeCount >= shapeLength && shapeInformative > 0) { - let gram = SHAPE_PREFIX - for (let k = shapeCount % shapeLength, n = 0; n < shapeLength; n += 1) { - gram += shapeWindow[k] - k = k + 1 === shapeLength ? 0 : k + 1 + + for (const layer of shapeLayers) { + if (shapeCount >= layer.length && layer.informative > 0) { + let gram = SHAPE_PREFIX + let k = (shapeCount - layer.length) % shapeWindowLength + for (let n = 0; n < layer.length; n += 1) { + gram += shapeWindow[k] + k = k + 1 === shapeWindowLength ? 0 : k + 1 + } + out.add(gram) } - out.add(gram) } } } diff --git a/src/types.d.ts b/src/types.d.ts index e45b737..566d64e 100644 --- a/src/types.d.ts +++ b/src/types.d.ts @@ -64,7 +64,8 @@ export interface HypGrepMetadata { blockSize: number // number of rows per logical block ngramLength: number // n-gram size used to build the index ngramChars: string // extra characters kept inside n-gram runs ('' for pre-structural indexes) - shapeNgramLength?: number // shape n-gram size, or undefined if the index has no shape layer + shapeNgramLength?: number // primary shape n-gram size, or undefined if the index has no shape layer + shortShapeNgramLength?: number // fallback shape n-gram size, or undefined if absent shapeChars: string // structural punctuation kept verbatim in shapes ('' if no shape layer) hexShapeNgramLength?: number // hexadecimal shape n-gram size, or undefined if absent textColumns: string[] // list of indexed text columns diff --git a/test/createIndex.test.js b/test/createIndex.test.js index a8add43..effe519 100644 --- a/test/createIndex.test.js +++ b/test/createIndex.test.js @@ -27,22 +27,23 @@ describe('createIndex', () => { expect(existsSync(TEST_INDEX)).toBe(true) const indexBuffer = await asyncBufferFromFile(TEST_INDEX) - expect(indexBuffer.byteLength).toBe(2620) + expect(indexBuffer.byteLength).toBe(2658) const indexMetadata = await parquetMetadataAsync(indexBuffer) expect(indexMetadata.row_groups.length).toBe(7) expect(indexMetadata.num_rows).toBe(676n) - expect(indexMetadata.key_value_metadata?.length).toBe(10) + expect(indexMetadata.key_value_metadata?.length).toBe(11) const kv = indexMetadata.key_value_metadata expect(kv?.[0]).toEqual({ key: 'hypgrep.version', value: '0' }) expect(kv?.[1]).toEqual({ key: 'hypgrep.block_size', value: '200' }) expect(kv?.[2]).toEqual({ key: 'hypgrep.ngram_length', value: '5' }) expect(kv?.[3]).toEqual({ key: 'hypgrep.ngram_chars', value: '"{}:' }) expect(kv?.[4]).toEqual({ key: 'hypgrep.shape_ngram_length', value: '8' }) - expect(kv?.[5]).toEqual({ key: 'hypgrep.shape_chars', value: '@.-_/:' }) - expect(kv?.[6]).toEqual({ key: 'hypgrep.hex_shape_ngram_length', value: '8' }) - expect(kv?.[7]).toEqual({ key: 'hypgrep.text_columns', value: 'id' }) - expect(kv?.[8]).toEqual({ key: 'hypgrep.source_rows', value: '676' }) - expect(kv?.[9]).toEqual({ key: 'hypgrep.source_bytelength', value: String(sourceFile.byteLength) }) + expect(kv?.[5]).toEqual({ key: 'hypgrep.short_shape_ngram_length', value: '4' }) + expect(kv?.[6]).toEqual({ key: 'hypgrep.shape_chars', value: '@.-_/:' }) + expect(kv?.[7]).toEqual({ key: 'hypgrep.hex_shape_ngram_length', value: '8' }) + expect(kv?.[8]).toEqual({ key: 'hypgrep.text_columns', value: 'id' }) + expect(kv?.[9]).toEqual({ key: 'hypgrep.source_rows', value: '676' }) + expect(kv?.[10]).toEqual({ key: 'hypgrep.source_bytelength', value: String(sourceFile.byteLength) }) }) it('should reject invalid sizing options', async () => { diff --git a/test/queryIndex.test.js b/test/queryIndex.test.js index b3a037a..4cdf813 100644 --- a/test/queryIndex.test.js +++ b/test/queryIndex.test.js @@ -240,6 +240,21 @@ describe('queryIndex', () => { } }) + it('treats a malformed short_shape_ngram_length as no fallback shape layer', () => { + for (const value of ['', 'null', '0', undefined]) { + const kv = [ + { key: 'hypgrep.version', value: '0' }, + { key: 'hypgrep.block_size', value: '100' }, + { key: 'hypgrep.ngram_length', value: '5' }, + { key: 'hypgrep.short_shape_ngram_length', value }, + { key: 'hypgrep.text_columns', value: 'id' }, + { key: 'hypgrep.source_rows', value: '10' }, + { key: 'hypgrep.source_bytelength', value: '100' }, + ] + expect(parseKvMetadata(kv).shortShapeNgramLength).toBeUndefined() + } + }) + it('treats a malformed hex_shape_ngram_length as no hex shape layer', () => { for (const value of ['', 'null', '0', undefined]) { const kv = [ diff --git a/test/shape.test.js b/test/shape.test.js index 37b302b..847cca7 100644 --- a/test/shape.test.js +++ b/test/shape.test.js @@ -101,7 +101,7 @@ describe('hexadecimal shapes', () => { expect(grams).toContain(HEX_SHAPE_PREFIX + 'HHHHHHHH') }) - it('fuses both projections without changing their tokens', () => { + it('fuses both ordinary lengths and the hex projection without changing their tokens', () => { const samples = [ 'call 415-555-1234', 'id 123e4567-e89b-12d3-a456-426614174000', @@ -110,8 +110,9 @@ describe('hexadecimal shapes', () => { ] for (const text of samples) { const separate = extractShapeNgrams(text, 4, SHAPE_CHARS) + extractShapeNgrams(text, 8, SHAPE_CHARS, separate) extractHexShapeNgrams(text, 8, SHAPE_CHARS, separate) - expect(extractStructuralNgrams(text, 4, 8, SHAPE_CHARS)).toEqual(separate) + expect(extractStructuralNgrams(text, [8, 4], 8, SHAPE_CHARS)).toEqual(separate) } }) }) @@ -119,15 +120,17 @@ describe('hexadecimal shapes', () => { const SRC = 'test/files/shape.source.parquet' const IDX = 'test/files/shape.index.parquet' -// One row per block. Four rows carry structured values; the rest are prose with +// One row per block. Five rows carry structured values; the rest are prose with // scattered digits that must NOT shape-match an SSN / phone / IP pattern. function writeSource() { const rows = [] for (let i = 0; i < 20; i += 1) { - if (i === 1) rows.push('request id 123e4567-e89b-12d3-a456-426614174000') + if (i === 1) rows.push('tag-123e4567-e89b-12d3-a456-426614174000') else if (i === 4) rows.push('my ssn is 123-45-6789 please keep it private') else if (i === 9) rows.push('reach me at 415-555-0137 any time') else if (i === 15) rows.push('the server lives at 10.0.12.255 on the lan') + else if (i === 18) rows.push('expires 07/30') + else if (i === 19) rows.push('prefix abc-noise') else rows.push(`order ${i} shipped on day ${i * 3} with code A${i}B`) } parquetWriteFile({ filename: SRC, columnData: [{ name: 'text', data: rows }] }) @@ -190,6 +193,26 @@ describe('shape layer (literal-free pattern regexes)', () => { expect(await find(uuid)).toEqual([1]) }) + it('prefers a hexadecimal shape over the short ordinary fallback', async () => { + writeSource() + await createIndex({ sourceFile: await asyncBufferFromFile(SRC), indexFile: fileWriter(IDX), blockSize: 1 }) + + const taggedUuid = /tag-[0-9a-f]{8}-[0-9a-f]{4}-[0-9a-f]{4}-[0-9a-f]{4}-[0-9a-f]{12}\b/i + expect(extractRegexShapes(taggedUuid, 8, SHAPE_CHARS)).toEqual([]) + expect(extractRegexShapes(taggedUuid, 4, SHAPE_CHARS).length).toBeGreaterThan(0) + expect(extractRegexHexShapes(taggedUuid, 8, SHAPE_CHARS).length).toBeGreaterThan(0) + expect(await candBlocks(taggedUuid)).toBe(1) + expect(await find(taggedUuid)).toEqual([1]) + }) + + it('falls back to four-symbol shapes for short structural regexes', async () => { + writeSource() + await createIndex({ sourceFile: await asyncBufferFromFile(SRC), indexFile: fileWriter(IDX), blockSize: 1 }) + + expect(await candBlocks(/\d{2}\/\d{2}/)).toBe(1) + expect(await find(/\d{2}\/\d{2}/)).toEqual([18]) + }) + it('returns exactly the brute-force result set (no false negatives)', async () => { writeSource() await createIndex({ sourceFile: await asyncBufferFromFile(SRC), indexFile: fileWriter(IDX), blockSize: 1 }) @@ -220,7 +243,8 @@ describe('shape layer (literal-free pattern regexes)', () => { writeSource() await createIndex({ sourceFile: await asyncBufferFromFile(SRC), indexFile: fileWriter(IDX), blockSize: 1 }) // Eight-symbol windows distinguish the credit-card grouping from the - // shorter SSN and phone groupings in the source. + // shorter SSN and phone groupings. The query must not use the less + // selective four-symbol fallback when a primary shape is available. expect(await candBlocks(/\d{4}-\d{4}-\d{4}-\d{4}/)).toBe(0) expect(await find(/\d{4}-\d{4}-\d{4}-\d{4}/)).toEqual([]) })