feat: metadata filter support strict mode (#170)

pingcap · Jun 22, 2024 · e28f8ba · e28f8ba
1 parent 3100f08
commit e28f8ba
Show file tree

Hide file tree

Showing 3 changed files with 13 additions and 4 deletions.
diff --git a/src/lib/llamaindex/builders/metadata-filter.ts b/src/lib/llamaindex/builders/metadata-filter.ts
@@ -15,7 +15,8 @@ export function buildMetadataFilter (serviceContext: ServiceContext, config: Met
         return new MetadataPostFilter({
           llm,
           metadata_fields: config.options?.metadata_fields,
-          filters: config.options?.filters
+          filters: config.options?.filters,
+          strict: config.options?.strict,
         });
       default:
         throw new Error(`Unknown metadata filter provider: ${config.provider}`)

diff --git a/src/lib/llamaindex/config/metadata-filter.ts b/src/lib/llamaindex/config/metadata-filter.ts
@@ -28,7 +28,8 @@ export enum MetadataFilterProvider {
 export const DefaultMetadataFilterOptions = z.object({
   llm: LLMConfigSchema.optional(),
   metadata_fields: z.array(metadataFieldSchema).optional(),
-  filters: z.array(metadataFilterSchema).optional()
+  filters: z.array(metadataFilterSchema).optional(),
+  strict: z.boolean().optional()
 });
 
 export const DefaultMetadataFilterConfig = z.object({

diff --git a/src/lib/llamaindex/postprocessors/postfilters/MetadataPostFilter.ts b/src/lib/llamaindex/postprocessors/postfilters/MetadataPostFilter.ts
@@ -66,13 +66,20 @@ export class MetadataPostFilter implements BaseNodePostprocessor {
    * Provide the filters to apply to the search.
    */
   filters: MetadataFieldFilter[] | null;
+  /**
+   * Whether to strictly filter out nodes that do not meet the filtering conditions.
+   * In strict mode, if a node has no metadata or lacks the metadata field in the filtering conditions,
+   * it is considered not to meet the filtering conditions.
+   */
+  strict: boolean;
 
   constructor(init?: MetadataPostFilterOptions) {
     this.serviceContext = init?.serviceContext ?? serviceContextFromDefaults();
     this.llm = init?.llm ?? this.serviceContext.llm;
     this.metadata_fields = init?.metadata_fields ?? [];
     this.filters = init?.filters ?? null;
     this.metadataFilterChoicePrompt = init?.metadataFilterChoicePrompt || defaultMetadataFilterChoicePrompt;
+    this.strict = init?.strict ?? false;
   }
 
   async postprocessNodes(nodes: NodeWithScore[], query: string): Promise<NodeWithScore[]> {
@@ -124,13 +131,13 @@ export class MetadataPostFilter implements BaseNodePostprocessor {
 
       // If the document metadata is not found, skip the filter.
       if (!metadata) {
-        return true;
+        return !this.strict;
       }
 
       return filters.every(filter => {
         // If the document metadata field is not set, skip the filter.
         if (!metadata[filter.name] || !filter.value || filter.value === '') {
-          return true;
+          return !this.strict;
         }
         return metadata[filter.name] === filter.value;
       });