From e28f8bad3cceccdc92052288bbb49eae11296de6 Mon Sep 17 00:00:00 2001
From: Mini256 <minianter@foxmail.com>
Date: Sat, 22 Jun 2024 23:27:34 +0800
Subject: [PATCH] feat: metadata filter support strict mode (#170)

---
 src/lib/llamaindex/builders/metadata-filter.ts        |  3 ++-
 src/lib/llamaindex/config/metadata-filter.ts          |  3 ++-
 .../postprocessors/postfilters/MetadataPostFilter.ts  | 11 +++++++++--
 3 files changed, 13 insertions(+), 4 deletions(-)

diff --git a/src/lib/llamaindex/builders/metadata-filter.ts b/src/lib/llamaindex/builders/metadata-filter.ts
index 411e3552..75c9381f 100644
--- a/src/lib/llamaindex/builders/metadata-filter.ts
+++ b/src/lib/llamaindex/builders/metadata-filter.ts
@@ -15,7 +15,8 @@ export function buildMetadataFilter (serviceContext: ServiceContext, config: Met
         return new MetadataPostFilter({
           llm,
           metadata_fields: config.options?.metadata_fields,
-          filters: config.options?.filters
+          filters: config.options?.filters,
+          strict: config.options?.strict,
         });
       default:
         throw new Error(`Unknown metadata filter provider: ${config.provider}`)
diff --git a/src/lib/llamaindex/config/metadata-filter.ts b/src/lib/llamaindex/config/metadata-filter.ts
index 743f38ab..205afcb5 100644
--- a/src/lib/llamaindex/config/metadata-filter.ts
+++ b/src/lib/llamaindex/config/metadata-filter.ts
@@ -28,7 +28,8 @@ export enum MetadataFilterProvider {
 export const DefaultMetadataFilterOptions = z.object({
   llm: LLMConfigSchema.optional(),
   metadata_fields: z.array(metadataFieldSchema).optional(),
-  filters: z.array(metadataFilterSchema).optional()
+  filters: z.array(metadataFilterSchema).optional(),
+  strict: z.boolean().optional()
 });
 
 export const DefaultMetadataFilterConfig = z.object({
diff --git a/src/lib/llamaindex/postprocessors/postfilters/MetadataPostFilter.ts b/src/lib/llamaindex/postprocessors/postfilters/MetadataPostFilter.ts
index f064fe28..37ece554 100644
--- a/src/lib/llamaindex/postprocessors/postfilters/MetadataPostFilter.ts
+++ b/src/lib/llamaindex/postprocessors/postfilters/MetadataPostFilter.ts
@@ -66,6 +66,12 @@ export class MetadataPostFilter implements BaseNodePostprocessor {
    * Provide the filters to apply to the search.
    */
   filters: MetadataFieldFilter[] | null;
+  /**
+   * Whether to strictly filter out nodes that do not meet the filtering conditions.
+   * In strict mode, if a node has no metadata or lacks the metadata field in the filtering conditions,
+   * it is considered not to meet the filtering conditions.
+   */
+  strict: boolean;
 
   constructor(init?: MetadataPostFilterOptions) {
     this.serviceContext = init?.serviceContext ?? serviceContextFromDefaults();
@@ -73,6 +79,7 @@ export class MetadataPostFilter implements BaseNodePostprocessor {
     this.metadata_fields = init?.metadata_fields ?? [];
     this.filters = init?.filters ?? null;
     this.metadataFilterChoicePrompt = init?.metadataFilterChoicePrompt || defaultMetadataFilterChoicePrompt;
+    this.strict = init?.strict ?? false;
   }
 
   async postprocessNodes(nodes: NodeWithScore[], query: string): Promise<NodeWithScore[]> {
@@ -124,13 +131,13 @@ export class MetadataPostFilter implements BaseNodePostprocessor {
 
       // If the document metadata is not found, skip the filter.
       if (!metadata) {
-        return true;
+        return !this.strict;
       }
 
       return filters.every(filter => {
         // If the document metadata field is not set, skip the filter.
         if (!metadata[filter.name] || !filter.value || filter.value === '') {
-          return true;
+          return !this.strict;
         }
         return metadata[filter.name] === filter.value;
       });