add code for splitting into individual results and deduplication

build/.yarnrc

@@ -0,0 +1,4 @@
+disturl "https://nodejs.org/download/release"
+target "20.13.1"
+runtime "node"
+arch "x64"

src/vs/workbench/services/search/common/searchExtTypes.ts

@@ -295,7 +295,7 @@ export class TextSearchMatchNew {
 	 */
 	constructor(
 		public uri: URI,
-		public ranges: { sourceRange: Range; previewRange: Range }[],
+		public ranges: { sourceRange: Range; previewRange: Range },
 		public previewText: string) { }
 
 }

src/vs/workbench/services/search/common/searchExtTypesInternal.ts

@@ -2,7 +2,8 @@
  *  Copyright (c) Microsoft Corporation. All rights reserved.
  *  Licensed under the MIT License. See License.txt in the project root for license information.
  *--------------------------------------------------------------------------------------------*/
-import { FileSearchProviderFolderOptions, FileSearchProviderOptions, TextSearchProviderFolderOptions, TextSearchProviderOptions } from './searchExtTypes.js';
+import { URI } from '../../../../base/common/uri.js';
+import { FileSearchProviderFolderOptions, FileSearchProviderOptions, TextSearchProviderFolderOptions, TextSearchProviderOptions, Range } from './searchExtTypes.js';
 
 interface RipgrepSearchOptionsCommon {
 	numThreads?: number;
@@ -19,3 +20,39 @@ export type FileSearchProviderOptionsRipgrep = & {
 export interface RipgrepTextSearchOptions extends TextSearchProviderOptionsRipgrep, RipgrepSearchOptionsCommon { }
 
 export interface RipgrepFileSearchOptions extends FileSearchProviderOptionsRipgrep, RipgrepSearchOptionsCommon { }
+
+/**
+ * The main match information for a {@link TextSearchResultNew}.
+ */
+export class RipgrepTextSearchMatch {
+	/**
+	 * @param uri The uri for the matching document.
+	 * @param ranges The ranges associated with this match.
+	 * @param previewText The text that is used to preview the match. The highlighted range in `previewText` is specified in `ranges`.
+	 */
+	constructor(
+		public uri: URI,
+		public ranges: { sourceRange: Range; previewRange: Range }[],
+		public previewText: string) { }
+
+}
+
+/**
+ * The potential context information for a {@link TextSearchResultNew}.
+ */
+export class RipgrepTextSearchContext {
+	/**
+	 * @param uri The uri for the matching document.
+	 * @param text The line of context text.
+	 * @param lineNumber The line number of this line of context.
+	 */
+	constructor(
+		public uri: URI,
+		public text: string,
+		public lineNumber: number) { }
+}
+
+/**
+ * A result payload for a text search, pertaining to matches within a single file.
+ */
+export type RipgrepTextSearchResult = RipgrepTextSearchMatch | RipgrepTextSearchContext;

src/vs/workbench/services/search/node/ripgrepTextSearchEngine.ts

@@ -14,41 +14,22 @@ import { createRegExp, escapeRegExpCharacters } from '../../../../base/common/st
 import { URI } from '../../../../base/common/uri.js';
 import { Progress } from '../../../../platform/progress/common/progress.js';
 import { DEFAULT_MAX_SEARCH_RESULTS, IExtendedExtensionSearchOptions, ITextSearchPreviewOptions, SearchError, SearchErrorCode, serializeSearchError, TextSearchMatch } from '../common/search.js';
-import { Range, TextSearchCompleteNew, TextSearchContextNew, TextSearchMatchNew, TextSearchProviderOptions, TextSearchQueryNew, TextSearchResultNew } from '../common/searchExtTypes.js';
+import { Range, TextSearchCompleteNew, TextSearchContextNew, TextSearchMatchNew, TextSearchProviderNew, TextSearchProviderOptions, TextSearchQueryNew, TextSearchResultNew } from '../common/searchExtTypes.js';
 import { AST as ReAST, RegExpParser, RegExpVisitor } from 'vscode-regexpp';
 import { rgPath } from '@vscode/ripgrep';
 import { anchorGlob, IOutputChannel, Maybe, rangeToSearchRange, searchRangeToRange } from './ripgrepSearchUtils.js';
-import type { RipgrepTextSearchOptions } from '../common/searchExtTypesInternal.js';
+import { RipgrepTextSearchContext, RipgrepTextSearchMatch, RipgrepTextSearchOptions, RipgrepTextSearchResult } from '../common/searchExtTypesInternal.js';
 import { newToOldPreviewOptions } from '../common/searchExtConversionTypes.js';
+import { ResourceMap } from '../../../../base/common/map.js';
 
 // If @vscode/ripgrep is in an .asar file, then the binary is unpacked.
 const rgDiskPath = rgPath.replace(/\bnode_modules\.asar\b/, 'node_modules.asar.unpacked');
 
 export class RipgrepTextSearchEngine {
 
-	constructor(private outputChannel: IOutputChannel, private readonly _numThreads?: number | undefined) { }
+	constructor(private outputChannel: IOutputChannel, protected readonly _numThreads?: number | undefined) { }
 
-	provideTextSearchResults(query: TextSearchQueryNew, options: TextSearchProviderOptions, progress: Progress<TextSearchResultNew>, token: CancellationToken): Promise<TextSearchCompleteNew> {
-		return Promise.all(options.folderOptions.map(folderOption => {
-			const extendedOptions: RipgrepTextSearchOptions = {
-				folderOptions: folderOption,
-				numThreads: this._numThreads,
-				maxResults: options.maxResults,
-				previewOptions: options.previewOptions,
-				maxFileSize: options.maxFileSize,
-				surroundingContext: options.surroundingContext
-			};
-			return this.provideTextSearchResultsWithRgOptions(query, extendedOptions, progress, token);
-		})).then((e => {
-			const complete: TextSearchCompleteNew = {
-				// todo: get this to actually check
-				limitHit: e.some(complete => !!complete && complete.limitHit)
-			};
-			return complete;
-		}));
-	}
-
-	provideTextSearchResultsWithRgOptions(query: TextSearchQueryNew, options: RipgrepTextSearchOptions, progress: Progress<TextSearchResultNew>, token: CancellationToken): Promise<TextSearchCompleteNew> {
+	provideTextSearchResultsWithRgOptions(query: TextSearchQueryNew, options: RipgrepTextSearchOptions, progress: Progress<RipgrepTextSearchResult>, token: CancellationToken): Promise<TextSearchCompleteNew> {
 		this.outputChannel.appendLine(`provideTextSearchResults ${query.pattern}, ${JSON.stringify({
 			...options,
 			...{
@@ -81,7 +62,7 @@ export class RipgrepTextSearchEngine {
 
 			let gotResult = false;
 			const ripgrepParser = new RipgrepParser(options.maxResults ?? DEFAULT_MAX_SEARCH_RESULTS, options.folderOptions.folder, newToOldPreviewOptions(options.previewOptions));
-			ripgrepParser.on('result', (match: TextSearchResultNew) => {
+			ripgrepParser.on('result', (match: RipgrepTextSearchResult) => {
 				gotResult = true;
 				dataWithoutResult = '';
 				progress.report(match);
@@ -223,7 +204,7 @@ export class RipgrepParser extends EventEmitter {
 	}
 
 
-	override on(event: 'result', listener: (result: TextSearchResultNew) => void): this;
+	override on(event: 'result', listener: (result: RipgrepTextSearchResult) => void): this;
 	override on(event: 'hitLimit', listener: () => void): this;
 	override on(event: string, listener: (...args: any[]) => void): this {
 		super.on(event, listener);
@@ -295,7 +276,7 @@ export class RipgrepParser extends EventEmitter {
 		}
 	}
 
-	private createTextSearchMatch(data: IRgMatch, uri: URI): TextSearchMatchNew {
+	private createTextSearchMatch(data: IRgMatch, uri: URI): RipgrepTextSearchMatch {
 		const lineNumber = data.line_number - 1;
 		const fullText = bytesOrTextToString(data.lines);
 		const fullTextBytes = Buffer.from(fullText);
@@ -351,7 +332,7 @@ export class RipgrepParser extends EventEmitter {
 		const searchRange = mapArrayOrNot(<Range[]>ranges, rangeToSearchRange);
 
 		const internalResult = new TextSearchMatch(fullText, searchRange, this.previewOptions);
-		return new TextSearchMatchNew(
+		return new RipgrepTextSearchMatch(
 			uri,
 			internalResult.rangeLocations.map(e => (
 				{
@@ -362,16 +343,16 @@ export class RipgrepParser extends EventEmitter {
 			internalResult.previewText);
 	}
 
-	private createTextSearchContexts(data: IRgMatch, uri: URI): TextSearchContextNew[] {
+	private createTextSearchContexts(data: IRgMatch, uri: URI): RipgrepTextSearchContext[] {
 		const text = bytesOrTextToString(data.lines);
 		const startLine = data.line_number;
 		return text
 			.replace(/\r?\n$/, '')
 			.split('\n')
-			.map((line, i) => new TextSearchContextNew(uri, line, startLine + i));
+			.map((line, i) => new RipgrepTextSearchContext(uri, line, startLine + i));
 	}
 
-	private onResult(match: TextSearchResultNew): void {
+	private onResult(match: RipgrepTextSearchResult): void {
 		this.emit('result', match);
 	}
 }
@@ -777,3 +758,135 @@ export function performBraceExpansionForRipgrep(pattern: string): string[] {
 		});
 	});
 }
+
+export class SimpleRipgrepTextSearchProvider extends RipgrepTextSearchEngine implements TextSearchProviderNew {
+	provideTextSearchResults(query: TextSearchQueryNew, options: TextSearchProviderOptions, progress: Progress<TextSearchResultNew>, token: CancellationToken): Promise<TextSearchCompleteNew> {
+		return Promise.all(options.folderOptions.map(folderOption => {
+			const extendedOptions: RipgrepTextSearchOptions = {
+				folderOptions: folderOption,
+				numThreads: this._numThreads,
+				maxResults: options.maxResults,
+				previewOptions: options.previewOptions,
+				maxFileSize: options.maxFileSize,
+				surroundingContext: options.surroundingContext
+			};
+
+			const dedupMatcher = new DeDuplicationMatcher(progress);
+
+			const splitRipgrepResultAndReport = (e: RipgrepTextSearchMatch) => {
+				e.ranges.forEach(r => {
+					const textSearchMatchNew = new TextSearchMatchNew(e.uri, { sourceRange: r.sourceRange, previewRange: r.previewRange }, e.previewText);
+					dedupMatcher.adopterProgress.report(textSearchMatchNew);
+				});
+			};
+			return this.provideTextSearchResultsWithRgOptions(query, extendedOptions, new Progress(p => {
+				if (p instanceof RipgrepTextSearchMatch) {
+					splitRipgrepResultAndReport(p);
+				} else {
+					dedupMatcher.adopterProgress.report(p);
+				}
+			}), token);
+		})).then((e => {
+			const complete: TextSearchCompleteNew = {
+				// todo: get this to actually check
+				limitHit: e.some(complete => !!complete && complete.limitHit)
+			};
+			return complete;
+		}));
+	}
+}
+
+class DeDuplicationMatcher {
+	private matchers = new ResourceMap<DeDuplicationMatcherForFile>();
+	public adopterProgress: Progress<TextSearchResultNew>;
+	constructor(private readonly progress: Progress<TextSearchResultNew>) {
+		this.adopterProgress = new Progress<TextSearchResultNew>(result => {
+			if (result instanceof TextSearchMatchNew) {
+				const startLine = result.ranges.sourceRange.start.line;
+				const endLine = result.ranges.sourceRange.end.line;
+				this.sendPreview(result.uri, startLine === endLine ? startLine : [startLine, endLine], result.previewText, result.ranges);
+			} else {
+				this.sendContext(result.uri, result.lineNumber, result.text);
+			}
+		});
+	}
+
+	sendContext(uri: URI, line: number, text: string) {
+		let matcher = this.matchers.get(uri);
+		if (!matcher) {
+			matcher = new DeDuplicationMatcherForFile(uri, this.progress);
+			this.matchers.set(uri, matcher);
+		}
+		if (matcher) {
+			matcher.sendContext(line, text);
+		}
+	}
+	sendPreview(uri: URI, location: number | [number, number], text: string, ranges: { sourceRange: Range; previewRange: Range }) {
+		let matcher = this.matchers.get(uri);
+		if (!matcher) {
+			matcher = new DeDuplicationMatcherForFile(uri, this.progress);
+			this.matchers.set(uri, matcher);
+		}
+		if (typeof (location) === 'number') {
+			matcher.sendOneLinePreview(location, text, ranges);
+		} else {
+			matcher.sendMultiLinePreview(location, text, ranges);
+		}
+	}
+}
+
+class DeDuplicationMatcherForFile {
+
+	// deduplicate context
+	private sentContextLines: Set<number> = new Set();
+
+	// deduplicate preview lines
+	private sentPreviewTextSingleLines: Set<number> = new Set();
+	private sentPreviewTextMultiLines: Map<number, Set<number>> = new Map();
+
+	constructor(private readonly uri: URI, private readonly progress: Progress<TextSearchResultNew>) { }
+
+	sendContext(lineNumber: number, text: string) {
+		if (!this.sentContextLines.has(lineNumber)) {
+			this.progress.report(
+				new TextSearchContextNew(
+					this.uri,
+					text,
+					lineNumber,
+				)
+			);
+			this.sentContextLines.add(lineNumber);
+		}
+	}
+	sendMultiLinePreview(lineNumber: [number, number], text: string, ranges: { sourceRange: Range; previewRange: Range }) {
+		let firstNumArray = this.sentPreviewTextMultiLines.get(lineNumber[0]);
+		const isIncluded = firstNumArray?.has(lineNumber[1]);
+		if (!isIncluded) {
+			this.progress.report(
+				new TextSearchMatchNew(
+					this.uri,
+					ranges,
+					text
+				)
+			);
+			if (!firstNumArray) {
+				firstNumArray = new Set();
+				this.sentPreviewTextMultiLines.set(lineNumber[0], firstNumArray);
+			}
+			firstNumArray.add(lineNumber[1]);
+		}
+	}
+	sendOneLinePreview(lineNumber: number, text: string, ranges: { sourceRange: Range; previewRange: Range }) {
+		if (!this.sentPreviewTextSingleLines.has(lineNumber)) {
+			this.progress.report(
+				new TextSearchMatchNew(
+					this.uri,
+					ranges,
+					text
+				)
+			);
+			this.sentPreviewTextSingleLines.add(lineNumber);
+		}
+
+	}
+}

src/vs/workbench/services/search/node/textSearchAdapter.ts

@@ -6,7 +6,7 @@
 import { CancellationToken } from '../../../../base/common/cancellation.js';
 import * as pfs from '../../../../base/node/pfs.js';
 import { IFileMatch, IProgressMessage, ITextQuery, ITextSearchMatch, ISerializedFileMatch, ISerializedSearchSuccess, resultIsMatch } from '../common/search.js';
-import { RipgrepTextSearchEngine } from './ripgrepTextSearchEngine.js';
+import { SimpleRipgrepTextSearchProvider } from './ripgrepTextSearchEngine.js';
 import { NativeTextSearchManager } from './textSearchManager.js';
 
 export class TextSearchEngineAdapter {
@@ -30,7 +30,7 @@ export class TextSearchEngineAdapter {
 				onMessage({ message: msg });
 			}
 		};
-		const textSearchManager = new NativeTextSearchManager(this.query, new RipgrepTextSearchEngine(pretendOutputChannel, this.numThreads), pfs);
+		const textSearchManager = new NativeTextSearchManager(this.query, new SimpleRipgrepTextSearchProvider(pretendOutputChannel, this.numThreads), pfs);
 		return new Promise((resolve, reject) => {
 			return textSearchManager
 				.search(