diff --git a/apps/server/src/workspaceEntries.test.ts b/apps/server/src/workspaceEntries.test.ts index d867ad910..f59ba2c94 100644 --- a/apps/server/src/workspaceEntries.test.ts +++ b/apps/server/src/workspaceEntries.test.ts @@ -22,13 +22,25 @@ function writeFile(cwd: string, relativePath: string, contents = ""): void { fs.writeFileSync(absolutePath, contents, "utf8"); } -function runGit(cwd: string, args: string[]): void { - const result = spawnSync("git", args, { cwd, encoding: "utf8" }); +function runGit(cwd: string, args: string[], options?: { config?: string[] }): void { + const gitArgs = [...(options?.config ?? []).flatMap((entry) => ["-c", entry]), ...args]; + const result = spawnSync("git", gitArgs, { cwd, encoding: "utf8" }); if (result.status !== 0) { - throw new Error(result.stderr || `git ${args.join(" ")} failed`); + throw new Error(result.stderr || `git ${gitArgs.join(" ")} failed`); } } +function initGitRepo(cwd: string): void { + runGit(cwd, ["init"]); + runGit(cwd, ["config", "user.email", "test@example.com"]); + runGit(cwd, ["config", "user.name", "Test User"]); +} + +function commitAll(cwd: string, message: string): void { + runGit(cwd, ["add", "."]); + runGit(cwd, ["commit", "-m", message]); +} + describe("searchWorkspaceEntries", () => { afterEach(() => { vi.restoreAllMocks(); @@ -144,6 +156,111 @@ describe("searchWorkspaceEntries", () => { assert.isFalse(paths.some((entryPath) => entryPath.startsWith(".convex/"))); }); + it("includes files inside initialized git submodules", async () => { + const submoduleOrigin = makeTempDir("t3code-workspace-submodule-origin-"); + initGitRepo(submoduleOrigin); + writeFile(submoduleOrigin, "src/submodule-file.ts", "export {};"); + writeFile(submoduleOrigin, "README.md", "# submodule\n"); + commitAll(submoduleOrigin, "Initial submodule"); + + const cwd = makeTempDir("t3code-workspace-submodule-root-"); + initGitRepo(cwd); + writeFile(cwd, "src/root.ts", "export {};"); + runGit(cwd, ["add", "src/root.ts"]); + runGit(cwd, ["commit", "-m", "Initial root"]); + runGit(cwd, ["submodule", "add", "-q", submoduleOrigin, "vendor/submodule"], { + config: ["protocol.file.allow=always"], + }); + writeFile(cwd, "vendor/submodule/untracked.ts", "export {};"); + + const result = await searchWorkspaceEntries({ cwd, query: "", limit: 100 }); + const paths = result.entries.map((entry) => entry.path); + const submoduleRootEntry = result.entries.find((entry) => entry.path === "vendor/submodule"); + + assert.include(paths, "vendor"); + assert.include(paths, "vendor/submodule"); + assert.include(paths, "vendor/submodule/README.md"); + assert.include(paths, "vendor/submodule/src"); + assert.include(paths, "vendor/submodule/src/submodule-file.ts"); + assert.include(paths, "vendor/submodule/untracked.ts"); + assert.deepInclude(submoduleRootEntry, { + path: "vendor/submodule", + kind: "directory", + parentPath: "vendor", + }); + assert.isFalse( + result.entries.some((entry) => entry.path === "vendor/submodule" && entry.kind === "file"), + ); + }); + + it("includes files inside nested initialized git submodules", async () => { + const nestedOrigin = makeTempDir("t3code-workspace-nested-submodule-origin-"); + initGitRepo(nestedOrigin); + writeFile(nestedOrigin, "src/nested-file.ts", "export {};"); + commitAll(nestedOrigin, "Initial nested submodule"); + + const submoduleOrigin = makeTempDir("t3code-workspace-parent-submodule-origin-"); + initGitRepo(submoduleOrigin); + writeFile(submoduleOrigin, "src/submodule-file.ts", "export {};"); + commitAll(submoduleOrigin, "Initial parent submodule"); + runGit(submoduleOrigin, ["submodule", "add", "-q", nestedOrigin, "deps/nested-submodule"], { + config: ["protocol.file.allow=always"], + }); + runGit(submoduleOrigin, ["commit", "-am", "Add nested submodule"]); + + const cwd = makeTempDir("t3code-workspace-nested-submodule-root-"); + initGitRepo(cwd); + writeFile(cwd, "src/root.ts", "export {};"); + runGit(cwd, ["add", "src/root.ts"]); + runGit(cwd, ["commit", "-m", "Initial root"]); + runGit(cwd, ["submodule", "add", "-q", submoduleOrigin, "vendor/submodule"], { + config: ["protocol.file.allow=always"], + }); + runGit(cwd, ["commit", "-am", "Add submodule"]); + runGit(cwd, ["submodule", "update", "--init", "--recursive"], { + config: ["protocol.file.allow=always"], + }); + + const result = await searchWorkspaceEntries({ cwd, query: "", limit: 200 }); + const paths = result.entries.map((entry) => entry.path); + + assert.include(paths, "vendor/submodule"); + assert.include(paths, "vendor/submodule/deps"); + assert.include(paths, "vendor/submodule/deps/nested-submodule"); + assert.include(paths, "vendor/submodule/deps/nested-submodule/src"); + assert.include(paths, "vendor/submodule/deps/nested-submodule/src/nested-file.ts"); + assert.isFalse( + result.entries.some( + (entry) => entry.path === "vendor/submodule/deps/nested-submodule" && entry.kind === "file", + ), + ); + }); + + it("applies submodule-local ignore rules when listing submodule files", async () => { + const submoduleOrigin = makeTempDir("t3code-workspace-submodule-ignore-origin-"); + initGitRepo(submoduleOrigin); + writeFile(submoduleOrigin, ".gitignore", "ignored.ts\n"); + writeFile(submoduleOrigin, "src/submodule-file.ts", "export {};"); + commitAll(submoduleOrigin, "Initial submodule"); + + const cwd = makeTempDir("t3code-workspace-submodule-ignore-root-"); + initGitRepo(cwd); + writeFile(cwd, "src/root.ts", "export {};"); + runGit(cwd, ["add", "src/root.ts"]); + runGit(cwd, ["commit", "-m", "Initial root"]); + runGit(cwd, ["submodule", "add", "-q", submoduleOrigin, "vendor/submodule"], { + config: ["protocol.file.allow=always"], + }); + writeFile(cwd, "vendor/submodule/ignored.ts", "export {};"); + writeFile(cwd, "vendor/submodule/keep.ts", "export {};"); + + const result = await searchWorkspaceEntries({ cwd, query: "", limit: 200 }); + const paths = result.entries.map((entry) => entry.path); + + assert.include(paths, "vendor/submodule/keep.ts"); + assert.notInclude(paths, "vendor/submodule/ignored.ts"); + }); + it("deduplicates concurrent index builds for the same cwd", async () => { const cwd = makeTempDir("t3code-workspace-concurrent-build-"); writeFile(cwd, "src/components/Composer.tsx"); diff --git a/apps/server/src/workspaceEntries.ts b/apps/server/src/workspaceEntries.ts index 684b005e8..1c57027a7 100644 --- a/apps/server/src/workspaceEntries.ts +++ b/apps/server/src/workspaceEntries.ts @@ -13,6 +13,7 @@ const WORKSPACE_CACHE_TTL_MS = 15_000; const WORKSPACE_CACHE_MAX_KEYS = 4; const WORKSPACE_INDEX_MAX_ENTRIES = 25_000; const WORKSPACE_SCAN_READDIR_CONCURRENCY = 32; +const GIT_SUBMODULE_SCAN_CONCURRENCY = 8; const GIT_CHECK_IGNORE_MAX_STDIN_BYTES = 256 * 1024; const IGNORED_DIRECTORY_NAMES = new Set([ ".git", @@ -215,6 +216,18 @@ function splitNullSeparatedPaths(input: string, truncated: boolean): string[] { return parts.filter((value) => value.length > 0); } +function splitLineSeparatedPaths(input: string, truncated: boolean): string[] { + const parts = input.split(/\r?\n/); + if (parts.length === 0) return []; + + // If output was truncated, the final token can be partial. + if (truncated && parts[parts.length - 1]?.length) { + parts.pop(); + } + + return parts.filter((value) => value.length > 0); +} + function directoryAncestorsOf(relativePath: string): string[] { const segments = relativePath.split("/").filter((segment) => segment.length > 0); if (segments.length <= 1) return []; @@ -335,7 +348,42 @@ async function filterGitIgnoredPaths(cwd: string, relativePaths: string[]): Prom return relativePaths.filter((relativePath) => !ignoredPaths.has(relativePath)); } -async function buildWorkspaceIndexFromGit(cwd: string): Promise { +interface GitWorkspaceFileListing { + filePaths: string[]; + truncated: boolean; +} + +async function listInitializedGitSubmodulePaths(cwd: string): Promise { + const resolvedCwd = await fs.realpath(cwd).catch(() => cwd); + const listedSubmodules = await runProcess("git", ["submodule", "foreach", "--quiet", "pwd"], { + cwd, + allowNonZeroExit: true, + timeoutMs: 20_000, + maxBufferBytes: 4 * 1024 * 1024, + outputMode: "truncate", + }).catch(() => null); + + if (!listedSubmodules || listedSubmodules.code !== 0) { + return []; + } + + return splitLineSeparatedPaths(listedSubmodules.stdout, Boolean(listedSubmodules.stdoutTruncated)) + .map((absolutePath) => toPosixPath(path.relative(resolvedCwd, absolutePath))) + .filter( + (relativePath) => + relativePath.length > 0 && + !relativePath.startsWith("../") && + relativePath !== ".." && + !path.isAbsolute(relativePath) && + !isPathInIgnoredDirectory(relativePath), + ); +} + +function prefixWorkspaceFilePath(prefix: string, relativePath: string): string { + return `${prefix}/${relativePath}`; +} + +async function listGitWorkspaceFilePaths(cwd: string): Promise { if (!(await isInsideGitWorkTree(cwd))) { return null; } @@ -355,16 +403,61 @@ async function buildWorkspaceIndexFromGit(cwd: string): Promise toPosixPath(entry)) - .filter((entry) => entry.length > 0 && !isPathInIgnoredDirectory(entry)); + .filter( + (entry) => + entry.length > 0 && !isPathInIgnoredDirectory(entry) && !submodulePathSet.has(entry), + ); const filePaths = await filterGitIgnoredPaths(cwd, listedPaths); + let truncated = Boolean(listedFiles.stdoutTruncated); + + const submoduleListings = await mapWithConcurrency( + submodulePaths, + GIT_SUBMODULE_SCAN_CONCURRENCY, + async (submodulePath) => { + const submoduleCwd = path.join(cwd, submodulePath); + const submoduleListing = await listGitWorkspaceFilePaths(submoduleCwd); + if (!submoduleListing) { + return null; + } + + return { + filePaths: submoduleListing.filePaths.map((relativePath) => + prefixWorkspaceFilePath(submodulePath, relativePath), + ), + truncated: submoduleListing.truncated, + }; + }, + ); + + for (const submoduleListing of submoduleListings) { + if (!submoduleListing) { + continue; + } + filePaths.push(...submoduleListing.filePaths); + truncated ||= submoduleListing.truncated; + } + + return { + filePaths, + truncated, + }; +} + +async function buildWorkspaceIndexFromGit(cwd: string): Promise { + const gitFileListing = await listGitWorkspaceFilePaths(cwd); + if (!gitFileListing) { + return null; + } const directorySet = new Set(); - for (const filePath of filePaths) { + for (const filePath of gitFileListing.filePaths) { for (const directoryPath of directoryAncestorsOf(filePath)) { if (!isPathInIgnoredDirectory(directoryPath)) { directorySet.add(directoryPath); @@ -382,7 +475,7 @@ async function buildWorkspaceIndexFromGit(cwd: string): Promise left.localeCompare(right)) .map( (filePath): ProjectEntry => ({ @@ -397,7 +490,7 @@ async function buildWorkspaceIndexFromGit(cwd: string): Promise WORKSPACE_INDEX_MAX_ENTRIES, + truncated: gitFileListing.truncated || entries.length > WORKSPACE_INDEX_MAX_ENTRIES, }; } diff --git a/apps/web/public/mockServiceWorker.js b/apps/web/public/mockServiceWorker.js index 85e901012..daa58d0f1 100644 --- a/apps/web/public/mockServiceWorker.js +++ b/apps/web/public/mockServiceWorker.js @@ -7,7 +7,7 @@ * - Please do NOT modify this file. */ -const PACKAGE_VERSION = '2.12.9' +const PACKAGE_VERSION = '2.12.10' const INTEGRITY_CHECKSUM = '4db4a41e972cec1b64cc569c66952d82' const IS_MOCKED_RESPONSE = Symbol('isMockedResponse') const activeClientIds = new Set()