Add workspace search for initialized git submodules

apps/server/src/workspaceEntries.test.ts

@@ -22,13 +22,25 @@ function writeFile(cwd: string, relativePath: string, contents = ""): void {
   fs.writeFileSync(absolutePath, contents, "utf8");
 }
 
-function runGit(cwd: string, args: string[]): void {
-  const result = spawnSync("git", args, { cwd, encoding: "utf8" });
+function runGit(cwd: string, args: string[], options?: { config?: string[] }): void {
+  const gitArgs = [...(options?.config ?? []).flatMap((entry) => ["-c", entry]), ...args];
+  const result = spawnSync("git", gitArgs, { cwd, encoding: "utf8" });
   if (result.status !== 0) {
-    throw new Error(result.stderr || `git ${args.join(" ")} failed`);
+    throw new Error(result.stderr || `git ${gitArgs.join(" ")} failed`);
   }
 }
 
+function initGitRepo(cwd: string): void {
+  runGit(cwd, ["init"]);
+  runGit(cwd, ["config", "user.email", "test@example.com"]);
+  runGit(cwd, ["config", "user.name", "Test User"]);
+}
+
+function commitAll(cwd: string, message: string): void {
+  runGit(cwd, ["add", "."]);
+  runGit(cwd, ["commit", "-m", message]);
+}
+
 describe("searchWorkspaceEntries", () => {
   afterEach(() => {
     vi.restoreAllMocks();
@@ -144,6 +156,111 @@ describe("searchWorkspaceEntries", () => {
     assert.isFalse(paths.some((entryPath) => entryPath.startsWith(".convex/")));
   });
 
+  it("includes files inside initialized git submodules", async () => {
+    const submoduleOrigin = makeTempDir("t3code-workspace-submodule-origin-");
+    initGitRepo(submoduleOrigin);
+    writeFile(submoduleOrigin, "src/submodule-file.ts", "export {};");
+    writeFile(submoduleOrigin, "README.md", "# submodule\n");
+    commitAll(submoduleOrigin, "Initial submodule");
+
+    const cwd = makeTempDir("t3code-workspace-submodule-root-");
+    initGitRepo(cwd);
+    writeFile(cwd, "src/root.ts", "export {};");
+    runGit(cwd, ["add", "src/root.ts"]);
+    runGit(cwd, ["commit", "-m", "Initial root"]);
+    runGit(cwd, ["submodule", "add", "-q", submoduleOrigin, "vendor/submodule"], {
+      config: ["protocol.file.allow=always"],
+    });
+    writeFile(cwd, "vendor/submodule/untracked.ts", "export {};");
+
+    const result = await searchWorkspaceEntries({ cwd, query: "", limit: 100 });
+    const paths = result.entries.map((entry) => entry.path);
+    const submoduleRootEntry = result.entries.find((entry) => entry.path === "vendor/submodule");
+
+    assert.include(paths, "vendor");
+    assert.include(paths, "vendor/submodule");
+    assert.include(paths, "vendor/submodule/README.md");
+    assert.include(paths, "vendor/submodule/src");
+    assert.include(paths, "vendor/submodule/src/submodule-file.ts");
+    assert.include(paths, "vendor/submodule/untracked.ts");
+    assert.deepInclude(submoduleRootEntry, {
+      path: "vendor/submodule",
+      kind: "directory",
+      parentPath: "vendor",
+    });
+    assert.isFalse(
+      result.entries.some((entry) => entry.path === "vendor/submodule" && entry.kind === "file"),
+    );
+  });
+
+  it("includes files inside nested initialized git submodules", async () => {
+    const nestedOrigin = makeTempDir("t3code-workspace-nested-submodule-origin-");
+    initGitRepo(nestedOrigin);
+    writeFile(nestedOrigin, "src/nested-file.ts", "export {};");
+    commitAll(nestedOrigin, "Initial nested submodule");
+
+    const submoduleOrigin = makeTempDir("t3code-workspace-parent-submodule-origin-");
+    initGitRepo(submoduleOrigin);
+    writeFile(submoduleOrigin, "src/submodule-file.ts", "export {};");
+    commitAll(submoduleOrigin, "Initial parent submodule");
+    runGit(submoduleOrigin, ["submodule", "add", "-q", nestedOrigin, "deps/nested-submodule"], {
+      config: ["protocol.file.allow=always"],
+    });
+    runGit(submoduleOrigin, ["commit", "-am", "Add nested submodule"]);
+
+    const cwd = makeTempDir("t3code-workspace-nested-submodule-root-");
+    initGitRepo(cwd);
+    writeFile(cwd, "src/root.ts", "export {};");
+    runGit(cwd, ["add", "src/root.ts"]);
+    runGit(cwd, ["commit", "-m", "Initial root"]);
+    runGit(cwd, ["submodule", "add", "-q", submoduleOrigin, "vendor/submodule"], {
+      config: ["protocol.file.allow=always"],
+    });
+    runGit(cwd, ["commit", "-am", "Add submodule"]);
+    runGit(cwd, ["submodule", "update", "--init", "--recursive"], {
+      config: ["protocol.file.allow=always"],
+    });
+
+    const result = await searchWorkspaceEntries({ cwd, query: "", limit: 200 });
+    const paths = result.entries.map((entry) => entry.path);
+
+    assert.include(paths, "vendor/submodule");
+    assert.include(paths, "vendor/submodule/deps");
+    assert.include(paths, "vendor/submodule/deps/nested-submodule");
+    assert.include(paths, "vendor/submodule/deps/nested-submodule/src");
+    assert.include(paths, "vendor/submodule/deps/nested-submodule/src/nested-file.ts");
+    assert.isFalse(
+      result.entries.some(
+        (entry) => entry.path === "vendor/submodule/deps/nested-submodule" && entry.kind === "file",
+      ),
+    );
+  });
+
+  it("applies submodule-local ignore rules when listing submodule files", async () => {
+    const submoduleOrigin = makeTempDir("t3code-workspace-submodule-ignore-origin-");
+    initGitRepo(submoduleOrigin);
+    writeFile(submoduleOrigin, ".gitignore", "ignored.ts\n");
+    writeFile(submoduleOrigin, "src/submodule-file.ts", "export {};");
+    commitAll(submoduleOrigin, "Initial submodule");
+
+    const cwd = makeTempDir("t3code-workspace-submodule-ignore-root-");
+    initGitRepo(cwd);
+    writeFile(cwd, "src/root.ts", "export {};");
+    runGit(cwd, ["add", "src/root.ts"]);
+    runGit(cwd, ["commit", "-m", "Initial root"]);
+    runGit(cwd, ["submodule", "add", "-q", submoduleOrigin, "vendor/submodule"], {
+      config: ["protocol.file.allow=always"],
+    });
+    writeFile(cwd, "vendor/submodule/ignored.ts", "export {};");
+    writeFile(cwd, "vendor/submodule/keep.ts", "export {};");
+
+    const result = await searchWorkspaceEntries({ cwd, query: "", limit: 200 });
+    const paths = result.entries.map((entry) => entry.path);
+
+    assert.include(paths, "vendor/submodule/keep.ts");
+    assert.notInclude(paths, "vendor/submodule/ignored.ts");
+  });
+
   it("deduplicates concurrent index builds for the same cwd", async () => {
     const cwd = makeTempDir("t3code-workspace-concurrent-build-");
     writeFile(cwd, "src/components/Composer.tsx");

apps/server/src/workspaceEntries.ts

@@ -13,6 +13,7 @@ const WORKSPACE_CACHE_TTL_MS = 15_000;
 const WORKSPACE_CACHE_MAX_KEYS = 4;
 const WORKSPACE_INDEX_MAX_ENTRIES = 25_000;
 const WORKSPACE_SCAN_READDIR_CONCURRENCY = 32;
+const GIT_SUBMODULE_SCAN_CONCURRENCY = 8;
 const GIT_CHECK_IGNORE_MAX_STDIN_BYTES = 256 * 1024;
 const IGNORED_DIRECTORY_NAMES = new Set([
   ".git",
@@ -215,6 +216,18 @@ function splitNullSeparatedPaths(input: string, truncated: boolean): string[] {
   return parts.filter((value) => value.length > 0);
 }
 
+function splitLineSeparatedPaths(input: string, truncated: boolean): string[] {
+  const parts = input.split(/\r?\n/);
+  if (parts.length === 0) return [];
+
+  // If output was truncated, the final token can be partial.
+  if (truncated && parts[parts.length - 1]?.length) {
+    parts.pop();
+  }
+
+  return parts.filter((value) => value.length > 0);
+}
+
 function directoryAncestorsOf(relativePath: string): string[] {
   const segments = relativePath.split("/").filter((segment) => segment.length > 0);
   if (segments.length <= 1) return [];
@@ -335,7 +348,42 @@ async function filterGitIgnoredPaths(cwd: string, relativePaths: string[]): Prom
   return relativePaths.filter((relativePath) => !ignoredPaths.has(relativePath));
 }
 
-async function buildWorkspaceIndexFromGit(cwd: string): Promise<WorkspaceIndex | null> {
+interface GitWorkspaceFileListing {
+  filePaths: string[];
+  truncated: boolean;
+}
+
+async function listInitializedGitSubmodulePaths(cwd: string): Promise<string[]> {
+  const resolvedCwd = await fs.realpath(cwd).catch(() => cwd);
+  const listedSubmodules = await runProcess("git", ["submodule", "foreach", "--quiet", "pwd"], {
+    cwd,
+    allowNonZeroExit: true,
+    timeoutMs: 20_000,
+    maxBufferBytes: 4 * 1024 * 1024,
+    outputMode: "truncate",
+  }).catch(() => null);
+
+  if (!listedSubmodules || listedSubmodules.code !== 0) {
+    return [];
+  }
+
+  return splitLineSeparatedPaths(listedSubmodules.stdout, Boolean(listedSubmodules.stdoutTruncated))
+    .map((absolutePath) => toPosixPath(path.relative(resolvedCwd, absolutePath)))
+    .filter(
+      (relativePath) =>
+        relativePath.length > 0 &&
+        !relativePath.startsWith("../") &&
+        relativePath !== ".." &&
+        !path.isAbsolute(relativePath) &&
+        !isPathInIgnoredDirectory(relativePath),
+    );
+}
+
+function prefixWorkspaceFilePath(prefix: string, relativePath: string): string {
+  return `${prefix}/${relativePath}`;
+}
+
+async function listGitWorkspaceFilePaths(cwd: string): Promise<GitWorkspaceFileListing | null> {
   if (!(await isInsideGitWorkTree(cwd))) {
     return null;
   }
@@ -355,16 +403,61 @@ async function buildWorkspaceIndexFromGit(cwd: string): Promise<WorkspaceIndex |
     return null;
   }
 
+  const submodulePaths = await listInitializedGitSubmodulePaths(cwd);
+  const submodulePathSet = new Set(submodulePaths);
   const listedPaths = splitNullSeparatedPaths(
     listedFiles.stdout,
     Boolean(listedFiles.stdoutTruncated),
   )
     .map((entry) => toPosixPath(entry))
-    .filter((entry) => entry.length > 0 && !isPathInIgnoredDirectory(entry));
+    .filter(
+      (entry) =>
+        entry.length > 0 && !isPathInIgnoredDirectory(entry) && !submodulePathSet.has(entry),
+    );
   const filePaths = await filterGitIgnoredPaths(cwd, listedPaths);
+  let truncated = Boolean(listedFiles.stdoutTruncated);
+
+  const submoduleListings = await mapWithConcurrency(
+    submodulePaths,
+    GIT_SUBMODULE_SCAN_CONCURRENCY,
+    async (submodulePath) => {
+      const submoduleCwd = path.join(cwd, submodulePath);
+      const submoduleListing = await listGitWorkspaceFilePaths(submoduleCwd);
+      if (!submoduleListing) {
+        return null;
+      }
+
+      return {
+        filePaths: submoduleListing.filePaths.map((relativePath) =>
+          prefixWorkspaceFilePath(submodulePath, relativePath),
+        ),
+        truncated: submoduleListing.truncated,
+      };
+    },
+  );
+
+  for (const submoduleListing of submoduleListings) {
+    if (!submoduleListing) {
+      continue;
+    }
+    filePaths.push(...submoduleListing.filePaths);
+    truncated ||= submoduleListing.truncated;
+  }
+
+  return {
+    filePaths,
+    truncated,
+  };
+}
+
+async function buildWorkspaceIndexFromGit(cwd: string): Promise<WorkspaceIndex | null> {
+  const gitFileListing = await listGitWorkspaceFilePaths(cwd);
+  if (!gitFileListing) {
+    return null;
+  }
 
   const directorySet = new Set<string>();
-  for (const filePath of filePaths) {
+  for (const filePath of gitFileListing.filePaths) {
     for (const directoryPath of directoryAncestorsOf(filePath)) {
       if (!isPathInIgnoredDirectory(directoryPath)) {
         directorySet.add(directoryPath);
@@ -382,7 +475,7 @@ async function buildWorkspaceIndexFromGit(cwd: string): Promise<WorkspaceIndex |
       }),
     )
     .map(toSearchableWorkspaceEntry);
-  const fileEntries = [...new Set(filePaths)]
+  const fileEntries = [...new Set(gitFileListing.filePaths)]
     .toSorted((left, right) => left.localeCompare(right))
     .map(
       (filePath): ProjectEntry => ({
@@ -397,7 +490,7 @@ async function buildWorkspaceIndexFromGit(cwd: string): Promise<WorkspaceIndex |
   return {
     scannedAt: Date.now(),
     entries: entries.slice(0, WORKSPACE_INDEX_MAX_ENTRIES),
-    truncated: Boolean(listedFiles.stdoutTruncated) || entries.length > WORKSPACE_INDEX_MAX_ENTRIES,
+    truncated: gitFileListing.truncated || entries.length > WORKSPACE_INDEX_MAX_ENTRIES,
   };
 }
 

apps/web/public/mockServiceWorker.js

@@ -7,7 +7,7 @@
  * - Please do NOT modify this file.
  */
 
-const PACKAGE_VERSION = '2.12.9'
+const PACKAGE_VERSION = '2.12.10'
 const INTEGRITY_CHECKSUM = '4db4a41e972cec1b64cc569c66952d82'
 const IS_MOCKED_RESPONSE = Symbol('isMockedResponse')
 const activeClientIds = new Set()