feat(git): cloneSubmodulesFilter (#33115)

docs/usage/configuration-options.md

@@ -456,11 +456,19 @@ For `sbt` note that Renovate will update the version string only for packages th
 
 ## cloneSubmodules
 
-Enabling this option will mean that any detected Git submodules will be cloned at time of repository clone.
+Enabling this option will mean that detected Git submodules will be cloned at time of repository clone.
+By default all will be cloned, but this can be customized by configuring `cloneSubmodulesFilter` too.
 Submodules are always cloned recursively.
 
 Important: private submodules aren't supported by Renovate, unless the underlying `ssh` layer already has the correct permissions.
 
+## cloneSubmodulesFilter
+
+Use this option together with `cloneSubmodules` if you wish to clone only a subset of submodules.
+
+This config option supports regex and glob filters, including negative matches.
+For more details on this syntax see Renovate's [string pattern matching documentation](./string-pattern-matching.md).
+
 ## commitBody
 
 Configure this if you wish Renovate to add a commit body, otherwise Renovate uses a regular single-line commit.

lib/config/options/index.ts

@@ -2856,6 +2856,14 @@ const options: RenovateOptions[] = [
     type: 'boolean',
     default: false,
   },
+  {
+    name: 'cloneSubmodulesFilter',
+    description:
+      'List of submodules names or patterns to clone when cloneSubmodules=true.',
+    type: 'array',
+    subType: 'string',
+    default: ['*'],
+  },
   {
     name: 'ignorePrAuthor',
     description:

lib/config/types.ts

@@ -239,6 +239,7 @@ export interface RenovateConfig
   baseBranch?: string;
   defaultBranch?: string;
   branchList?: string[];
+  cloneSubmodulesFilter?: string[];
   description?: string | string[];
   force?: RenovateConfig;
   errors?: ValidationMessage[];

lib/modules/platform/azure/index.ts

@@ -193,6 +193,7 @@ export async function getJsonFile(
 export async function initRepo({
   repository,
   cloneSubmodules,
+  cloneSubmodulesFilter,
 }: RepoParams): Promise<RepoResult> {
   logger.debug(`initRepo("${repository}")`);
   config = { repository } as Config;
@@ -240,6 +241,7 @@ export async function initRepo({
     url,
     extraCloneOpts: getStorageExtraCloneOpts(opts),
     cloneSubmodules,
+    cloneSubmodulesFilter,
   });
   const repoConfig: RepoResult = {
     defaultBranch,

lib/modules/platform/bitbucket-server/index.ts

@@ -223,6 +223,7 @@ export async function getJsonFile(
 export async function initRepo({
   repository,
   cloneSubmodules,
+  cloneSubmodulesFilter,
   ignorePrAuthor,
   gitUrl,
 }: RepoParams): Promise<RepoResult> {
@@ -274,6 +275,7 @@ export async function initRepo({
       url,
       extraCloneOpts: getExtraCloneOpts(opts),
       cloneSubmodules,
+      cloneSubmodulesFilter,
       fullClone: semver.lte(defaults.version, '8.0.0'),
     });
 

lib/modules/platform/bitbucket/index.ts

@@ -185,6 +185,7 @@ export async function getJsonFile(
 export async function initRepo({
   repository,
   cloneSubmodules,
+  cloneSubmodulesFilter,
   ignorePrAuthor,
   bbUseDevelopmentBranch,
 }: RepoParams): Promise<RepoResult> {
@@ -262,6 +263,7 @@ export async function initRepo({
     ...config,
     url,
     cloneSubmodules,
+    cloneSubmodulesFilter,
   });
   const repoConfig: RepoResult = {
     defaultBranch: mainBranch,

lib/modules/platform/gitea/index.ts

@@ -70,6 +70,7 @@ interface GiteaRepoConfig {
   labelList: Promise<Label[]> | null;
   defaultBranch: string;
   cloneSubmodules: boolean;
+  cloneSubmodulesFilter: string[] | undefined;
   hasIssuesEnabled: boolean;
 }
 
@@ -255,13 +256,15 @@ const platform: Platform = {
   async initRepo({
     repository,
     cloneSubmodules,
+    cloneSubmodulesFilter,
     gitUrl,
   }: RepoParams): Promise<RepoResult> {
     let repo: Repo;
 
     config = {} as any;
     config.repository = repository;
     config.cloneSubmodules = !!cloneSubmodules;
+    config.cloneSubmodulesFilter = cloneSubmodulesFilter;
 
     // Try to fetch information about repository
     try {

lib/modules/platform/github/index.ts

@@ -447,13 +447,15 @@ export async function initRepo({
   forkToken,
   renovateUsername,
   cloneSubmodules,
+  cloneSubmodulesFilter,
   ignorePrAuthor,
 }: RepoParams): Promise<RepoResult> {
   logger.debug(`initRepo("${repository}")`);
   // config is used by the platform api itself, not necessary for the app layer to know
   config = {
     repository,
     cloneSubmodules,
+    cloneSubmodulesFilter,
     ignorePrAuthor,
   } as any;
   // istanbul ignore if

lib/modules/platform/gitlab/__snapshots__/index.spec.ts.snap

@@ -135,6 +135,7 @@ exports[`modules/platform/gitlab/index initRepo should fall back respecting when
   [
     {
       "cloneSubmodules": undefined,
+      "cloneSubmodulesFilter": undefined,
       "defaultBranch": "master",
       "ignorePrAuthor": undefined,
       "mergeMethod": "merge",
@@ -150,6 +151,7 @@ exports[`modules/platform/gitlab/index initRepo should use ssh_url_to_repo if gi
   [
     {
       "cloneSubmodules": undefined,
+      "cloneSubmodulesFilter": undefined,
       "defaultBranch": "master",
       "ignorePrAuthor": undefined,
       "mergeMethod": "merge",

lib/modules/platform/gitlab/index.ts

@@ -81,6 +81,7 @@ let config: {
   mergeMethod: MergeMethod;
   defaultBranch: string;
   cloneSubmodules: boolean | undefined;
+  cloneSubmodulesFilter: string[] | undefined;
   ignorePrAuthor: boolean | undefined;
   squash: boolean;
 } = {} as any;
@@ -299,6 +300,7 @@ function getRepoUrl(
 export async function initRepo({
   repository,
   cloneSubmodules,
+  cloneSubmodulesFilter,
   ignorePrAuthor,
   gitUrl,
   endpoint,
@@ -307,6 +309,7 @@ export async function initRepo({
   config = {} as any;
   config.repository = urlEscape(repository);
   config.cloneSubmodules = cloneSubmodules;
+  config.cloneSubmodulesFilter = cloneSubmodulesFilter;
   config.ignorePrAuthor = ignorePrAuthor;
 
   let res: HttpResponse<RepoResponse>;

lib/modules/platform/types.ts

@@ -47,6 +47,7 @@ export interface RepoParams {
   forkProcessing?: 'enabled' | 'disabled';
   renovateUsername?: string;
   cloneSubmodules?: boolean;
+  cloneSubmodulesFilter?: string[];
   ignorePrAuthor?: boolean;
   bbUseDevelopmentBranch?: boolean;
   includeMirrors?: boolean;

lib/util/git/index.spec.ts

@@ -271,6 +271,7 @@ describe('util/git/index', () => {
       await repo.commit('Add submodules');
       await git.initRepo({
         cloneSubmodules: true,
+        cloneSubmodulesFilter: ['file'],
         url: base.path,
       });
       await git.syncGit();

lib/util/git/index.ts

@@ -24,6 +24,7 @@ import type { GitProtocol } from '../../types/git';
 import { incLimitedValue } from '../../workers/global/limits';
 import { getCache } from '../cache/repository';
 import { newlineRegex, regEx } from '../regex';
+import { matchRegexOrGlobList } from '../string-match';
 import { parseGitAuthor } from './author';
 import {
   getCachedBehindBaseResult,
@@ -344,14 +345,24 @@ export async function getSubmodules(): Promise<string[]> {
   }
 }
 
-export async function cloneSubmodules(shouldClone: boolean): Promise<void> {
+export async function cloneSubmodules(
+  shouldClone: boolean,
+  cloneSubmodulesFilter: string[] | undefined,
+): Promise<void> {
   if (!shouldClone || submodulesInitizialized) {
     return;
   }
   submodulesInitizialized = true;
   await syncGit();
   const submodules = await getSubmodules();
   for (const submodule of submodules) {
+    if (!matchRegexOrGlobList(submodule, cloneSubmodulesFilter ?? ['*'])) {
+      logger.debug(
+        { cloneSubmodulesFilter },
+        `Skipping submodule ${submodule}`,
+      );
+      continue;
+    }
     try {
       logger.debug(`Cloning git submodule at ${submodule}`);
       await gitRetry(() =>
@@ -458,7 +469,7 @@ export async function syncGit(): Promise<void> {
     throw err;
   }
   // This will only happen now if set in global config
-  await cloneSubmodules(!!config.cloneSubmodules);
+  await cloneSubmodules(!!config.cloneSubmodules, config.cloneSubmodulesFilter);
   try {
     const latestCommit = (await git.log({ n: 1 })).latest;
     logger.debug({ latestCommit }, 'latest repository commit');

lib/util/git/types.ts

@@ -21,6 +21,7 @@ export interface StorageConfig {
   url: string;
   extraCloneOpts?: GitOptions;
   cloneSubmodules?: boolean;
+  cloneSubmodulesFilter?: string[];
   fullClone?: boolean;
 }
 

lib/workers/repository/init/index.ts

@@ -72,6 +72,6 @@ export async function initRepo(
       'Full resolved config and hostRules including presets',
     );
   }
-  await cloneSubmodules(!!config.cloneSubmodules);
+  await cloneSubmodules(!!config.cloneSubmodules, config.cloneSubmodulesFilter);
   return config;
 }