add flag to initialize submodules --git-recurse-submodules (#342)

* add flag to initialize submodules --git-recurse-submodules

- When set, it will recursively fetch all submodules.
- If any submodule uses a SSH URL (i.e. `[email protected]:ORG/REPO.git`), it will
  be replaced with a HTTPS one (i.e. `https://github.com/ORG/REPO.git`) so that
  token authentication can be used to clone private repositories.
- Any public Git repository should work (i.e. those not hosted on Github).

* fix recursion decrement

* lint

* use parameterized Github URL to rewrite SSH URLs

---------

Co-authored-by: Vincent Behar <[email protected]>

docs/current-version/content/repos/commit.md

@@ -12,6 +12,8 @@ Flags related to the `git clone` operation:
 
 - `--git-clone-dir` (string): optional path to a directory used to clone the git repositories. Default to a newly created directory in the system temporary directory (defined by the `TMPDIR` environment variable, defaulting to `/tmp`). Note that by default all files created inside this directory will be deleted at the end of the process, unless the `--keep-files` flag is set.
 
+- `--git-recurse-submodules` (bool): recursively initialize all submodules. Disabled by default.
+
 ## Git index/stage
 
 By default, all files changed by the [updaters](#updaters) will be added to the git "index" - so that they can be added to the git commit. This is configurable through the following flags:

main.go

@@ -84,6 +84,7 @@ func init() {
 	pflag.StringVar(&options.UpdateOptions.Git.CloneDir, "git-clone-dir", temporaryDirectory(), "Directory used to clone the repositories.")
 	pflag.StringArrayVar(&options.UpdateOptions.Git.StagePatterns, "git-stage-pattern", nil, "List of path patterns that will be added to the git index and committed.")
 	pflag.BoolVar(&options.UpdateOptions.Git.StageAllChanged, "git-stage-all-changed", true, "Commit all files changed.")
+	pflag.BoolVar(&options.UpdateOptions.Git.RecurseSubmodules, "git-recurse-submodules", false, "Recursively initialize all submodules.")
 	pflag.StringVar(&options.UpdateOptions.Git.AuthorName, "git-author-name", firstNonEmpyValue(os.Getenv("GIT_AUTHOR_NAME"), git.ConfigValue("user.name")), `Name of the author of the git commit. Default to the GIT_AUTHOR_NAME env var, or the "user.name" git config value.`)
 	pflag.StringVar(&options.UpdateOptions.Git.AuthorEmail, "git-author-email", firstNonEmpyValue(os.Getenv("GIT_AUTHOR_EMAIL"), git.ConfigValue("user.email")), `Email of the author of the git commit. Default to the GIT_AUTHOR_EMAIL env var, or the "user.email" git config value.`)
 	pflag.StringVar(&options.UpdateOptions.Git.CommitterName, "git-committer-name", firstNonEmpyValue(os.Getenv("GIT_COMMITTER_NAME"), git.ConfigValue("user.name")), `Name of the committer. Default to the GIT_COMMITTER_NAME env var, or the "user.name" git config value.`)

repository/git.go

@@ -16,15 +16,16 @@ import (
 	"github.com/go-git/go-git/v5/config"
 	"github.com/go-git/go-git/v5/plumbing"
 	"github.com/go-git/go-git/v5/plumbing/object"
+	"github.com/go-git/go-git/v5/plumbing/transport"
 	"github.com/go-git/go-git/v5/plumbing/transport/http"
 	"github.com/go-git/go-git/v5/utils/merkletrie"
 	"github.com/google/go-github/v57/github"
 	"github.com/shurcooL/githubv4"
 	"github.com/sirupsen/logrus"
 )
 
-func cloneGitRepository(ctx context.Context, repo Repository, localPath string, options GitHubOptions) (*git.Repository, error) {
-	gitURL, err := url.JoinPath(options.URL, repo.GitFullName())
+func cloneGitRepository(ctx context.Context, repo Repository, localPath string, options UpdateOptions) (*git.Repository, error) {
+	gitURL, err := url.JoinPath(options.GitHub.URL, repo.GitFullName())
 	if err != nil {
 		// likely the Url passed is malformed
 		return nil, fmt.Errorf("invalid github url format: %w", err)
@@ -41,7 +42,7 @@ func cloneGitRepository(ctx context.Context, repo Repository, localPath string,
 		"local-path":    localPath,
 	}).Trace("Cloning git repository")
 
-	_, token, err := githubClient(ctx, options)
+	_, token, err := githubClient(ctx, options.GitHub)
 	if err != nil {
 		return nil, fmt.Errorf("failed to create github client: %w", err)
 	}
@@ -58,6 +59,21 @@ func cloneGitRepository(ctx context.Context, repo Repository, localPath string,
 		return nil, fmt.Errorf("failed to clone git repository from %s to %s: %w", gitURL, localPath, err)
 	}
 
+	recurseSubmodules := git.NoRecurseSubmodules
+	if options.Git.RecurseSubmodules {
+		recurseSubmodules = git.DefaultSubmoduleRecursionDepth
+	}
+
+	githubURL, err := url.Parse(options.GitHub.URL)
+	if err != nil {
+		return nil, fmt.Errorf("failed to parse Github URL: %w", err)
+	}
+
+	err = initSubmodules(ctx, gitRepo, token, recurseSubmodules, githubURL)
+	if err != nil {
+		return nil, fmt.Errorf("failed to initialize submodules for git repository %s: %w", gitURL, err)
+	}
+
 	logrus.WithFields(logrus.Fields{
 		"git-url":       gitURL,
 		"git-reference": referenceName.String(),
@@ -67,6 +83,67 @@ func cloneGitRepository(ctx context.Context, repo Repository, localPath string,
 	return gitRepo, nil
 }
 
+func initSubmodules(ctx context.Context, repo *git.Repository, token string, recurseSubmodules git.SubmoduleRescursivity, githubURL *url.URL) error {
+	if recurseSubmodules == git.NoRecurseSubmodules {
+		return nil
+	}
+	recurseSubmodules--
+
+	wt, err := repo.Worktree()
+	if err != nil {
+		return fmt.Errorf("failed to get worktree: %w", err)
+	}
+
+	subModules, err := wt.Submodules()
+	if err != nil {
+		return fmt.Errorf("failed to get submodules: %w", err)
+	}
+
+	for _, s := range subModules {
+		// Hack: rewrite Github hosted submodule SSH URLs to use HTTPS because token auth only works with that
+		// go-git does not expose a way of doing insteadOf type rewrites for submodules, so this will have to do for now
+		s.Config().URL = strings.Replace(s.Config().URL, fmt.Sprintf("git@%s:", githubURL.Hostname()), githubURL.String(), 1)
+
+		// Only use basic auth for Github. This lets us use any public Git repo not hosted on Github.
+		var auth transport.AuthMethod
+		if strings.HasPrefix(s.Config().URL, githubURL.String()) {
+			auth = &http.BasicAuth{
+				Username: "x-access-token",
+				Password: token,
+			}
+		} else {
+			auth = nil
+		}
+
+		logrus.WithFields(logrus.Fields{
+			"submodule-name":   s.Config().Name,
+			"submodule-url":    s.Config().URL,
+			"submodule-branch": s.Config().Branch,
+			"submodule-path":   s.Config().Path,
+		}).Trace("Initializing submodule")
+
+		err = s.UpdateContext(ctx, &git.SubmoduleUpdateOptions{
+			Init: true,
+			Auth: auth,
+		})
+		if err != nil {
+			return fmt.Errorf("failed to initialize submodule: %w", err)
+		}
+
+		sRepo, err := s.Repository()
+		if err != nil {
+			return fmt.Errorf("failed to get submodule repo: %w", err)
+		}
+
+		err = initSubmodules(ctx, sRepo, token, recurseSubmodules, githubURL)
+		if err != nil {
+			return err
+		}
+	}
+
+	return nil
+}
+
 type createBranchOptions struct {
 	GitHubOpts GitHubOptions
 	Repository Repository

repository/options.go

@@ -44,6 +44,7 @@ type GitOptions struct {
 	BranchPrefix         string
 	SigningKeyPath       string
 	SigningKeyPassphrase string
+	RecurseSubmodules    bool
 }
 
 // GitHubOptions holds all the options required to perform github operations: auth, PRs, ...

repository/strategy.go

@@ -24,7 +24,7 @@ type Strategy struct {
 // - a boolean indicating whether changes have been made to the repository
 // - a pull request if one has been created (or updated)
 func (s *Strategy) Run(ctx context.Context) (bool, *github.PullRequest, error) {
-	gitRepo, err := cloneGitRepository(ctx, s.Repository, s.RepoPath, s.Options.GitHub)
+	gitRepo, err := cloneGitRepository(ctx, s.Repository, s.RepoPath, s.Options)
 	if err != nil {
 		return false, nil, fmt.Errorf("failed to clone repository %s: %w", s.Repository.FullName(), err)
 	}