From b34ea9e3356d28e99e1e7c32d22d5ed49699b526 Mon Sep 17 00:00:00 2001 From: Anil Keshav Date: Thu, 28 Sep 2023 11:31:51 +0200 Subject: [PATCH] fix (gitOpsUpdateDeployment) add CA bundle options to plain clone and commit to trust enterprise github instances (#4602) * downloading ca cert bundle when added as config * adding logging statements * allowing bats test to handle ca cert * adding info message * hard coding file names * including correct http client util bundle * removing logging message not needed * adding cert bundle to commit and push * improving the condition to add ca cert in commit and push * fixing unit test * fixing unit test * fixing unit test * fixing unit test * fixing unit test --- cmd/batsExecuteTests.go | 4 +- cmd/gitopsUpdateDeployment.go | 90 +++++++++++++++---- cmd/gitopsUpdateDeployment_generated.go | 39 +++++--- cmd/gitopsUpdateDeployment_test.go | 12 ++- pkg/git/git.go | 24 +++-- pkg/git/git_test.go | 8 +- .../metadata/gitopsUpdateDeployment.yaml | 7 ++ 7 files changed, 138 insertions(+), 46 deletions(-) diff --git a/cmd/batsExecuteTests.go b/cmd/batsExecuteTests.go index 6bd28f704f..aa4251ebfa 100644 --- a/cmd/batsExecuteTests.go +++ b/cmd/batsExecuteTests.go @@ -108,7 +108,9 @@ func runBatsExecuteTests(config *batsExecuteTestsOptions, telemetryData *telemet } func (b *batsExecuteTestsUtilsBundle) CloneRepo(URL string) error { - _, err := pipergit.PlainClone("", "", URL, "bats-core") + // ToDo: BatsExecute test needs to check if the repo can come from a + // enterprise github instance and needs ca-cert handelling seperately + _, err := pipergit.PlainClone("", "", URL, "bats-core", []byte{}) return err } diff --git a/cmd/gitopsUpdateDeployment.go b/cmd/gitopsUpdateDeployment.go index 6695019418..7220025de8 100644 --- a/cmd/gitopsUpdateDeployment.go +++ b/cmd/gitopsUpdateDeployment.go @@ -3,9 +3,19 @@ package cmd import ( "bytes" "fmt" + "io" + "net/http" + "os" + "path" + "path/filepath" + "regexp" + "strings" + "time" + "github.com/SAP/jenkins-library/pkg/command" "github.com/SAP/jenkins-library/pkg/docker" gitUtil "github.com/SAP/jenkins-library/pkg/git" + piperhttp "github.com/SAP/jenkins-library/pkg/http" "github.com/SAP/jenkins-library/pkg/log" "github.com/SAP/jenkins-library/pkg/piperutils" "github.com/SAP/jenkins-library/pkg/telemetry" @@ -13,12 +23,6 @@ import ( "github.com/go-git/go-git/v5/plumbing" "github.com/go-git/go-git/v5/plumbing/object" "github.com/pkg/errors" - "io" - "os" - "path/filepath" - "regexp" - "strings" - "time" ) const toolKubectl = "kubectl" @@ -27,8 +31,8 @@ const toolKustomize = "kustomize" type iGitopsUpdateDeploymentGitUtils interface { CommitFiles(filePaths []string, commitMessage, author string) (plumbing.Hash, error) - PushChangesToRepository(username, password string, force *bool) error - PlainClone(username, password, serverURL, directory string) error + PushChangesToRepository(username, password string, force *bool, caCerts []byte) error + PlainClone(username, password, serverURL, directory string, caCerts []byte) error ChangeBranch(branchName string) error } @@ -36,6 +40,7 @@ type gitopsUpdateDeploymentFileUtils interface { TempDir(dir, pattern string) (name string, err error) RemoveAll(path string) error FileWrite(path string, content []byte, perm os.FileMode) error + FileRead(path string) ([]byte, error) Glob(pattern string) ([]string, error) } @@ -51,6 +56,25 @@ type gitopsUpdateDeploymentGitUtils struct { repository *git.Repository } +type gitopsUpdateDeploymentUtilsBundle struct { + *piperhttp.Client +} + +type gitopsUpdateDeploymentUtils interface { + DownloadFile(url, filename string, header http.Header, cookies []*http.Cookie) error +} + +func newGitopsUpdateDeploymentUtilsBundle() gitopsUpdateDeploymentUtils { + utils := gitopsUpdateDeploymentUtilsBundle{ + Client: &piperhttp.Client{}, + } + return &utils +} + +func (g *gitopsUpdateDeploymentUtilsBundle) DownloadFile(url, filename string, header http.Header, cookies []*http.Cookie) error { + return g.Client.DownloadFile(url, filename, header, cookies) +} + func (g *gitopsUpdateDeploymentGitUtils) CommitFiles(filePaths []string, commitMessage, author string) (plumbing.Hash, error) { for _, path := range filePaths { _, err := g.worktree.Add(path) @@ -71,13 +95,13 @@ func (g *gitopsUpdateDeploymentGitUtils) CommitFiles(filePaths []string, commitM return commit, nil } -func (g *gitopsUpdateDeploymentGitUtils) PushChangesToRepository(username, password string, force *bool) error { - return gitUtil.PushChangesToRepository(username, password, force, g.repository) +func (g *gitopsUpdateDeploymentGitUtils) PushChangesToRepository(username, password string, force *bool, caCerts []byte) error { + return gitUtil.PushChangesToRepository(username, password, force, g.repository, caCerts) } -func (g *gitopsUpdateDeploymentGitUtils) PlainClone(username, password, serverURL, directory string) error { +func (g *gitopsUpdateDeploymentGitUtils) PlainClone(username, password, serverURL, directory string, caCerts []byte) error { var err error - g.repository, err = gitUtil.PlainClone(username, password, serverURL, directory) + g.repository, err = gitUtil.PlainClone(username, password, serverURL, directory, caCerts) if err != nil { return errors.Wrapf(err, "plain clone failed '%s'", serverURL) } @@ -126,7 +150,12 @@ func runGitopsUpdateDeployment(config *gitopsUpdateDeploymentOptions, command gi } }() - err = cloneRepositoryAndChangeBranch(config, gitUtils, temporaryFolder) + certs, err := downloadCACertbunde(config.CustomTLSCertificateLinks, gitUtils, fileUtils) + if err != nil { + return err + } + + err = cloneRepositoryAndChangeBranch(config, gitUtils, fileUtils, temporaryFolder, certs) if err != nil { return errors.Wrap(err, "repository could not get prepared") } @@ -190,7 +219,7 @@ func runGitopsUpdateDeployment(config *gitopsUpdateDeploymentOptions, command gi } } - commit, err := commitAndPushChanges(config, gitUtils, allFiles) + commit, err := commitAndPushChanges(config, gitUtils, allFiles, certs) if err != nil { return errors.Wrap(err, "failed to commit and push changes") } @@ -292,8 +321,9 @@ func logNotRequiredButFilledFieldForKustomize(config *gitopsUpdateDeploymentOpti } } -func cloneRepositoryAndChangeBranch(config *gitopsUpdateDeploymentOptions, gitUtils iGitopsUpdateDeploymentGitUtils, temporaryFolder string) error { - err := gitUtils.PlainClone(config.Username, config.Password, config.ServerURL, temporaryFolder) +func cloneRepositoryAndChangeBranch(config *gitopsUpdateDeploymentOptions, gitUtils iGitopsUpdateDeploymentGitUtils, fileUtils gitopsUpdateDeploymentFileUtils, temporaryFolder string, certs []byte) error { + + err := gitUtils.PlainClone(config.Username, config.Password, config.ServerURL, temporaryFolder, certs) if err != nil { return errors.Wrap(err, "failed to plain clone repository") } @@ -305,6 +335,30 @@ func cloneRepositoryAndChangeBranch(config *gitopsUpdateDeploymentOptions, gitUt return nil } +func downloadCACertbunde(customTlsCertificateLinks []string, gitUtils iGitopsUpdateDeploymentGitUtils, fileUtils gitopsUpdateDeploymentFileUtils) ([]byte, error) { + certs := []byte{} + utils := newGitopsUpdateDeploymentUtilsBundle() + if len(customTlsCertificateLinks) > 0 { + for _, customTlsCertificateLink := range customTlsCertificateLinks { + log.Entry().Infof("Downloading CA certs %s into file '%s'", customTlsCertificateLink, path.Base(customTlsCertificateLink)) + err := utils.DownloadFile(customTlsCertificateLink, path.Base(customTlsCertificateLink), nil, nil) + if err != nil { + return certs, nil + } + + content, err := fileUtils.FileRead(path.Base(customTlsCertificateLink)) + if err != nil { + return certs, nil + } + log.Entry().Infof("CA certs added successfully to cert pool") + + certs = append(certs, content...) + } + } + + return certs, nil +} + func executeKubectl(config *gitopsUpdateDeploymentOptions, command gitopsUpdateDeploymentExecRunner, filePath string) ([]byte, error) { var outputBytes []byte registryImage, err := buildRegistryPlusImage(config) @@ -444,7 +498,7 @@ func buildRegistryPlusImageAndTagSeparately(config *gitopsUpdateDeploymentOption } -func commitAndPushChanges(config *gitopsUpdateDeploymentOptions, gitUtils iGitopsUpdateDeploymentGitUtils, filePaths []string) (plumbing.Hash, error) { +func commitAndPushChanges(config *gitopsUpdateDeploymentOptions, gitUtils iGitopsUpdateDeploymentGitUtils, filePaths []string, certs []byte) (plumbing.Hash, error) { commitMessage := config.CommitMessage if commitMessage == "" { @@ -456,7 +510,7 @@ func commitAndPushChanges(config *gitopsUpdateDeploymentOptions, gitUtils iGitop return [20]byte{}, errors.Wrap(err, "committing changes failed") } - err = gitUtils.PushChangesToRepository(config.Username, config.Password, &config.ForcePush) + err = gitUtils.PushChangesToRepository(config.Username, config.Password, &config.ForcePush, certs) if err != nil { return [20]byte{}, errors.Wrap(err, "pushing changes failed") } diff --git a/cmd/gitopsUpdateDeployment_generated.go b/cmd/gitopsUpdateDeployment_generated.go index 97eb254291..ca1c926643 100644 --- a/cmd/gitopsUpdateDeployment_generated.go +++ b/cmd/gitopsUpdateDeployment_generated.go @@ -16,20 +16,21 @@ import ( ) type gitopsUpdateDeploymentOptions struct { - BranchName string `json:"branchName,omitempty"` - CommitMessage string `json:"commitMessage,omitempty"` - ServerURL string `json:"serverUrl,omitempty"` - ForcePush bool `json:"forcePush,omitempty"` - Username string `json:"username,omitempty"` - Password string `json:"password,omitempty"` - FilePath string `json:"filePath,omitempty"` - ContainerName string `json:"containerName,omitempty"` - ContainerRegistryURL string `json:"containerRegistryUrl,omitempty"` - ContainerImageNameTag string `json:"containerImageNameTag,omitempty"` - ChartPath string `json:"chartPath,omitempty"` - HelmValues []string `json:"helmValues,omitempty"` - DeploymentName string `json:"deploymentName,omitempty"` - Tool string `json:"tool,omitempty" validate:"possible-values=kubectl helm kustomize"` + BranchName string `json:"branchName,omitempty"` + CommitMessage string `json:"commitMessage,omitempty"` + ServerURL string `json:"serverUrl,omitempty"` + ForcePush bool `json:"forcePush,omitempty"` + Username string `json:"username,omitempty"` + Password string `json:"password,omitempty"` + FilePath string `json:"filePath,omitempty"` + ContainerName string `json:"containerName,omitempty"` + ContainerRegistryURL string `json:"containerRegistryUrl,omitempty"` + ContainerImageNameTag string `json:"containerImageNameTag,omitempty"` + ChartPath string `json:"chartPath,omitempty"` + HelmValues []string `json:"helmValues,omitempty"` + DeploymentName string `json:"deploymentName,omitempty"` + Tool string `json:"tool,omitempty" validate:"possible-values=kubectl helm kustomize"` + CustomTLSCertificateLinks []string `json:"customTlsCertificateLinks,omitempty"` } // GitopsUpdateDeploymentCommand Updates Kubernetes Deployment Manifest in an Infrastructure Git Repository @@ -155,6 +156,7 @@ func addGitopsUpdateDeploymentFlags(cmd *cobra.Command, stepConfig *gitopsUpdate cmd.Flags().StringSliceVar(&stepConfig.HelmValues, "helmValues", []string{}, "List of helm values as YAML file reference or URL (as per helm parameter description for `-f` / `--values`)") cmd.Flags().StringVar(&stepConfig.DeploymentName, "deploymentName", os.Getenv("PIPER_deploymentName"), "Defines the name of the deployment. In case of `kustomize` this is the name or alias of the image in the `kustomization.yaml`") cmd.Flags().StringVar(&stepConfig.Tool, "tool", `kubectl`, "Defines the tool which should be used to update the deployment description.") + cmd.Flags().StringSliceVar(&stepConfig.CustomTLSCertificateLinks, "customTlsCertificateLinks", []string{}, "List containing download links of custom TLS certificates. This is required to ensure trusted connections to registries with custom certificates.") cmd.MarkFlagRequired("branchName") cmd.MarkFlagRequired("serverUrl") @@ -343,6 +345,15 @@ func gitopsUpdateDeploymentMetadata() config.StepData { Aliases: []config.Alias{}, Default: `kubectl`, }, + { + Name: "customTlsCertificateLinks", + ResourceRef: []config.ResourceReference{}, + Scope: []string{"PARAMETERS", "STAGES", "STEPS"}, + Type: "[]string", + Mandatory: false, + Aliases: []config.Alias{}, + Default: []string{}, + }, }, }, Containers: []config.Container{ diff --git a/cmd/gitopsUpdateDeployment_test.go b/cmd/gitopsUpdateDeployment_test.go index 2d05f1ce39..a776fdb83c 100644 --- a/cmd/gitopsUpdateDeployment_test.go +++ b/cmd/gitopsUpdateDeployment_test.go @@ -772,6 +772,7 @@ type filesMock struct { failOnCreation bool failOnDeletion bool failOnWrite bool + failOnRead bool failOnGlob bool path string } @@ -783,6 +784,13 @@ func (f filesMock) FileWrite(path string, content []byte, perm os.FileMode) erro return piperutils.Files{}.FileWrite(path, content, perm) } +func (f filesMock) FileRead(path string) ([]byte, error) { + if f.failOnRead { + return []byte{}, errors.New("error appeared") + } + return piperutils.Files{}.FileRead(path) +} + func (f filesMock) TempDir(dir string, pattern string) (name string, err error) { if f.failOnCreation { return "", errors.New("error appeared") @@ -848,7 +856,7 @@ func (v *gitUtilsMock) CommitFiles(newFiles []string, commitMessage string, _ st return [20]byte{123}, nil } -func (v gitUtilsMock) PushChangesToRepository(_ string, _ string, force *bool) error { +func (v gitUtilsMock) PushChangesToRepository(_ string, _ string, force *bool, caCerts []byte) error { if v.failOnPush { return errors.New("error on push") } @@ -858,7 +866,7 @@ func (v gitUtilsMock) PushChangesToRepository(_ string, _ string, force *bool) e return nil } -func (v *gitUtilsMock) PlainClone(_, _, _, directory string) error { +func (v *gitUtilsMock) PlainClone(_, _, _, directory string, caCerts []byte) error { if v.skipClone { return nil } diff --git a/pkg/git/git.go b/pkg/git/git.go index f843d1ee36..c5758f281a 100644 --- a/pkg/git/git.go +++ b/pkg/git/git.go @@ -1,13 +1,14 @@ package git import ( + "time" + "github.com/SAP/jenkins-library/pkg/log" "github.com/go-git/go-git/v5" "github.com/go-git/go-git/v5/plumbing" "github.com/go-git/go-git/v5/plumbing/object" "github.com/go-git/go-git/v5/plumbing/transport/http" "github.com/pkg/errors" - "time" ) // utilsWorkTree interface abstraction of git.Worktree to enable tests @@ -53,14 +54,18 @@ func commitSingleFile(filePath, commitMessage, author string, worktree utilsWork } // PushChangesToRepository Pushes all committed changes in the repository to the remote repository -func PushChangesToRepository(username, password string, force *bool, repository *git.Repository) error { - return pushChangesToRepository(username, password, force, repository) +func PushChangesToRepository(username, password string, force *bool, repository *git.Repository, caCerts []byte) error { + return pushChangesToRepository(username, password, force, repository, caCerts) } -func pushChangesToRepository(username, password string, force *bool, repository utilsRepository) error { +func pushChangesToRepository(username, password string, force *bool, repository utilsRepository, caCerts []byte) error { pushOptions := &git.PushOptions{ Auth: &http.BasicAuth{Username: username, Password: password}, } + + if len(caCerts) > 0 { + pushOptions.CABundle = caCerts + } if force != nil { pushOptions.Force = *force } @@ -72,16 +77,21 @@ func pushChangesToRepository(username, password string, force *bool, repository } // PlainClone Clones a non-bare repository to the provided directory -func PlainClone(username, password, serverURL, directory string) (*git.Repository, error) { +func PlainClone(username, password, serverURL, directory string, caCerts []byte) (*git.Repository, error) { abstractedGit := &abstractionGit{} - return plainClone(username, password, serverURL, directory, abstractedGit) + return plainClone(username, password, serverURL, directory, abstractedGit, caCerts) } -func plainClone(username, password, serverURL, directory string, abstractionGit utilsGit) (*git.Repository, error) { +func plainClone(username, password, serverURL, directory string, abstractionGit utilsGit, caCerts []byte) (*git.Repository, error) { gitCloneOptions := git.CloneOptions{ Auth: &http.BasicAuth{Username: username, Password: password}, URL: serverURL, } + + if len(caCerts) > 0 { + gitCloneOptions.CABundle = caCerts + } + repository, err := abstractionGit.plainClone(directory, false, &gitCloneOptions) if err != nil { return nil, errors.Wrap(err, "failed to clone git") diff --git a/pkg/git/git_test.go b/pkg/git/git_test.go index 39e50e3348..f5cf660db4 100644 --- a/pkg/git/git_test.go +++ b/pkg/git/git_test.go @@ -51,13 +51,13 @@ func TestPushChangesToRepository(t *testing.T) { t.Parallel() err := pushChangesToRepository("user", "password", nil, RepositoryMock{ test: t, - }) + }, []byte{}) assert.NoError(t, err) }) t.Run("error pushing", func(t *testing.T) { t.Parallel() - err := pushChangesToRepository("user", "password", nil, RepositoryMockError{}) + err := pushChangesToRepository("user", "password", nil, RepositoryMockError{}, []byte{}) assert.EqualError(t, err, "failed to push commit: error on push commits") }) } @@ -67,7 +67,7 @@ func TestPlainClone(t *testing.T) { t.Run("successful clone", func(t *testing.T) { t.Parallel() abstractedGit := &UtilsGitMock{} - _, err := plainClone("user", "password", "URL", "directory", abstractedGit) + _, err := plainClone("user", "password", "URL", "directory", abstractedGit, []byte{}) assert.NoError(t, err) assert.Equal(t, "directory", abstractedGit.path) assert.False(t, abstractedGit.isBare) @@ -78,7 +78,7 @@ func TestPlainClone(t *testing.T) { t.Run("error on cloning", func(t *testing.T) { t.Parallel() abstractedGit := UtilsGitMockError{} - _, err := plainClone("user", "password", "URL", "directory", abstractedGit) + _, err := plainClone("user", "password", "URL", "directory", abstractedGit, []byte{}) assert.EqualError(t, err, "failed to clone git: error during clone") }) } diff --git a/resources/metadata/gitopsUpdateDeployment.yaml b/resources/metadata/gitopsUpdateDeployment.yaml index 22bceee9bf..5349e2e74b 100644 --- a/resources/metadata/gitopsUpdateDeployment.yaml +++ b/resources/metadata/gitopsUpdateDeployment.yaml @@ -190,6 +190,13 @@ spec: - kubectl - helm - kustomize + - name: customTlsCertificateLinks + type: "[]string" + description: List containing download links of custom TLS certificates. This is required to ensure trusted connections to registries with custom certificates. + scope: + - PARAMETERS + - STAGES + - STEPS containers: - image: dtzar/helm-kubectl:3.8.0 workingDir: /config