From 13f1e94adefacd20c7a6f95135760a22b9701dab Mon Sep 17 00:00:00 2001 From: Pavel Busko Date: Tue, 4 Jul 2023 14:19:02 +0200 Subject: [PATCH 01/10] fix(cnbBuild): read dockerConfigJSON from CPE and merge it with user-provided (#4444) Co-authored-by: Ralf Pannemans --- cmd/cnbBuild.go | 68 +++++++------------ cmd/cnbBuild_generated.go | 49 +++++-------- cmd/cnbBuild_test.go | 11 ++- integration/integration_cnb_test.go | 19 +++--- .../custom/dockerConfigJSON | 1 + .../TestCnbIntegration/.pipeline/config.json | 5 ++ .../testdata/TestCnbIntegration/config.yml | 2 +- pkg/docker/docker.go | 59 ++++++++++++++++ pkg/docker/docker_test.go | 44 ++++++++++++ resources/metadata/cnbBuild.yaml | 33 ++------- 10 files changed, 170 insertions(+), 121 deletions(-) create mode 100644 integration/testdata/TestCnbIntegration/.pipeline/commonPipelineEnvironment/custom/dockerConfigJSON create mode 100644 integration/testdata/TestCnbIntegration/.pipeline/config.json diff --git a/cmd/cnbBuild.go b/cmd/cnbBuild.go index 6b3dd3fa2c..4ef75a4c07 100644 --- a/cmd/cnbBuild.go +++ b/cmd/cnbBuild.go @@ -7,7 +7,6 @@ import ( "os" "path" "path/filepath" - "regexp" "github.com/SAP/jenkins-library/pkg/buildsettings" "github.com/SAP/jenkins-library/pkg/certutils" @@ -212,28 +211,26 @@ func extractZip(source, target string) error { return nil } -func renameDockerConfig(config *cnbBuildOptions, utils cnbutils.BuildUtils) error { - if filepath.Base(config.DockerConfigJSON) != "config.json" { - log.Entry().Debugf("Renaming docker config file from '%s' to 'config.json'", filepath.Base(config.DockerConfigJSON)) +func ensureDockerConfig(config *cnbBuildOptions, utils cnbutils.BuildUtils) error { + newFile := "/tmp/config.json" + if config.DockerConfigJSON == "" { + config.DockerConfigJSON = newFile - newPath := filepath.Join(filepath.Dir(config.DockerConfigJSON), "config.json") - alreadyExists, err := utils.FileExists(newPath) - if err != nil { - return err - } - - if alreadyExists { - return nil - } + return utils.FileWrite(config.DockerConfigJSON, []byte("{}"), os.ModePerm) + } - err = utils.FileRename(config.DockerConfigJSON, newPath) - if err != nil { - return err - } + log.Entry().Debugf("Copying docker config file from '%s' to '%s'", config.DockerConfigJSON, newFile) + _, err := utils.Copy(config.DockerConfigJSON, newFile) + if err != nil { + return err + } - config.DockerConfigJSON = newPath + err = utils.Chmod(newFile, 0644) + if err != nil { + return err } + config.DockerConfigJSON = newFile return nil } @@ -382,38 +379,19 @@ func callCnbBuild(config *cnbBuildOptions, telemetryData *telemetry.CustomData, } commonPipelineEnvironment.custom.buildSettingsInfo = buildSettingsInfo - if len(config.DockerConfigJSON) > 0 { - err = renameDockerConfig(config, utils) - if err != nil { - log.SetErrorCategory(log.ErrorConfiguration) - return errors.Wrapf(err, "failed to rename DockerConfigJSON file '%s'", config.DockerConfigJSON) - } + err = ensureDockerConfig(config, utils) + if err != nil { + log.SetErrorCategory(log.ErrorConfiguration) + return errors.Wrapf(err, "failed to create/rename DockerConfigJSON file") } - if config.ContainerRegistryUser != "" && config.ContainerRegistryPassword != "" { - log.Entry().Debug("enhancing docker config with the provided credentials") - if config.DockerConfigJSON == "" { - config.DockerConfigJSON = "/tmp/config.json" - } - log.Entry().Debugf("using docker config file %q", config.DockerConfigJSON) - - if matched, _ := regexp.MatchString("^(http|https)://.*", config.ContainerRegistryURL); !matched { - config.ContainerRegistryURL = fmt.Sprintf("https://%s", config.ContainerRegistryURL) - } - - containerRegistry, err := docker.ContainerRegistryFromURL(config.ContainerRegistryURL) + if config.DockerConfigJSONCPE != "" { + log.Entry().Debugf("merging docker config file '%s' into '%s'", config.DockerConfigJSONCPE, config.DockerConfigJSON) + err = docker.MergeDockerConfigJSON(config.DockerConfigJSONCPE, config.DockerConfigJSON, utils) if err != nil { log.SetErrorCategory(log.ErrorConfiguration) - return errors.Wrapf(err, "failed to read registry url %q", config.ContainerRegistryURL) + return errors.Wrapf(err, "failed to merge DockerConfigJSON files") } - - _, err = docker.CreateDockerConfigJSON(containerRegistry, config.ContainerRegistryUser, config.ContainerRegistryPassword, "", config.DockerConfigJSON, utils) - if err != nil { - log.SetErrorCategory(log.ErrorBuild) - return errors.Wrapf(err, "failed to update DockerConfigJSON file %q", config.DockerConfigJSON) - } - - log.Entry().Debugf("docker config %q has been updated", config.DockerConfigJSON) } mergedConfigs, err := processConfigs(*config, config.MultipleImages) diff --git a/cmd/cnbBuild_generated.go b/cmd/cnbBuild_generated.go index c714123ed9..945daa6600 100644 --- a/cmd/cnbBuild_generated.go +++ b/cmd/cnbBuild_generated.go @@ -26,13 +26,12 @@ type cnbBuildOptions struct { ContainerImageAlias string `json:"containerImageAlias,omitempty"` ContainerImageTag string `json:"containerImageTag,omitempty"` ContainerRegistryURL string `json:"containerRegistryUrl,omitempty"` - ContainerRegistryUser string `json:"containerRegistryUser,omitempty"` - ContainerRegistryPassword string `json:"containerRegistryPassword,omitempty"` Buildpacks []string `json:"buildpacks,omitempty"` BuildEnvVars map[string]interface{} `json:"buildEnvVars,omitempty"` Path string `json:"path,omitempty"` ProjectDescriptor string `json:"projectDescriptor,omitempty"` DockerConfigJSON string `json:"dockerConfigJSON,omitempty"` + DockerConfigJSONCPE string `json:"dockerConfigJSONCPE,omitempty"` CustomTLSCertificateLinks []string `json:"customTlsCertificateLinks,omitempty"` AdditionalTags []string `json:"additionalTags,omitempty"` Bindings map[string]interface{} `json:"bindings,omitempty"` @@ -158,6 +157,7 @@ func CnbBuildCommand() *cobra.Command { return err } log.RegisterSecret(stepConfig.DockerConfigJSON) + log.RegisterSecret(stepConfig.DockerConfigJSONCPE) if len(GeneralConfig.HookConfig.SentryConfig.Dsn) > 0 { sentryHook := log.NewSentryHook(GeneralConfig.HookConfig.SentryConfig.Dsn, GeneralConfig.CorrelationID) @@ -226,13 +226,12 @@ func addCnbBuildFlags(cmd *cobra.Command, stepConfig *cnbBuildOptions) { cmd.Flags().StringVar(&stepConfig.ContainerImageAlias, "containerImageAlias", os.Getenv("PIPER_containerImageAlias"), "Logical name used for this image.\n") cmd.Flags().StringVar(&stepConfig.ContainerImageTag, "containerImageTag", os.Getenv("PIPER_containerImageTag"), "Tag of the container which will be built") cmd.Flags().StringVar(&stepConfig.ContainerRegistryURL, "containerRegistryUrl", os.Getenv("PIPER_containerRegistryUrl"), "Container registry where the image should be pushed to.\n\n**Note**: `containerRegistryUrl` should include only the domain. If you want to publish an image under `docker.io/example/my-image`, you must set `containerRegistryUrl: \"docker.io\"` and `containerImageName: \"example/my-image\"`.\n") - cmd.Flags().StringVar(&stepConfig.ContainerRegistryUser, "containerRegistryUser", os.Getenv("PIPER_containerRegistryUser"), "Username of the container registry where the image should be pushed to - which will updated in a docker config json file. If a docker config json file is provided via parameter `dockerConfigJSON`, then the existing file will be enhanced") - cmd.Flags().StringVar(&stepConfig.ContainerRegistryPassword, "containerRegistryPassword", os.Getenv("PIPER_containerRegistryPassword"), "Password of the container registry where the image should be pushed to - which will updated in a docker config json file. If a docker config json file is provided via parameter `dockerConfigJSON`, then the existing file will be enhanced") cmd.Flags().StringSliceVar(&stepConfig.Buildpacks, "buildpacks", []string{}, "List of custom buildpacks to use in the form of `$HOSTNAME/$REPO[:$TAG]`.") cmd.Flags().StringVar(&stepConfig.Path, "path", os.Getenv("PIPER_path"), "Glob that should either point to a directory with your sources or one artifact in zip format.\nThis property determines the input to the buildpack.\n") cmd.Flags().StringVar(&stepConfig.ProjectDescriptor, "projectDescriptor", `project.toml`, "Relative path to the project.toml file.\nSee [buildpacks.io](https://buildpacks.io/docs/reference/config/project-descriptor/) for the reference.\nParameters passed to the cnbBuild step will take precedence over the parameters set in the project.toml file, except the `env` block.\nEnvironment variables declared in a project descriptor file, will be merged with the `buildEnvVars` property, with the `buildEnvVars` having a precedence.\n\n*Note*: The project descriptor path should be relative to what is set in the [path](#path) property. If the `path` property is pointing to a zip archive (e.g. jar file), project descriptor path will be relative to the root of the workspace.\n\n*Note*: Inline buildpacks (see [specification](https://buildpacks.io/docs/reference/config/project-descriptor/#build-_table-optional_)) are not supported yet.\n") cmd.Flags().StringVar(&stepConfig.DockerConfigJSON, "dockerConfigJSON", os.Getenv("PIPER_dockerConfigJSON"), "Path to the file `.docker/config.json` - this is typically provided by your CI/CD system. You can find more details about the Docker credentials in the [Docker documentation](https://docs.docker.com/engine/reference/commandline/login/).") + cmd.Flags().StringVar(&stepConfig.DockerConfigJSONCPE, "dockerConfigJSONCPE", os.Getenv("PIPER_dockerConfigJSONCPE"), "This property is intended only for reading the `dockerConfigJSON` from the Common Pipeline Environment. If you want to provide your own credentials, please refer to the [dockerConfigJSON](#dockerConfigJSON) property. If both properties are set, the config files will be merged, with the [dockerConfigJSON](#dockerConfigJSON) having higher priority.") cmd.Flags().StringSliceVar(&stepConfig.CustomTLSCertificateLinks, "customTlsCertificateLinks", []string{}, "List containing download links of custom TLS certificates. This is required to ensure trusted connections to registries with custom certificates.") cmd.Flags().StringSliceVar(&stepConfig.AdditionalTags, "additionalTags", []string{}, "List of tags which will be pushed to the registry (additionally to the provided `containerImageTag`), e.g. \"latest\".") @@ -312,34 +311,6 @@ func cnbBuildMetadata() config.StepData { Aliases: []config.Alias{{Name: "dockerRegistryUrl"}}, Default: os.Getenv("PIPER_containerRegistryUrl"), }, - { - Name: "containerRegistryUser", - ResourceRef: []config.ResourceReference{ - { - Name: "commonPipelineEnvironment", - Param: "container/repositoryUsername", - }, - }, - Scope: []string{"GENERAL", "PARAMETERS", "STAGES", "STEPS"}, - Type: "string", - Mandatory: false, - Aliases: []config.Alias{{Name: "dockerRegistryUser"}}, - Default: os.Getenv("PIPER_containerRegistryUser"), - }, - { - Name: "containerRegistryPassword", - ResourceRef: []config.ResourceReference{ - { - Name: "commonPipelineEnvironment", - Param: "container/repositoryPassword", - }, - }, - Scope: []string{"GENERAL", "PARAMETERS", "STAGES", "STEPS"}, - Type: "string", - Mandatory: false, - Aliases: []config.Alias{{Name: "dockerRegistryPassword"}}, - Default: os.Getenv("PIPER_containerRegistryPassword"), - }, { Name: "buildpacks", ResourceRef: []config.ResourceReference{ @@ -400,6 +371,20 @@ func cnbBuildMetadata() config.StepData { Aliases: []config.Alias{}, Default: os.Getenv("PIPER_dockerConfigJSON"), }, + { + Name: "dockerConfigJSONCPE", + ResourceRef: []config.ResourceReference{ + { + Name: "commonPipelineEnvironment", + Param: "custom/dockerConfigJSON", + }, + }, + Scope: []string{}, + Type: "string", + Mandatory: false, + Aliases: []config.Alias{}, + Default: os.Getenv("PIPER_dockerConfigJSONCPE"), + }, { Name: "customTlsCertificateLinks", ResourceRef: []config.ResourceReference{}, diff --git a/cmd/cnbBuild_test.go b/cmd/cnbBuild_test.go index 98f8b0f691..2f1fa8226c 100644 --- a/cmd/cnbBuild_test.go +++ b/cmd/cnbBuild_test.go @@ -230,11 +230,8 @@ func TestRunCnbBuild(t *testing.T) { assert.Contains(t, runner.Calls[0].Params, fmt.Sprintf("%s/%s:%s", config.ContainerRegistryURL, config.ContainerImageName, config.ContainerImageTag)) assert.Contains(t, runner.Calls[0].Params, fmt.Sprintf("%s/%s:latest", config.ContainerRegistryURL, config.ContainerImageName)) - initialFileExists, _ := utils.FileExists("/path/to/test.json") - renamedFileExists, _ := utils.FileExists("/path/to/config.json") - - assert.False(t, initialFileExists) - assert.True(t, renamedFileExists) + copiedFileExists, _ := utils.FileExists("/tmp/config.json") + assert.True(t, copiedFileExists) }) t.Run("success case (customTlsCertificates)", func(t *testing.T) { @@ -420,7 +417,7 @@ func TestRunCnbBuild(t *testing.T) { addBuilderFiles(&utils) err := callCnbBuild(&config, &telemetry.CustomData{}, &utils, &cnbBuildCommonPipelineEnvironment{}, &piperhttp.Client{}) - assert.EqualError(t, err, "failed to generate CNB_REGISTRY_AUTH: could not read 'not-there/config.json'") + assert.EqualError(t, err, "failed to create/rename DockerConfigJSON file: cannot copy 'not-there/config.json': file does not exist") }) t.Run("error case: DockerConfigJSON file not there (not config.json)", func(t *testing.T) { @@ -436,7 +433,7 @@ func TestRunCnbBuild(t *testing.T) { addBuilderFiles(&utils) err := callCnbBuild(&config, &telemetry.CustomData{}, &utils, &cnbBuildCommonPipelineEnvironment{}, &piperhttp.Client{}) - assert.EqualError(t, err, "failed to rename DockerConfigJSON file 'not-there': renaming file 'not-there' is not supported, since it does not exist, or is not a leaf-entry") + assert.EqualError(t, err, "failed to create/rename DockerConfigJSON file: cannot copy 'not-there': file does not exist") }) t.Run("error case: dockerImage is not a valid builder", func(t *testing.T) { diff --git a/integration/integration_cnb_test.go b/integration/integration_cnb_test.go index 159cb0bfee..944651cf8e 100644 --- a/integration/integration_cnb_test.go +++ b/integration/integration_cnb_test.go @@ -44,7 +44,7 @@ func TestCNBIntegrationNPMProject(t *testing.T) { container := givenThisContainer(t, IntegrationTestDockerExecRunnerBundle{ Image: baseBuilder, User: "cnb", - TestDir: []string{"testdata"}, + TestDir: []string{"testdata", "TestCnbIntegration"}, Network: fmt.Sprintf("container:%s", registryContainer.GetContainerID()), Environment: map[string]string{ "PIPER_VAULTCREDENTIAL_DYNATRACE_API_KEY": "api-key-content", @@ -54,14 +54,14 @@ func TestCNBIntegrationNPMProject(t *testing.T) { container2 := givenThisContainer(t, IntegrationTestDockerExecRunnerBundle{ Image: baseBuilder, User: "cnb", - TestDir: []string{"testdata"}, + TestDir: []string{"testdata", "TestCnbIntegration"}, Network: fmt.Sprintf("container:%s", registryContainer.GetContainerID()), Environment: map[string]string{ "PIPER_VAULTCREDENTIAL_DYNATRACE_API_KEY": "api-key-content", }, }) - err := container.whenRunningPiperCommand("cnbBuild", "--noTelemetry", "--verbose", "--path", "TestCnbIntegration/project", "--customConfig", "TestCnbIntegration/config.yml", "--containerImageName", "node", "--containerImageTag", "0.0.1", "--dockerConfigJSON", "TestCnbIntegration/config.json", "--containerRegistryUrl", fmt.Sprintf("http://%s", registryURL), "--containerRegistryUser", "foo", "--containerRegistryPassword", "bar", "--defaultProcess", "greeter") + err := container.whenRunningPiperCommand("cnbBuild", "--noTelemetry", "--verbose", "--path", "project", "--customConfig", "config.yml", "--containerImageName", "node", "--containerImageTag", "0.0.1", "--dockerConfigJSON", "config.json", "--containerRegistryUrl", fmt.Sprintf("http://%s", registryURL), "--defaultProcess", "greeter") assert.NoError(t, err) container.assertHasOutput(t, "running command: /cnb/lifecycle/creator") container.assertHasOutput(t, "Selected Node Engine version (using BP_NODE_VERSION): 16") @@ -70,10 +70,10 @@ func TestCNBIntegrationNPMProject(t *testing.T) { container.assertHasOutput(t, "Setting default process type 'greeter'") container.assertHasOutput(t, "*** Images (sha256:") container.assertHasOutput(t, "SUCCESS") - container.assertFileContentEquals(t, "/project/TestCnbIntegration/config.json", "{\"auths\":{\"localhost:5000\":{\"auth\":\"Zm9vOmJhcg==\"},\"test.registry.io\":{}}}") + container.assertFileContentEquals(t, "/tmp/config.json", "{\n\t\"auths\": {\n\t\t\"test.registry.io\": {},\n\t\t\"test2.registry.io\": {}\n\t}\n}") container.terminate(t) - err = container2.whenRunningPiperCommand("cnbBuild", "--noTelemetry", "--verbose", "--path", "TestCnbIntegration/project", "--customConfig", "TestCnbIntegration/config.yml", "--containerImageName", "node", "--containerImageTag", "0.0.1", "--containerRegistryUrl", registryURL, "--containerRegistryUser", "foo", "--containerRegistryPassword", "bar", "--projectDescriptor", "project-with-id.toml") + err = container2.whenRunningPiperCommand("cnbBuild", "--noTelemetry", "--verbose", "--path", "project", "--customConfig", "config.yml", "--containerImageName", "node", "--containerImageTag", "0.0.1", "--containerRegistryUrl", registryURL, "--projectDescriptor", "project-with-id.toml") assert.NoError(t, err) container2.assertHasOutput(t, "running command: /cnb/lifecycle/creator") container2.assertHasOutput(t, "Selected Node Engine version (using BP_NODE_VERSION): 16") @@ -81,7 +81,7 @@ func TestCNBIntegrationNPMProject(t *testing.T) { container2.assertHasOutput(t, fmt.Sprintf("Saving %s/node:0.0.1", registryURL)) container2.assertHasOutput(t, "*** Images (sha256:") container2.assertHasOutput(t, "SUCCESS") - container2.assertFileContentEquals(t, "/tmp/config.json", "{\"auths\":{\"localhost:5000\":{\"auth\":\"Zm9vOmJhcg==\"}}}") + container2.assertFileContentEquals(t, "/tmp/config.json", "{\n\t\"auths\": {\n\t\t\"test2.registry.io\": {}\n\t}\n}") container2.terminate(t) } @@ -243,17 +243,16 @@ func TestCNBIntegrationBindings(t *testing.T) { container := givenThisContainer(t, IntegrationTestDockerExecRunnerBundle{ Image: baseBuilder, User: "cnb", - TestDir: []string{"testdata"}, + TestDir: []string{"testdata", "TestCnbIntegration"}, Network: fmt.Sprintf("container:%s", registryContainer.GetContainerID()), Environment: map[string]string{ "PIPER_VAULTCREDENTIAL_DYNATRACE_API_KEY": "api-key-content", }, }) - err := container.whenRunningPiperCommand("cnbBuild", "--noTelemetry", "--verbose", "--customConfig", "TestCnbIntegration/config.yml", "--containerImageName", "not-found", "--containerImageTag", "0.0.1", "--containerRegistryUrl", registryURL, "--path", "TestMtaIntegration/maven") - assert.Error(t, err) + err := container.whenRunningPiperCommand("cnbBuild", "--noTelemetry", "--verbose", "--customConfig", "config.yml", "--containerImageName", "not-found", "--containerImageTag", "0.0.1", "--containerRegistryUrl", registryURL, "--path", "project") + assert.NoError(t, err) - container.assertHasOutput(t, "bindings/maven-settings/settings.xml: only whitespace content allowed before start tag") container.assertHasFiles(t, "/tmp/platform/bindings/dummy-binding/type", "/tmp/platform/bindings/dummy-binding/dummy.yml", diff --git a/integration/testdata/TestCnbIntegration/.pipeline/commonPipelineEnvironment/custom/dockerConfigJSON b/integration/testdata/TestCnbIntegration/.pipeline/commonPipelineEnvironment/custom/dockerConfigJSON new file mode 100644 index 0000000000..7370907038 --- /dev/null +++ b/integration/testdata/TestCnbIntegration/.pipeline/commonPipelineEnvironment/custom/dockerConfigJSON @@ -0,0 +1 @@ +.pipeline/config.json \ No newline at end of file diff --git a/integration/testdata/TestCnbIntegration/.pipeline/config.json b/integration/testdata/TestCnbIntegration/.pipeline/config.json new file mode 100644 index 0000000000..3316381b5e --- /dev/null +++ b/integration/testdata/TestCnbIntegration/.pipeline/config.json @@ -0,0 +1,5 @@ +{ + "auths": { + "test2.registry.io": {} + } +} \ No newline at end of file diff --git a/integration/testdata/TestCnbIntegration/config.yml b/integration/testdata/TestCnbIntegration/config.yml index e2428c1c83..da73f0510e 100644 --- a/integration/testdata/TestCnbIntegration/config.yml +++ b/integration/testdata/TestCnbIntegration/config.yml @@ -12,7 +12,7 @@ steps: type: dummy data: - key: dummy.yml - file: TestCnbIntegration/config.yml + file: config.yml dynatrace: type: Dynatrace data: diff --git a/pkg/docker/docker.go b/pkg/docker/docker.go index d2bb9d7241..dc614119fe 100644 --- a/pkg/docker/docker.go +++ b/pkg/docker/docker.go @@ -1,6 +1,7 @@ package docker import ( + "bytes" "encoding/base64" "encoding/json" "fmt" @@ -14,6 +15,9 @@ import ( "github.com/SAP/jenkins-library/pkg/piperutils" "github.com/pkg/errors" + "github.com/docker/cli/cli/config" + "github.com/docker/cli/cli/config/configfile" + cranecmd "github.com/google/go-containerregistry/cmd/crane/cmd" "github.com/google/go-containerregistry/pkg/authn" "github.com/google/go-containerregistry/pkg/crane" @@ -27,6 +31,61 @@ type AuthEntry struct { Auth string `json:"auth,omitempty"` } +// MergeDockerConfigJSON merges two docker config.json files. +func MergeDockerConfigJSON(sourcePath, targetPath string, utils piperutils.FileUtils) error { + if exists, _ := utils.FileExists(sourcePath); !exists { + return fmt.Errorf("source dockerConfigJSON file %q does not exist", sourcePath) + } + + sourceReader, err := utils.Open(sourcePath) + if err != nil { + return errors.Wrapf(err, "failed to open file %q", sourcePath) + } + defer sourceReader.Close() + + sourceConfig, err := config.LoadFromReader(sourceReader) + if err != nil { + return errors.Wrapf(err, "failed to read file %q", sourcePath) + } + + var targetConfig *configfile.ConfigFile + if exists, _ := utils.FileExists(targetPath); !exists { + log.Entry().Warnf("target dockerConfigJSON file %q does not exist, creating a new one", sourcePath) + targetConfig = configfile.New(targetPath) + } else { + targetReader, err := utils.Open(targetPath) + if err != nil { + return errors.Wrapf(err, "failed to open file %q", targetReader) + } + defer targetReader.Close() + targetConfig, err = config.LoadFromReader(targetReader) + if err != nil { + return errors.Wrapf(err, "failed to read file %q", targetPath) + } + } + + for registry, auth := range sourceConfig.GetAuthConfigs() { + targetConfig.AuthConfigs[registry] = auth + } + + buf := bytes.NewBuffer(nil) + err = targetConfig.SaveToWriter(buf) + if err != nil { + return errors.Wrapf(err, "failed to save file %q", targetPath) + } + + err = utils.MkdirAll(filepath.Dir(targetPath), 0777) + if err != nil { + return fmt.Errorf("failed to create directory path for the file %q: %w", targetPath, err) + } + err = utils.FileWrite(targetPath, buf.Bytes(), 0666) + if err != nil { + return fmt.Errorf("failed to write %q: %w", targetPath, err) + } + + return nil +} + // CreateDockerConfigJSON creates / updates a Docker config.json with registry credentials func CreateDockerConfigJSON(registryURL, username, password, targetPath, configPath string, utils piperutils.FileUtils) (string, error) { diff --git a/pkg/docker/docker_test.go b/pkg/docker/docker_test.go index 399651fd35..d3d8d89922 100644 --- a/pkg/docker/docker_test.go +++ b/pkg/docker/docker_test.go @@ -148,3 +148,47 @@ func TestImageListWithFilePath(t *testing.T) { }) } } + +func TestMergeDockerConfigJSON(t *testing.T) { + t.Parallel() + + t.Run("success - both files present", func(t *testing.T) { + sourceFile := "/tmp/source.json" + targetFile := "/tmp/target.json" + expectedContent := "{\n\t\"auths\": {\n\t\t\"bar\": {},\n\t\t\"foo\": {\n\t\t\t\"auth\": \"Zm9vOmJhcg==\"\n\t\t}\n\t}\n}" + + utilsMock := mock.FilesMock{} + utilsMock.AddFile(targetFile, []byte("{\"auths\": {\"foo\": {\"auth\": \"dGVzdDp0ZXN0\"}}}")) + utilsMock.AddFile(sourceFile, []byte("{\"auths\": {\"bar\": {}, \"foo\": {\"auth\": \"Zm9vOmJhcg==\"}}}")) + + err := MergeDockerConfigJSON(sourceFile, targetFile, &utilsMock) + assert.NoError(t, err) + + content, err := utilsMock.FileRead(targetFile) + assert.NoError(t, err) + assert.Equal(t, expectedContent, string(content)) + }) + + t.Run("success - target file is missing", func(t *testing.T) { + sourceFile := "/tmp/source.json" + targetFile := "/tmp/target.json" + expectedContent := "{\n\t\"auths\": {\n\t\t\"bar\": {},\n\t\t\"foo\": {\n\t\t\t\"auth\": \"Zm9vOmJhcg==\"\n\t\t}\n\t}\n}" + + utilsMock := mock.FilesMock{} + utilsMock.AddFile(sourceFile, []byte("{\"auths\": {\"bar\": {}, \"foo\": {\"auth\": \"Zm9vOmJhcg==\"}}}")) + + err := MergeDockerConfigJSON(sourceFile, targetFile, &utilsMock) + assert.NoError(t, err) + + content, err := utilsMock.FileRead(targetFile) + assert.NoError(t, err) + assert.Equal(t, expectedContent, string(content)) + }) + + t.Run("error - source file is missing", func(t *testing.T) { + utilsMock := mock.FilesMock{} + err := MergeDockerConfigJSON("missing-file", "also-missing-file", &utilsMock) + assert.Error(t, err) + assert.Equal(t, "source dockerConfigJSON file \"missing-file\" does not exist", err.Error()) + }) +} diff --git a/resources/metadata/cnbBuild.yaml b/resources/metadata/cnbBuild.yaml index 809ca5dbdc..9fe08524f7 100644 --- a/resources/metadata/cnbBuild.yaml +++ b/resources/metadata/cnbBuild.yaml @@ -95,32 +95,6 @@ spec: resourceRef: - name: commonPipelineEnvironment param: container/registryUrl - - name: containerRegistryUser - aliases: - - name: dockerRegistryUser - type: string - description: Username of the container registry where the image should be pushed to - which will updated in a docker config json file. If a docker config json file is provided via parameter `dockerConfigJSON`, then the existing file will be enhanced - scope: - - GENERAL - - PARAMETERS - - STAGES - - STEPS - resourceRef: - - name: commonPipelineEnvironment - param: container/repositoryUsername - - name: containerRegistryPassword - aliases: - - name: dockerRegistryPassword - type: string - description: Password of the container registry where the image should be pushed to - which will updated in a docker config json file. If a docker config json file is provided via parameter `dockerConfigJSON`, then the existing file will be enhanced - scope: - - GENERAL - - PARAMETERS - - STAGES - - STEPS - resourceRef: - - name: commonPipelineEnvironment - param: container/repositoryPassword - name: buildpacks type: "[]string" description: List of custom buildpacks to use in the form of `$HOSTNAME/$REPO[:$TAG]`. @@ -181,6 +155,13 @@ spec: - type: vaultSecretFile name: dockerConfigFileVaultSecretName default: docker-config + - name: dockerConfigJSONCPE + type: string + description: This property is intended only for reading the `dockerConfigJSON` from the Common Pipeline Environment. If you want to provide your own credentials, please refer to the [dockerConfigJSON](#dockerConfigJSON) property. If both properties are set, the config files will be merged, with the [dockerConfigJSON](#dockerConfigJSON) having higher priority. + secret: true + resourceRef: + - name: commonPipelineEnvironment + param: custom/dockerConfigJSON - name: customTlsCertificateLinks type: "[]string" description: List containing download links of custom TLS certificates. This is required to ensure trusted connections to registries with custom certificates. From 8fe1d5553e6c805a56adfc46cacdd8bb481deccb Mon Sep 17 00:00:00 2001 From: "renovate[bot]" <29139614+renovate[bot]@users.noreply.github.com> Date: Tue, 4 Jul 2023 14:35:44 +0200 Subject: [PATCH 02/10] chore(deps): update actions/setup-python action to v4 (#4437) Co-authored-by: renovate[bot] <29139614+renovate[bot]@users.noreply.github.com> --- .github/workflows/verify-yaml.yml | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/.github/workflows/verify-yaml.yml b/.github/workflows/verify-yaml.yml index ee13aeb3d8..d2873b0425 100644 --- a/.github/workflows/verify-yaml.yml +++ b/.github/workflows/verify-yaml.yml @@ -15,7 +15,7 @@ jobs: - uses: styfle/cancel-workflow-action@0.10.0 - uses: actions/checkout@master - - uses: actions/setup-python@v2 + - uses: actions/setup-python@v4 with: python-version: 3.8 From c7ab4240e94be05da0cf052554d287f5098b6c5a Mon Sep 17 00:00:00 2001 From: "dependabot[bot]" <49699333+dependabot[bot]@users.noreply.github.com> Date: Tue, 4 Jul 2023 13:00:11 +0000 Subject: [PATCH 03/10] build(deps): bump github.com/docker/distribution (#4359) Bumps [github.com/docker/distribution](https://github.com/docker/distribution) from 2.8.1+incompatible to 2.8.2+incompatible. - [Release notes](https://github.com/docker/distribution/releases) - [Commits](https://github.com/docker/distribution/compare/v2.8.1...v2.8.2) --- updated-dependencies: - dependency-name: github.com/docker/distribution dependency-type: indirect ... Signed-off-by: dependabot[bot] Co-authored-by: dependabot[bot] <49699333+dependabot[bot]@users.noreply.github.com> Co-authored-by: Jordi van Liempt <35920075+jliempt@users.noreply.github.com> --- go.mod | 2 +- go.sum | 4 ++-- 2 files changed, 3 insertions(+), 3 deletions(-) diff --git a/go.mod b/go.mod index 50e48dc45f..382c52b565 100644 --- a/go.mod +++ b/go.mod @@ -140,7 +140,7 @@ require ( github.com/digitalocean/godo v1.7.5 // indirect github.com/dimchansky/utfbom v1.1.1 // indirect github.com/dnaeon/go-vcr v1.2.0 // indirect - github.com/docker/distribution v2.8.1+incompatible // indirect + github.com/docker/distribution v2.8.2+incompatible // indirect github.com/docker/docker v20.10.17+incompatible // indirect github.com/docker/docker-credential-helpers v0.6.4 // indirect github.com/docker/go-connections v0.4.0 // indirect diff --git a/go.sum b/go.sum index c7e6de3a09..95d5f98776 100644 --- a/go.sum +++ b/go.sum @@ -588,8 +588,8 @@ github.com/docker/cli v20.10.17+incompatible/go.mod h1:JLrzqnKDaYBop7H2jaqPtU4hH github.com/docker/distribution v0.0.0-20190905152932-14b96e55d84c/go.mod h1:0+TTO4EOBfRPhZXAeF1Vu+W3hHZ8eLp8PgKVZlcvtFY= github.com/docker/distribution v2.7.1-0.20190205005809-0d3efadf0154+incompatible/go.mod h1:J2gT2udsDAN96Uj4KfcMRqY0/ypR+oyYUYmja8H+y+w= github.com/docker/distribution v2.7.1+incompatible/go.mod h1:J2gT2udsDAN96Uj4KfcMRqY0/ypR+oyYUYmja8H+y+w= -github.com/docker/distribution v2.8.1+incompatible h1:Q50tZOPR6T/hjNsyc9g8/syEs6bk8XXApsHjKukMl68= -github.com/docker/distribution v2.8.1+incompatible/go.mod h1:J2gT2udsDAN96Uj4KfcMRqY0/ypR+oyYUYmja8H+y+w= +github.com/docker/distribution v2.8.2+incompatible h1:T3de5rq0dB1j30rp0sA2rER+m322EBzniBPB6ZIzuh8= +github.com/docker/distribution v2.8.2+incompatible/go.mod h1:J2gT2udsDAN96Uj4KfcMRqY0/ypR+oyYUYmja8H+y+w= github.com/docker/docker v1.4.2-0.20190924003213-a8608b5b67c7/go.mod h1:eEKB0N0r5NX/I1kEveEz05bcu8tLC/8azJZsviup8Sk= github.com/docker/docker v1.4.2-0.20200319182547-c7ad2b866182/go.mod h1:eEKB0N0r5NX/I1kEveEz05bcu8tLC/8azJZsviup8Sk= github.com/docker/docker v20.10.5+incompatible/go.mod h1:eEKB0N0r5NX/I1kEveEz05bcu8tLC/8azJZsviup8Sk= From 920c2480e704dfe95fed5822fc745d05e5b95bce Mon Sep 17 00:00:00 2001 From: "renovate[bot]" <29139614+renovate[bot]@users.noreply.github.com> Date: Tue, 4 Jul 2023 15:53:40 +0200 Subject: [PATCH 04/10] chore(deps): update actions/stale action to v8 (#4438) Co-authored-by: renovate[bot] <29139614+renovate[bot]@users.noreply.github.com> --- .github/workflows/stale.yml | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/.github/workflows/stale.yml b/.github/workflows/stale.yml index e092a54ef5..7f1c4c9324 100644 --- a/.github/workflows/stale.yml +++ b/.github/workflows/stale.yml @@ -8,7 +8,7 @@ jobs: runs-on: ubuntu-latest steps: - uses: styfle/cancel-workflow-action@0.10.0 - - uses: actions/stale@v4 + - uses: actions/stale@v8 with: repo-token: ${{ secrets.GITHUB_TOKEN }} stale-issue-message: 'Thank you for your contribution! This issue is stale because it has been open 60 days with no activity. In order to keep it open, please remove stale label or add a comment within the next 10 days. If you need a Piper team member to remove the stale label make sure to add `@SAP/jenkins-library-team` to your comment.' From fa11eb47aa842329df60b427b177ad3e23515767 Mon Sep 17 00:00:00 2001 From: "dependabot[bot]" <49699333+dependabot[bot]@users.noreply.github.com> Date: Tue, 4 Jul 2023 16:10:54 +0200 Subject: [PATCH 05/10] build(deps): bump github.com/opencontainers/runc from 1.1.2 to 1.1.5 (#4307) Bumps [github.com/opencontainers/runc](https://github.com/opencontainers/runc) from 1.1.2 to 1.1.5. - [Release notes](https://github.com/opencontainers/runc/releases) - [Changelog](https://github.com/opencontainers/runc/blob/v1.1.5/CHANGELOG.md) - [Commits](https://github.com/opencontainers/runc/compare/v1.1.2...v1.1.5) --- updated-dependencies: - dependency-name: github.com/opencontainers/runc dependency-type: indirect ... Signed-off-by: dependabot[bot] Co-authored-by: dependabot[bot] <49699333+dependabot[bot]@users.noreply.github.com> Co-authored-by: Jordi van Liempt <35920075+jliempt@users.noreply.github.com> --- go.mod | 2 +- go.sum | 6 +++--- 2 files changed, 4 insertions(+), 4 deletions(-) diff --git a/go.mod b/go.mod index 382c52b565..d0057e6bfd 100644 --- a/go.mod +++ b/go.mod @@ -263,7 +263,7 @@ require ( github.com/onsi/ginkgo v1.16.5 // indirect github.com/opencontainers/go-digest v1.0.0 // indirect github.com/opencontainers/image-spec v1.0.3-0.20220114050600-8b9d41f48198 // indirect - github.com/opencontainers/runc v1.1.2 // indirect + github.com/opencontainers/runc v1.1.5 // indirect github.com/opentracing/opentracing-go v1.2.0 // indirect github.com/oracle/oci-go-sdk v13.1.0+incompatible // indirect github.com/packethost/packngo v0.1.1-0.20180711074735-b9cb5096f54c // indirect diff --git a/go.sum b/go.sum index 95d5f98776..a0f987ce80 100644 --- a/go.sum +++ b/go.sum @@ -1664,8 +1664,8 @@ github.com/opencontainers/runc v1.0.0-rc8.0.20190926000215-3e425f80a8c9/go.mod h github.com/opencontainers/runc v1.0.0-rc9/go.mod h1:qT5XzbpPznkRYVz/mWwUaVBUv2rmF59PVA73FjuZG0U= github.com/opencontainers/runc v1.0.0-rc93/go.mod h1:3NOsor4w32B2tC0Zbl8Knk4Wg84SM2ImC1fxBuqJ/H0= github.com/opencontainers/runc v1.0.2/go.mod h1:aTaHFFwQXuA71CiyxOdFFIorAoemI04suvGRQFzWTD0= -github.com/opencontainers/runc v1.1.2 h1:2VSZwLx5k/BfsBxMMipG/LYUnmqOD/BPkIVgQUcTlLw= -github.com/opencontainers/runc v1.1.2/go.mod h1:Tj1hFw6eFWp/o33uxGf5yF2BX5yz2Z6iptFpuvbbKqc= +github.com/opencontainers/runc v1.1.5 h1:L44KXEpKmfWDcS02aeGm8QNTFXTo2D+8MYGDIJ/GDEs= +github.com/opencontainers/runc v1.1.5/go.mod h1:1J5XiS+vdZ3wCyZybsuxXZWGrgSr8fFJHLXuG2PsnNg= github.com/opencontainers/runtime-spec v0.1.2-0.20190507144316-5b71a03e2700/go.mod h1:jwyrGlmzljRJv/Fgzds9SsS/C5hL+LL3ko9hs6T5lQ0= github.com/opencontainers/runtime-spec v1.0.1/go.mod h1:jwyrGlmzljRJv/Fgzds9SsS/C5hL+LL3ko9hs6T5lQ0= github.com/opencontainers/runtime-spec v1.0.2-0.20190207185410-29686dbc5559/go.mod h1:jwyrGlmzljRJv/Fgzds9SsS/C5hL+LL3ko9hs6T5lQ0= @@ -1846,7 +1846,7 @@ github.com/sean-/pager v0.0.0-20180208200047-666be9bf53b5/go.mod h1:BeybITEsBEg6 github.com/sean-/seed v0.0.0-20170313163322-e2103e2c3529 h1:nn5Wsu0esKSJiIVhscUtVbo7ada43DJhG55ua/hjS5I= github.com/sean-/seed v0.0.0-20170313163322-e2103e2c3529/go.mod h1:DxrIzT+xaE7yg65j358z/aeFdxmN0P9QXhEzd20vsDc= github.com/seccomp/libseccomp-golang v0.9.1/go.mod h1:GbW5+tmTXfcxTToHLXlScSlAvWlF4P2Ca7zGrPiEpWo= -github.com/seccomp/libseccomp-golang v0.9.2-0.20210429002308-3879420cc921/go.mod h1:JA8cRccbGaA1s33RQf7Y1+q9gHmZX1yB/z9WDN1C6fg= +github.com/seccomp/libseccomp-golang v0.9.2-0.20220502022130-f33da4d89646/go.mod h1:JA8cRccbGaA1s33RQf7Y1+q9gHmZX1yB/z9WDN1C6fg= github.com/securego/gosec/v2 v2.9.1/go.mod h1:oDcDLcatOJxkCGaCaq8lua1jTnYf6Sou4wdiJ1n4iHc= github.com/sergi/go-diff v1.0.0/go.mod h1:0CfEIISq7TuYL3j771MWULgwwjU+GofnZX9QAmXWZgo= github.com/sergi/go-diff v1.1.0/go.mod h1:STckp+ISIX8hZLjrqAeVduY0gWCT9IjLuqbuNXdaHfM= From 3e0da62a0b21ac0949fe473926e1ecd86cc3b9fa Mon Sep 17 00:00:00 2001 From: "renovate[bot]" <29139614+renovate[bot]@users.noreply.github.com> Date: Tue, 4 Jul 2023 17:34:16 +0200 Subject: [PATCH 06/10] chore(deps): update actions/checkout action to v3 (#4440) Co-authored-by: renovate[bot] <29139614+renovate[bot]@users.noreply.github.com> --- .github/workflows/documentation.yml | 2 +- .github/workflows/markdown.yml | 2 +- .github/workflows/release-go.yml | 2 +- .github/workflows/update-go-dependencies.yml | 2 +- .github/workflows/upload-go-master.yml | 2 +- .github/workflows/verify-go.yml | 10 +++++----- .github/workflows/verify-groovy.yml | 2 +- 7 files changed, 11 insertions(+), 11 deletions(-) diff --git a/.github/workflows/documentation.yml b/.github/workflows/documentation.yml index 1f511785b9..9ca4be3d20 100644 --- a/.github/workflows/documentation.yml +++ b/.github/workflows/documentation.yml @@ -14,7 +14,7 @@ jobs: steps: - uses: styfle/cancel-workflow-action@0.10.0 - - uses: actions/checkout@v2 + - uses: actions/checkout@v3 - uses: actions/setup-java@v1 with: diff --git a/.github/workflows/markdown.yml b/.github/workflows/markdown.yml index 61c1e3d9fa..bb049f6ea5 100644 --- a/.github/workflows/markdown.yml +++ b/.github/workflows/markdown.yml @@ -16,7 +16,7 @@ jobs: name: 'Format' steps: - uses: styfle/cancel-workflow-action@0.10.0 - - uses: actions/checkout@v2 + - uses: actions/checkout@v3 - name: Markdown Linting uses: nosborn/github-action-markdown-cli@v1.1.1 with: diff --git a/.github/workflows/release-go.yml b/.github/workflows/release-go.yml index 4bd06d3bd5..0ba39cd3ce 100644 --- a/.github/workflows/release-go.yml +++ b/.github/workflows/release-go.yml @@ -12,7 +12,7 @@ jobs: runs-on: ubuntu-latest steps: - uses: styfle/cancel-workflow-action@0.10.0 - - uses: actions/checkout@v1 + - uses: actions/checkout@v3 # Workaround for https://github.com/SAP/jenkins-library/issues/1723, build only works with jdk8 currently - uses: actions/setup-java@v1 with: diff --git a/.github/workflows/update-go-dependencies.yml b/.github/workflows/update-go-dependencies.yml index 8618c1fb77..6ba075ac96 100644 --- a/.github/workflows/update-go-dependencies.yml +++ b/.github/workflows/update-go-dependencies.yml @@ -10,7 +10,7 @@ jobs: runs-on: ubuntu-latest steps: - uses: styfle/cancel-workflow-action@0.10.0 - - uses: actions/checkout@v2 + - uses: actions/checkout@v3 - uses: actions/setup-go@v1 with: go-version: '1.18.x' diff --git a/.github/workflows/upload-go-master.yml b/.github/workflows/upload-go-master.yml index 7eaff4e109..cbacd19448 100644 --- a/.github/workflows/upload-go-master.yml +++ b/.github/workflows/upload-go-master.yml @@ -10,7 +10,7 @@ jobs: runs-on: ubuntu-latest steps: - uses: styfle/cancel-workflow-action@0.10.0 - - uses: actions/checkout@v1 + - uses: actions/checkout@v3 - uses: actions/setup-go@v1 with: go-version: '1.18.x' diff --git a/.github/workflows/verify-go.yml b/.github/workflows/verify-go.yml index 469a2489d7..2dbe8e8588 100644 --- a/.github/workflows/verify-go.yml +++ b/.github/workflows/verify-go.yml @@ -24,7 +24,7 @@ jobs: restore-keys: | ${{ runner.os }}-golang- - name: checkout - uses: actions/checkout@v2 + uses: actions/checkout@v3 - name: unit-test uses: paambaati/codeclimate-action@v4 env: @@ -49,7 +49,7 @@ jobs: ${{ runner.os }}-golang-format ${{ runner.os }}-golang- - name: checkout - uses: actions/checkout@v2 + uses: actions/checkout@v3 - name: format run: go fmt ./... - name: verify @@ -62,7 +62,7 @@ jobs: go-version: '1.19.x' # action requires go@1.19 - name: checkout - uses: actions/checkout@v2 + uses: actions/checkout@v3 with: fetch-depth: 1 - name: staticcheck @@ -85,7 +85,7 @@ jobs: ${{ runner.os }}-golang-generate ${{ runner.os }}-golang- - name: checkout - uses: actions/checkout@v2 + uses: actions/checkout@v3 - name: generate run: go run pkg/generator/step-metadata.go - name: verify @@ -105,7 +105,7 @@ jobs: ${{ runner.os }}-golang-dependencies ${{ runner.os }}-golang- - name: checkout - uses: actions/checkout@v2 + uses: actions/checkout@v3 - name: cleanup dependencies run: go mod tidy - name: verify diff --git a/.github/workflows/verify-groovy.yml b/.github/workflows/verify-groovy.yml index 45d28c9fc9..9f4dbab913 100644 --- a/.github/workflows/verify-groovy.yml +++ b/.github/workflows/verify-groovy.yml @@ -14,7 +14,7 @@ jobs: steps: - uses: styfle/cancel-workflow-action@0.10.0 - - uses: actions/checkout@v2 + - uses: actions/checkout@v3 - uses: actions/setup-java@v1 with: From 380144b457a10be0d368c16e7627bb9ade4c0b13 Mon Sep 17 00:00:00 2001 From: "renovate[bot]" <29139614+renovate[bot]@users.noreply.github.com> Date: Wed, 5 Jul 2023 10:14:28 +0200 Subject: [PATCH 07/10] chore(deps): update actions/setup-go action to v4 (#4435) Co-authored-by: renovate[bot] <29139614+renovate[bot]@users.noreply.github.com> --- .github/workflows/documentation.yml | 2 +- .github/workflows/integration-tests-pr.yml | 6 +++--- .github/workflows/integration-tests.yml | 6 +++--- .github/workflows/update-go-dependencies.yml | 2 +- .github/workflows/upload-go-master.yml | 2 +- .github/workflows/verify-go.yml | 10 +++++----- 6 files changed, 14 insertions(+), 14 deletions(-) diff --git a/.github/workflows/documentation.yml b/.github/workflows/documentation.yml index 9ca4be3d20..b24a3a754e 100644 --- a/.github/workflows/documentation.yml +++ b/.github/workflows/documentation.yml @@ -20,7 +20,7 @@ jobs: with: java-version: '1.8' - - uses: actions/setup-go@v1 + - uses: actions/setup-go@v4 with: go-version: '1.18.x' diff --git a/.github/workflows/integration-tests-pr.yml b/.github/workflows/integration-tests-pr.yml index a50b696bb3..1aaad2d2a5 100644 --- a/.github/workflows/integration-tests-pr.yml +++ b/.github/workflows/integration-tests-pr.yml @@ -74,7 +74,7 @@ jobs: - uses: actions/checkout@v3 with: ref: ${{ needs.start.outputs.sha }} - - uses: actions/setup-go@v3 + - uses: actions/setup-go@v4 with: go-version: ${{ needs.start.outputs.go_version }} - name: Build @@ -98,7 +98,7 @@ jobs: - uses: actions/checkout@v3 with: ref: ${{ needs.start.outputs.sha }} - - uses: actions/setup-go@v3 + - uses: actions/setup-go@v4 with: go-version: ${{ needs.start.outputs.go_version }} - name: Build @@ -130,7 +130,7 @@ jobs: - uses: actions/checkout@v3 with: ref: ${{ needs.start.outputs.sha }} - - uses: actions/setup-go@v3 + - uses: actions/setup-go@v4 with: go-version: ${{ needs.start.outputs.go_version }} - name: Download Piper binary diff --git a/.github/workflows/integration-tests.yml b/.github/workflows/integration-tests.yml index fc362bf370..12b7e3cafe 100644 --- a/.github/workflows/integration-tests.yml +++ b/.github/workflows/integration-tests.yml @@ -45,7 +45,7 @@ jobs: - uses: actions/checkout@v3 with: ref: ${{ needs.start.outputs.sha }} - - uses: actions/setup-go@v3 + - uses: actions/setup-go@v4 with: go-version: ${{ needs.start.outputs.go_version }} - name: Build @@ -70,7 +70,7 @@ jobs: - uses: actions/checkout@v3 with: ref: ${{ needs.start.outputs.sha }} - - uses: actions/setup-go@v3 + - uses: actions/setup-go@v4 with: go-version: ${{ needs.start.outputs.go_version }} - name: Build @@ -102,7 +102,7 @@ jobs: - uses: actions/checkout@v3 with: ref: ${{ needs.start.outputs.sha }} - - uses: actions/setup-go@v3 + - uses: actions/setup-go@v4 with: go-version: ${{ needs.start.outputs.go_version }} - name: Download Piper binary diff --git a/.github/workflows/update-go-dependencies.yml b/.github/workflows/update-go-dependencies.yml index 6ba075ac96..b79ab238d4 100644 --- a/.github/workflows/update-go-dependencies.yml +++ b/.github/workflows/update-go-dependencies.yml @@ -11,7 +11,7 @@ jobs: steps: - uses: styfle/cancel-workflow-action@0.10.0 - uses: actions/checkout@v3 - - uses: actions/setup-go@v1 + - uses: actions/setup-go@v4 with: go-version: '1.18.x' - name: Perform update diff --git a/.github/workflows/upload-go-master.yml b/.github/workflows/upload-go-master.yml index cbacd19448..f910b16440 100644 --- a/.github/workflows/upload-go-master.yml +++ b/.github/workflows/upload-go-master.yml @@ -11,7 +11,7 @@ jobs: steps: - uses: styfle/cancel-workflow-action@0.10.0 - uses: actions/checkout@v3 - - uses: actions/setup-go@v1 + - uses: actions/setup-go@v4 with: go-version: '1.18.x' - env: diff --git a/.github/workflows/verify-go.yml b/.github/workflows/verify-go.yml index 2dbe8e8588..c5e3d19042 100644 --- a/.github/workflows/verify-go.yml +++ b/.github/workflows/verify-go.yml @@ -13,7 +13,7 @@ jobs: runs-on: ubuntu-latest steps: - uses: styfle/cancel-workflow-action@0.10.0 - - uses: actions/setup-go@v1 + - uses: actions/setup-go@v4 with: go-version: '1.18.x' - name: Cache Golang Packages @@ -37,7 +37,7 @@ jobs: format: runs-on: ubuntu-latest steps: - - uses: actions/setup-go@v1 + - uses: actions/setup-go@v4 with: go-version: '1.18.x' - name: Cache Golang Packages @@ -57,7 +57,7 @@ jobs: lint: runs-on: ubuntu-latest steps: - - uses: actions/setup-go@v1 + - uses: actions/setup-go@v4 with: go-version: '1.19.x' # action requires go@1.19 @@ -73,7 +73,7 @@ jobs: generate: runs-on: ubuntu-latest steps: - - uses: actions/setup-go@v1 + - uses: actions/setup-go@v4 with: go-version: '1.18.x' - name: Cache Golang Packages @@ -93,7 +93,7 @@ jobs: dependencies: runs-on: ubuntu-latest steps: - - uses: actions/setup-go@v1 + - uses: actions/setup-go@v4 with: go-version: '1.18.x' - name: Cache Golang Packages From 9b1aebfd13314f5ed843c5d7cfd762d954aefb6a Mon Sep 17 00:00:00 2001 From: "renovate[bot]" <29139614+renovate[bot]@users.noreply.github.com> Date: Wed, 5 Jul 2023 11:35:34 +0200 Subject: [PATCH 08/10] chore(deps): update actions/setup-java action to v3 (#4436) * chore(deps): update actions/setup-java action to v3 * Apply suggestions from code review --------- Co-authored-by: renovate[bot] <29139614+renovate[bot]@users.noreply.github.com> Co-authored-by: Christopher Fenner <26137398+CCFenner@users.noreply.github.com> --- .github/workflows/consumer-tests-pr.yml | 3 ++- .github/workflows/consumer-tests.yml | 3 ++- .github/workflows/documentation.yml | 5 +++-- .github/workflows/release-go.yml | 5 +++-- .github/workflows/verify-groovy.yml | 5 +++-- 5 files changed, 13 insertions(+), 8 deletions(-) diff --git a/.github/workflows/consumer-tests-pr.yml b/.github/workflows/consumer-tests-pr.yml index 7b14193a72..8700cbbd81 100644 --- a/.github/workflows/consumer-tests-pr.yml +++ b/.github/workflows/consumer-tests-pr.yml @@ -44,9 +44,10 @@ jobs: with: repository: ${{ steps.repository.outputs.repository }} ref: ${{ steps.branch_name.outputs.branch_name }} - - uses: actions/setup-java@v1 + - uses: actions/setup-java@v3 with: java-version: 11 + distribution: zulu - name: Install Groovy run: | sudo apt-get update diff --git a/.github/workflows/consumer-tests.yml b/.github/workflows/consumer-tests.yml index 35e1b0121d..5d7be028f8 100644 --- a/.github/workflows/consumer-tests.yml +++ b/.github/workflows/consumer-tests.yml @@ -14,9 +14,10 @@ jobs: steps: - uses: styfle/cancel-workflow-action@0.10.0 - uses: actions/checkout@v3 - - uses: actions/setup-java@v1 + - uses: actions/setup-java@v3 with: java-version: 11 + distribution: zulu - name: Install Groovy run: | sudo apt-get update diff --git a/.github/workflows/documentation.yml b/.github/workflows/documentation.yml index b24a3a754e..8a46e53ad5 100644 --- a/.github/workflows/documentation.yml +++ b/.github/workflows/documentation.yml @@ -16,9 +16,10 @@ jobs: - uses: actions/checkout@v3 - - uses: actions/setup-java@v1 + - uses: actions/setup-java@v3 with: - java-version: '1.8' + java-version: 8 + distribution: zulu - uses: actions/setup-go@v4 with: diff --git a/.github/workflows/release-go.yml b/.github/workflows/release-go.yml index 0ba39cd3ce..ba4f5a6802 100644 --- a/.github/workflows/release-go.yml +++ b/.github/workflows/release-go.yml @@ -14,9 +14,10 @@ jobs: - uses: styfle/cancel-workflow-action@0.10.0 - uses: actions/checkout@v3 # Workaround for https://github.com/SAP/jenkins-library/issues/1723, build only works with jdk8 currently - - uses: actions/setup-java@v1 + - uses: actions/setup-java@v3 with: - java-version: '8' + java-version: 8 + distribution: zulu - name: Prepare Release run: | curl --insecure --silent --location --write-out '%{http_code}' --output ./piper_master https://github.com/SAP/jenkins-library/releases/latest/download/piper_master diff --git a/.github/workflows/verify-groovy.yml b/.github/workflows/verify-groovy.yml index 9f4dbab913..ea017bd342 100644 --- a/.github/workflows/verify-groovy.yml +++ b/.github/workflows/verify-groovy.yml @@ -16,9 +16,10 @@ jobs: - uses: actions/checkout@v3 - - uses: actions/setup-java@v1 + - uses: actions/setup-java@v3 with: - java-version: 1.8 + java-version: 8 + distribution: zulu - name: Cache Maven Packages uses: actions/cache@v1 From b9bb5265ca1b240ec080596e97c538795239c108 Mon Sep 17 00:00:00 2001 From: "renovate[bot]" <29139614+renovate[bot]@users.noreply.github.com> Date: Wed, 5 Jul 2023 12:08:14 +0200 Subject: [PATCH 09/10] chore(deps): update actions/cache action to v3 (#4439) Co-authored-by: renovate[bot] <29139614+renovate[bot]@users.noreply.github.com> --- .github/workflows/documentation.yml | 4 ++-- .github/workflows/verify-go.yml | 8 ++++---- .github/workflows/verify-groovy.yml | 2 +- 3 files changed, 7 insertions(+), 7 deletions(-) diff --git a/.github/workflows/documentation.yml b/.github/workflows/documentation.yml index 8a46e53ad5..ffd80b2667 100644 --- a/.github/workflows/documentation.yml +++ b/.github/workflows/documentation.yml @@ -29,7 +29,7 @@ jobs: run: sudo apt-get update && sudo apt-get install groovy -y - name: Cache Maven Packages - uses: actions/cache@v1 + uses: actions/cache@v3 with: path: ~/.m2/repository key: ${{ runner.os }}-maven-${{ hashFiles('**/pom.xml') }} @@ -37,7 +37,7 @@ jobs: ${{ runner.os }}-maven- - name: Cache Go Packages - uses: actions/cache@v1 + uses: actions/cache@v3 with: path: ~/go/pkg/mod key: ${{ runner.os }}-go-${{ hashFiles('**/go.sum') }} diff --git a/.github/workflows/verify-go.yml b/.github/workflows/verify-go.yml index c5e3d19042..9e0d1d7e85 100644 --- a/.github/workflows/verify-go.yml +++ b/.github/workflows/verify-go.yml @@ -17,7 +17,7 @@ jobs: with: go-version: '1.18.x' - name: Cache Golang Packages - uses: actions/cache@v1 + uses: actions/cache@v3 with: path: ~/go/pkg/mod key: ${{ runner.os }}-golang-${{ hashFiles('go.sum') }} @@ -41,7 +41,7 @@ jobs: with: go-version: '1.18.x' - name: Cache Golang Packages - uses: actions/cache@v1 + uses: actions/cache@v3 with: path: ~/go/pkg/mod key: ${{ runner.os }}-golang-format${{ hashFiles('go.sum') }} @@ -77,7 +77,7 @@ jobs: with: go-version: '1.18.x' - name: Cache Golang Packages - uses: actions/cache@v1 + uses: actions/cache@v3 with: path: ~/go/pkg/mod key: ${{ runner.os }}-golang-generate${{ hashFiles('go.sum') }} @@ -97,7 +97,7 @@ jobs: with: go-version: '1.18.x' - name: Cache Golang Packages - uses: actions/cache@v1 + uses: actions/cache@v3 with: path: ~/go/pkg/mod key: ${{ runner.os }}-golang-dependencies${{ hashFiles('go.sum') }} diff --git a/.github/workflows/verify-groovy.yml b/.github/workflows/verify-groovy.yml index ea017bd342..ca64a9c968 100644 --- a/.github/workflows/verify-groovy.yml +++ b/.github/workflows/verify-groovy.yml @@ -22,7 +22,7 @@ jobs: distribution: zulu - name: Cache Maven Packages - uses: actions/cache@v1 + uses: actions/cache@v3 with: path: ~/.m2/repository key: ${{ runner.os }}-maven-${{ hashFiles('**/pom.xml') }} From 610e21230677cb91f1a8056de392b13cd913c1bc Mon Sep 17 00:00:00 2001 From: Pavel Busko Date: Thu, 6 Jul 2023 11:34:05 +0200 Subject: [PATCH 10/10] feat(cnbBuild) Add support for pre and post-buildpacks (#4448) * Add pre and post buildpacks Co-authored-by: Johannes Dillmann Co-authored-by: Ralf Pannemans Co-authored-by: Pavel Busko * fix integration tests Co-authored-by: Pavel Busko Co-authored-by: Ralf Pannemans * simplify if clauses Co-authored-by: Pavel Busko --------- Co-authored-by: Johannes Dillmann Co-authored-by: Ralf Pannemans --- cmd/cnbBuild.go | 41 ++++- cmd/cnbBuild_generated.go | 34 +++- cmd/cnbBuild_test.go | 124 ++++++++++++-- go.mod | 3 +- go.sum | 2 - .../github_actions_integration_test_list.yml | 1 + integration/integration_cnb_test.go | 32 +++- .../testdata/TestCnbIntegration/config.yml | 3 + .../TestCnbIntegration/project/package.json | 1 + pkg/cnbutils/buildpack.go | 89 +++++++--- pkg/cnbutils/buildpack_test.go | 50 +++++- pkg/cnbutils/order.go | 76 ++++++++- pkg/cnbutils/order_test.go | 156 +++++++++++++++++- pkg/cnbutils/project/descriptor.go | 102 ++++++------ pkg/cnbutils/project/descriptor_test.go | 100 ++++++++++- pkg/cnbutils/project/metadata/metadata.go | 2 +- .../project/metadata/metadata_test.go | 9 +- pkg/cnbutils/project/types/types.go | 58 +++++++ pkg/cnbutils/project/v01/project.go | 30 ++++ pkg/cnbutils/project/v02/project.go | 78 +++++++++ pkg/cnbutils/report.go | 2 +- pkg/cnbutils/report_test.go | 2 +- pkg/mock/dockerClient.go | 12 +- resources/metadata/cnbBuild.yaml | 22 ++- 24 files changed, 913 insertions(+), 116 deletions(-) create mode 100644 pkg/cnbutils/project/types/types.go create mode 100644 pkg/cnbutils/project/v01/project.go create mode 100644 pkg/cnbutils/project/v02/project.go diff --git a/cmd/cnbBuild.go b/cmd/cnbBuild.go index 4ef75a4c07..9c80aa8ab9 100644 --- a/cmd/cnbBuild.go +++ b/cmd/cnbBuild.go @@ -112,10 +112,29 @@ func processConfigs(main cnbBuildOptions, multipleImages []map[string]interface{ return result, nil } -func setCustomBuildpacks(bpacks []string, dockerCreds string, utils cnbutils.BuildUtils) (string, string, error) { +func setCustomBuildpacks(bpacks, preBuildpacks, postBuildpacks []string, dockerCreds string, utils cnbutils.BuildUtils) (string, string, error) { buildpacksPath := "/tmp/buildpacks" orderPath := "/tmp/buildpacks/order.toml" - newOrder, err := cnbutils.DownloadBuildpacks(buildpacksPath, bpacks, dockerCreds, utils) + err := cnbutils.DownloadBuildpacks(buildpacksPath, append(bpacks, append(preBuildpacks, postBuildpacks...)...), dockerCreds, utils) + if err != nil { + return "", "", err + } + + if len(bpacks) == 0 && (len(postBuildpacks) > 0 || len(preBuildpacks) > 0) { + matches, err := utils.Glob("/cnb/buildpacks/*") + if err != nil { + return "", "", err + } + + for _, match := range matches { + err = cnbutils.CreateVersionSymlinks(buildpacksPath, match, utils) + if err != nil { + return "", "", err + } + } + } + + newOrder, err := cnbutils.CreateOrder(bpacks, preBuildpacks, postBuildpacks, dockerCreds, utils) if err != nil { return "", "", err } @@ -475,10 +494,18 @@ func runCnbBuild(config *cnbBuildOptions, cnbTelemetry *cnbBuildTelemetry, utils config.mergeEnvVars(descriptor.EnvVars) - if (config.Buildpacks == nil || len(config.Buildpacks) == 0) && len(descriptor.Buildpacks) > 0 { + if len(config.Buildpacks) == 0 { config.Buildpacks = descriptor.Buildpacks } + if len(config.PreBuildpacks) == 0 { + config.PreBuildpacks = descriptor.PreBuildpacks + } + + if len(config.PostBuildpacks) == 0 { + config.PostBuildpacks = descriptor.PostBuildpacks + } + if descriptor.Exclude != nil { exclude = descriptor.Exclude } @@ -563,11 +590,13 @@ func runCnbBuild(config *cnbBuildOptions, cnbTelemetry *cnbBuildTelemetry, utils metadata.WriteProjectMetadata(GeneralConfig.EnvRootPath, utils) var buildpacksPath = "/cnb/buildpacks" - var orderPath = "/cnb/order.toml" + var orderPath = cnbutils.DefaultOrderPath - if config.Buildpacks != nil && len(config.Buildpacks) > 0 { + if len(config.Buildpacks) > 0 || len(config.PreBuildpacks) > 0 || len(config.PostBuildpacks) > 0 { log.Entry().Infof("Setting custom buildpacks: '%v'", config.Buildpacks) - buildpacksPath, orderPath, err = setCustomBuildpacks(config.Buildpacks, config.DockerConfigJSON, utils) + log.Entry().Infof("Pre-buildpacks: '%v'", config.PreBuildpacks) + log.Entry().Infof("Post-buildpacks: '%v'", config.PostBuildpacks) + buildpacksPath, orderPath, err = setCustomBuildpacks(config.Buildpacks, config.PreBuildpacks, config.PostBuildpacks, config.DockerConfigJSON, utils) defer func() { _ = utils.RemoveAll(buildpacksPath) }() defer func() { _ = utils.RemoveAll(orderPath) }() if err != nil { diff --git a/cmd/cnbBuild_generated.go b/cmd/cnbBuild_generated.go index 945daa6600..0a150d1607 100644 --- a/cmd/cnbBuild_generated.go +++ b/cmd/cnbBuild_generated.go @@ -27,6 +27,8 @@ type cnbBuildOptions struct { ContainerImageTag string `json:"containerImageTag,omitempty"` ContainerRegistryURL string `json:"containerRegistryUrl,omitempty"` Buildpacks []string `json:"buildpacks,omitempty"` + PreBuildpacks []string `json:"preBuildpacks,omitempty"` + PostBuildpacks []string `json:"postBuildpacks,omitempty"` BuildEnvVars map[string]interface{} `json:"buildEnvVars,omitempty"` Path string `json:"path,omitempty"` ProjectDescriptor string `json:"projectDescriptor,omitempty"` @@ -226,7 +228,9 @@ func addCnbBuildFlags(cmd *cobra.Command, stepConfig *cnbBuildOptions) { cmd.Flags().StringVar(&stepConfig.ContainerImageAlias, "containerImageAlias", os.Getenv("PIPER_containerImageAlias"), "Logical name used for this image.\n") cmd.Flags().StringVar(&stepConfig.ContainerImageTag, "containerImageTag", os.Getenv("PIPER_containerImageTag"), "Tag of the container which will be built") cmd.Flags().StringVar(&stepConfig.ContainerRegistryURL, "containerRegistryUrl", os.Getenv("PIPER_containerRegistryUrl"), "Container registry where the image should be pushed to.\n\n**Note**: `containerRegistryUrl` should include only the domain. If you want to publish an image under `docker.io/example/my-image`, you must set `containerRegistryUrl: \"docker.io\"` and `containerImageName: \"example/my-image\"`.\n") - cmd.Flags().StringSliceVar(&stepConfig.Buildpacks, "buildpacks", []string{}, "List of custom buildpacks to use in the form of `$HOSTNAME/$REPO[:$TAG]`.") + cmd.Flags().StringSliceVar(&stepConfig.Buildpacks, "buildpacks", []string{}, "List of custom buildpacks to use in the form of `$HOSTNAME/$REPO[:$TAG]`. When this property is specified, buildpacks which are part of the builder will be ignored.") + cmd.Flags().StringSliceVar(&stepConfig.PreBuildpacks, "preBuildpacks", []string{}, "Buildpacks to prepend to the groups in the builder's order.") + cmd.Flags().StringSliceVar(&stepConfig.PostBuildpacks, "postBuildpacks", []string{}, "Buildpacks to append to the groups in the builder's order.") cmd.Flags().StringVar(&stepConfig.Path, "path", os.Getenv("PIPER_path"), "Glob that should either point to a directory with your sources or one artifact in zip format.\nThis property determines the input to the buildpack.\n") cmd.Flags().StringVar(&stepConfig.ProjectDescriptor, "projectDescriptor", `project.toml`, "Relative path to the project.toml file.\nSee [buildpacks.io](https://buildpacks.io/docs/reference/config/project-descriptor/) for the reference.\nParameters passed to the cnbBuild step will take precedence over the parameters set in the project.toml file, except the `env` block.\nEnvironment variables declared in a project descriptor file, will be merged with the `buildEnvVars` property, with the `buildEnvVars` having a precedence.\n\n*Note*: The project descriptor path should be relative to what is set in the [path](#path) property. If the `path` property is pointing to a zip archive (e.g. jar file), project descriptor path will be relative to the root of the workspace.\n\n*Note*: Inline buildpacks (see [specification](https://buildpacks.io/docs/reference/config/project-descriptor/#build-_table-optional_)) are not supported yet.\n") @@ -325,6 +329,34 @@ func cnbBuildMetadata() config.StepData { Aliases: []config.Alias{}, Default: []string{}, }, + { + Name: "preBuildpacks", + ResourceRef: []config.ResourceReference{ + { + Name: "commonPipelineEnvironment", + Param: "container/preBuildpacks", + }, + }, + Scope: []string{"PARAMETERS", "STAGES", "STEPS"}, + Type: "[]string", + Mandatory: false, + Aliases: []config.Alias{}, + Default: []string{}, + }, + { + Name: "postBuildpacks", + ResourceRef: []config.ResourceReference{ + { + Name: "commonPipelineEnvironment", + Param: "container/postBuildpacks", + }, + }, + Scope: []string{"PARAMETERS", "STAGES", "STEPS"}, + Type: "[]string", + Mandatory: false, + Aliases: []config.Alias{}, + Default: []string{}, + }, { Name: "buildEnvVars", ResourceRef: []config.ResourceReference{}, diff --git a/cmd/cnbBuild_test.go b/cmd/cnbBuild_test.go index 2f1fa8226c..f5c7592a70 100644 --- a/cmd/cnbBuild_test.go +++ b/cmd/cnbBuild_test.go @@ -25,23 +25,56 @@ import ( const imageRegistry = "some-registry" func newCnbBuildTestsUtils() cnbutils.MockUtils { + imageStub := func(imageRef, target string) (v1.Image, error) { + fakeImage := &fake.FakeImage{} + var imageConfig v1.Config + switch imageRef { + case "pre-test": + imageConfig = v1.Config{ + Labels: map[string]string{ + "io.buildpacks.buildpackage.metadata": "{\"id\": \"pre-testbuildpack\", \"version\": \"0.0.1\"}", + }, + } + case "post-test": + imageConfig = v1.Config{ + Labels: map[string]string{ + "io.buildpacks.buildpackage.metadata": "{\"id\": \"post-testbuildpack\", \"version\": \"0.0.1\"}", + }, + } + default: + imageConfig = v1.Config{ + Labels: map[string]string{ + "io.buildpacks.buildpackage.metadata": "{\"id\": \"testbuildpack\", \"version\": \"0.0.1\"}", + }, + } + } + + fakeImage.ConfigFileReturns(&v1.ConfigFile{ + Config: imageConfig, + }, nil) + + return fakeImage, nil + } + utils := cnbutils.MockUtils{ ExecMockRunner: &mock.ExecMockRunner{}, FilesMock: &mock.FilesMock{}, - DownloadMock: &mock.DownloadMock{}, - } - - fakeImage := &fake.FakeImage{} - fakeImage.ConfigFileReturns(&v1.ConfigFile{ - Config: v1.Config{ - Labels: map[string]string{ - "io.buildpacks.buildpackage.metadata": "{\"id\": \"testbuildpack\", \"version\": \"0.0.1\"}", + DownloadMock: &mock.DownloadMock{ + ImageContentStub: imageStub, + ImageInfoStub: func(imageRef string) (v1.Image, error) { + return imageStub(imageRef, "") }, }, - }, nil) + } - utils.RemoteImageInfo = fakeImage - utils.ReturnImage = fakeImage + utils.AddFile("/cnb/order.toml", []byte(`[[order]] + [[order.group]] + id = "buildpacks/java" + version = "1.8.0" +[[order]] + [[order.group]] + id = "buildpacks/nodejs" + version = "1.6.0"`)) utils.AddFile("/layers/report.toml", []byte(`[build] [image] tags = ["localhost:5000/not-found:0.0.1"] @@ -234,6 +267,75 @@ func TestRunCnbBuild(t *testing.T) { assert.True(t, copiedFileExists) }) + t.Run("success case (custom buildpacks, pre and post buildpacks and custom env variables, renaming docker conf file, additional tag)", func(t *testing.T) { + t.Parallel() + config := cnbBuildOptions{ + ContainerImageName: "my-image", + ContainerImageTag: "0.0.1", + ContainerRegistryURL: imageRegistry, + DockerConfigJSON: "/path/to/test.json", + PreBuildpacks: []string{"pre-test"}, + PostBuildpacks: []string{"post-test"}, + Buildpacks: []string{"test"}, + BuildEnvVars: map[string]interface{}{ + "FOO": "BAR", + }, + AdditionalTags: []string{"latest"}, + } + + utils := newCnbBuildTestsUtils() + utils.FilesMock.AddFile(config.DockerConfigJSON, []byte(`{"auths":{"my-registry":{"auth":"dXNlcjpwYXNz"}}}`)) + addBuilderFiles(&utils) + + err := callCnbBuild(&config, &telemetry.CustomData{}, &utils, &cnbBuildCommonPipelineEnvironment{}, &piperhttp.Client{}) + + require.NoError(t, err) + runner := utils.ExecMockRunner + assert.Contains(t, runner.Env, "CNB_REGISTRY_AUTH={\"my-registry\":\"Basic dXNlcjpwYXNz\"}") + assert.Equal(t, creatorPath, runner.Calls[0].Exec) + assert.Contains(t, runner.Calls[0].Params, "/tmp/buildpacks") + assert.Contains(t, runner.Calls[0].Params, "/tmp/buildpacks/order.toml") + assert.Contains(t, runner.Calls[0].Params, fmt.Sprintf("%s/%s:%s", config.ContainerRegistryURL, config.ContainerImageName, config.ContainerImageTag)) + assert.Contains(t, runner.Calls[0].Params, fmt.Sprintf("%s/%s:latest", config.ContainerRegistryURL, config.ContainerImageName)) + + copiedFileExists, _ := utils.FileExists("/tmp/config.json") + assert.True(t, copiedFileExists) + }) + + t.Run("success case (custom pre and post buildpacks and custom env variables, renaming docker conf file, additional tag)", func(t *testing.T) { + t.Parallel() + config := cnbBuildOptions{ + ContainerImageName: "my-image", + ContainerImageTag: "0.0.1", + ContainerRegistryURL: imageRegistry, + DockerConfigJSON: "/path/to/test.json", + PostBuildpacks: []string{"post-test"}, + PreBuildpacks: []string{"pre-test"}, + BuildEnvVars: map[string]interface{}{ + "FOO": "BAR", + }, + AdditionalTags: []string{"latest"}, + } + + utils := newCnbBuildTestsUtils() + utils.FilesMock.AddFile(config.DockerConfigJSON, []byte(`{"auths":{"my-registry":{"auth":"dXNlcjpwYXNz"}}}`)) + addBuilderFiles(&utils) + + err := callCnbBuild(&config, &telemetry.CustomData{}, &utils, &cnbBuildCommonPipelineEnvironment{}, &piperhttp.Client{}) + + require.NoError(t, err) + runner := utils.ExecMockRunner + assert.Contains(t, runner.Env, "CNB_REGISTRY_AUTH={\"my-registry\":\"Basic dXNlcjpwYXNz\"}") + assert.Equal(t, creatorPath, runner.Calls[0].Exec) + assert.Contains(t, runner.Calls[0].Params, "/tmp/buildpacks") + assert.Contains(t, runner.Calls[0].Params, "/tmp/buildpacks/order.toml") + assert.Contains(t, runner.Calls[0].Params, fmt.Sprintf("%s/%s:%s", config.ContainerRegistryURL, config.ContainerImageName, config.ContainerImageTag)) + assert.Contains(t, runner.Calls[0].Params, fmt.Sprintf("%s/%s:latest", config.ContainerRegistryURL, config.ContainerImageName)) + + copiedFileExists, _ := utils.FileExists("/tmp/config.json") + assert.True(t, copiedFileExists) + }) + t.Run("success case (customTlsCertificates)", func(t *testing.T) { t.Parallel() httpmock.Activate() diff --git a/go.mod b/go.mod index d0057e6bfd..704331cfab 100644 --- a/go.mod +++ b/go.mod @@ -9,6 +9,7 @@ replace golang.org/x/crypto v0.0.0-20220622213112-05595931fe9d => golang.org/x/c require ( cloud.google.com/go/storage v1.22.1 github.com/Azure/azure-sdk-for-go/sdk/storage/azblob v0.4.0 + github.com/BurntSushi/toml v1.1.0 github.com/Jeffail/gabs/v2 v2.6.1 github.com/Masterminds/sprig v2.22.0+incompatible github.com/antchfx/htmlquery v1.2.4 @@ -46,7 +47,6 @@ require ( github.com/mitchellh/mapstructure v1.5.0 github.com/motemen/go-nuts v0.0.0-20210915132349-615a782f2c69 github.com/package-url/packageurl-go v0.1.0 - github.com/pelletier/go-toml v1.9.5 github.com/piper-validation/fortify-client-go v0.0.0-20220126145513-7b3e9a72af01 github.com/pkg/errors v0.9.1 github.com/sabhiram/go-gitignore v0.0.0-20210923224102-525f6e181f06 @@ -87,7 +87,6 @@ require ( github.com/Azure/go-autorest/autorest/validation v0.3.1 // indirect github.com/Azure/go-autorest/logger v0.2.1 // indirect github.com/Azure/go-autorest/tracing v0.6.0 // indirect - github.com/BurntSushi/toml v1.1.0 // indirect github.com/CycloneDX/cyclonedx-go v0.6.0 github.com/DataDog/datadog-go v3.2.0+incompatible // indirect github.com/Jeffail/gabs v1.1.1 // indirect diff --git a/go.sum b/go.sum index a0f987ce80..2194e632ac 100644 --- a/go.sum +++ b/go.sum @@ -1707,8 +1707,6 @@ github.com/pelletier/go-toml v1.7.0/go.mod h1:vwGMzjaWMwyfHwgIBhI2YUM4fB6nL6lVAv github.com/pelletier/go-toml v1.8.1/go.mod h1:T2/BmBdy8dvIRq1a/8aqjN41wvWlN4lrapLU/GW4pbc= github.com/pelletier/go-toml v1.9.3/go.mod h1:u1nR/EPcESfeI/szUZKdtJ0xRNbUoANCkoOuaOx1Y+c= github.com/pelletier/go-toml v1.9.4/go.mod h1:u1nR/EPcESfeI/szUZKdtJ0xRNbUoANCkoOuaOx1Y+c= -github.com/pelletier/go-toml v1.9.5 h1:4yBQzkHv+7BHq2PQUZF3Mx0IYxG7LsP222s7Agd3ve8= -github.com/pelletier/go-toml v1.9.5/go.mod h1:u1nR/EPcESfeI/szUZKdtJ0xRNbUoANCkoOuaOx1Y+c= github.com/peterbourgon/diskv v2.0.1+incompatible h1:UBdAOUP5p4RWqPBg048CAvpKN+vxiaj6gdUUzhl4XmI= github.com/peterbourgon/diskv v2.0.1+incompatible/go.mod h1:uqqh8zWWbv1HBMNONnaR/tNboyR3/BZd58JJSHlUSCU= github.com/petermattis/goid v0.0.0-20180202154549-b0b1615b78e5 h1:q2e307iGHPdTGp0hoxKjt1H5pDo6utceo3dQVK3I5XQ= diff --git a/integration/github_actions_integration_test_list.yml b/integration/github_actions_integration_test_list.yml index fc9acd2dff..006f177b2d 100644 --- a/integration/github_actions_integration_test_list.yml +++ b/integration/github_actions_integration_test_list.yml @@ -9,6 +9,7 @@ run: - '"TestCNBIntegrationNPMCustomBuildpacksBuildpacklessProject"' - '"TestCNBIntegrationNPMCustomBuildpacksFullProject"' - '"TestCNBIntegrationProjectDescriptor"' + - '"TestCNBIntegrationPrePostBuildpacks"' - '"TestGolangIntegration"' - '"TestGradleIntegration"' diff --git a/integration/integration_cnb_test.go b/integration/integration_cnb_test.go index 944651cf8e..1319e1bcc1 100644 --- a/integration/integration_cnb_test.go +++ b/integration/integration_cnb_test.go @@ -17,7 +17,7 @@ import ( const ( registryURL = "localhost:5000" - baseBuilder = "paketobuildpacks/builder:0.3.26-base" + baseBuilder = "paketobuildpacks/builder:0.3.280-base" ) func setupDockerRegistry(t *testing.T, ctx context.Context) testcontainers.Container { @@ -135,7 +135,7 @@ func TestCNBIntegrationZipPath(t *testing.T) { container.assertHasOutput(t, "running command: /cnb/lifecycle/creator", "Installing Go", - "Paketo Go Build Buildpack", + "Paketo Buildpack for Go Build", fmt.Sprintf("Saving %s/not-found:0.0.1", registryURL), "*** Images (sha256:", "SUCCESS", @@ -279,9 +279,9 @@ func TestCNBIntegrationMultiImage(t *testing.T) { assert.NoError(t, err) container.assertHasOutput(t, - "Previous image with name \"localhost:5000/io-buildpacks-my-app:latest\" not found", + "Image with name \"localhost:5000/io-buildpacks-my-app:latest\" not found", "Saving localhost:5000/io-buildpacks-my-app:latest...", - "Previous image with name \"localhost:5000/go-app:v1.0.0\" not found", + "Image with name \"localhost:5000/go-app:v1.0.0\" not found", "Saving localhost:5000/go-app:v1.0.0...", "Using cached buildpack", "Saving localhost:5000/my-app2:latest...", @@ -334,3 +334,27 @@ func TestCNBIntegrationPreserveFilesIgnored(t *testing.T) { container.assertHasOutput(t, "skipping preserving files because the source") container.terminate(t) } + +func TestCNBIntegrationPrePostBuildpacks(t *testing.T) { + t.Parallel() + ctx := context.Background() + registryContainer := setupDockerRegistry(t, ctx) + defer registryContainer.Terminate(ctx) + + container := givenThisContainer(t, IntegrationTestDockerExecRunnerBundle{ + Image: baseBuilder, + User: "cnb", + TestDir: []string{"testdata", "TestCnbIntegration"}, + Network: fmt.Sprintf("container:%s", registryContainer.GetContainerID()), + Environment: map[string]string{ + "PIPER_VAULTCREDENTIAL_DYNATRACE_API_KEY": "api-key-content", + }, + }) + + err := container.whenRunningPiperCommand("cnbBuild", "--noTelemetry", "--verbose", "--projectDescriptor", "", "--path", "project", "--customConfig", "config.yml", "--containerImageTag", "0.0.1", "--containerImageName", "not-found", "--containerRegistryUrl", registryURL, "--postBuildpacks", "paketobuildpacks/datadog") + assert.NoError(t, err) + container.assertHasOutput(t, "Setting custom buildpacks: '[]'") + container.assertHasOutput(t, "Pre-buildpacks: '[]'") + container.assertHasOutput(t, "Post-buildpacks: '[paketobuildpacks/datadog]'") + container.terminate(t) +} diff --git a/integration/testdata/TestCnbIntegration/config.yml b/integration/testdata/TestCnbIntegration/config.yml index da73f0510e..78aed906a0 100644 --- a/integration/testdata/TestCnbIntegration/config.yml +++ b/integration/testdata/TestCnbIntegration/config.yml @@ -3,6 +3,9 @@ general: collectTelemetryData: false steps: cnbBuild: + buildEnvVars: + BP_DATADOG_ENABLED: true + BP_EAR_KEY: 74657374 bindings: maven-settings: type: maven diff --git a/integration/testdata/TestCnbIntegration/project/package.json b/integration/testdata/TestCnbIntegration/project/package.json index 2c34dae7ee..be34f94b01 100644 --- a/integration/testdata/TestCnbIntegration/project/package.json +++ b/integration/testdata/TestCnbIntegration/project/package.json @@ -1,6 +1,7 @@ { "name": "test-mta-js", "version": "1.0.0", + "main": "srv/hello.js", "dependencies": { "jest": "^26.0.1", "jest-jenkins-reporter": "^1.0.2" diff --git a/pkg/cnbutils/buildpack.go b/pkg/cnbutils/buildpack.go index 5dc929f09d..8b744206dd 100644 --- a/pkg/cnbutils/buildpack.go +++ b/pkg/cnbutils/buildpack.go @@ -27,36 +27,30 @@ type License struct { URI string `toml:"uri" json:"uri"` } -func DownloadBuildpacks(path string, bpacks []string, dockerCreds string, utils BuildUtils) (Order, error) { +func DownloadBuildpacks(path string, bpacks []string, dockerCreds string, utils BuildUtils) error { if dockerCreds != "" { os.Setenv("DOCKER_CONFIG", filepath.Dir(dockerCreds)) } - var orderEntry OrderEntry - order := Order{ - Utils: utils, - } - err := utils.MkdirAll(bpCacheDir, os.ModePerm) if err != nil { - return Order{}, errors.Wrap(err, "failed to create temp directory for buildpack cache") + return errors.Wrap(err, "failed to create temp directory for buildpack cache") } for _, bpack := range bpacks { - var bpackMeta BuildPackMetadata imageInfo, err := utils.GetRemoteImageInfo(bpack) if err != nil { - return Order{}, errors.Wrap(err, "failed to get remote image info of buildpack") + return errors.Wrap(err, "failed to get remote image info of buildpack") } hash, err := imageInfo.Digest() if err != nil { - return Order{}, errors.Wrap(err, "failed to get image digest") + return errors.Wrap(err, "failed to get image digest") } cacheDir := filepath.Join(bpCacheDir, hash.String()) cacheExists, err := utils.DirExists(cacheDir) if err != nil { - return Order{}, errors.Wrapf(err, "failed to check if cache dir '%s' exists", cacheDir) + return errors.Wrapf(err, "failed to check if cache dir '%s' exists", cacheDir) } if cacheExists { @@ -64,36 +58,83 @@ func DownloadBuildpacks(path string, bpacks []string, dockerCreds string, utils } else { err := utils.MkdirAll(cacheDir, os.ModePerm) if err != nil { - return Order{}, errors.Wrap(err, "failed to create temp directory for buildpack cache") + return errors.Wrap(err, "failed to create temp directory for buildpack cache") } log.Entry().Infof("Downloading buildpack '%s' to %s", bpack, cacheDir) - img, err := utils.DownloadImageContent(bpack, cacheDir) + _, err = utils.DownloadImageContent(bpack, cacheDir) if err != nil { - return Order{}, errors.Wrapf(err, "failed download buildpack image '%s'", bpack) + return errors.Wrapf(err, "failed download buildpack image '%s'", bpack) } - imageInfo = img + } + + matches, err := utils.Glob(filepath.Join(cacheDir, "cnb/buildpacks/*")) + if err != nil { + return err + } + + for _, match := range matches { + err = CreateVersionSymlinks(path, match, utils) + if err != nil { + return err + } + } + } + + return nil +} + +func GetMetadata(bpacks []string, utils BuildUtils) ([]BuildPackMetadata, error) { + var metadata []BuildPackMetadata + + for _, bpack := range bpacks { + var bpackMeta BuildPackMetadata + imageInfo, err := utils.GetRemoteImageInfo(bpack) + if err != nil { + return nil, err } imgConf, err := imageInfo.ConfigFile() if err != nil { - return Order{}, errors.Wrapf(err, "failed to read '%s' image config", bpack) + return nil, errors.Wrapf(err, "failed to read '%s' image config", bpack) } err = json.Unmarshal([]byte(imgConf.Config.Labels["io.buildpacks.buildpackage.metadata"]), &bpackMeta) if err != nil { - return Order{}, errors.Wrapf(err, "failed unmarshal '%s' image label", bpack) + return nil, err } - log.Entry().Debugf("Buildpack metadata: '%v'", bpackMeta) - orderEntry.Group = append(orderEntry.Group, bpackMeta) + metadata = append(metadata, bpackMeta) + } + + return metadata, nil +} + +func CreateVersionSymlinks(basePath, buildpackDir string, utils BuildUtils) error { + newBuildpackPath := filepath.Join(basePath, filepath.Base(buildpackDir)) + err := utils.MkdirAll(newBuildpackPath, os.ModePerm) + if err != nil { + return err + } - err = CopyProject(filepath.Join(cacheDir, "cnb/buildpacks"), path, nil, nil, utils) + versions, err := utils.Glob(filepath.Join(buildpackDir, "*")) + if err != nil { + return err + } + + for _, version := range versions { + newVersionPath := filepath.Join(newBuildpackPath, filepath.Base(version)) + exists, err := utils.DirExists(newVersionPath) if err != nil { - return Order{}, err + return err } - } - order.Order = []OrderEntry{orderEntry} + if !exists { + err = utils.Symlink(version, newVersionPath) + if err != nil { + return err + } + } + } - return order, nil + return nil } diff --git a/pkg/cnbutils/buildpack_test.go b/pkg/cnbutils/buildpack_test.go index a480272716..af9a694b39 100644 --- a/pkg/cnbutils/buildpack_test.go +++ b/pkg/cnbutils/buildpack_test.go @@ -4,11 +4,13 @@ package cnbutils_test import ( + "fmt" "testing" "github.com/SAP/jenkins-library/pkg/cnbutils" "github.com/SAP/jenkins-library/pkg/mock" v1 "github.com/google/go-containerregistry/pkg/v1" + "github.com/google/go-containerregistry/pkg/v1/fake" fakeImage "github.com/google/go-containerregistry/pkg/v1/fake" "github.com/stretchr/testify/assert" ) @@ -20,8 +22,9 @@ func TestBuildpackDownload(t *testing.T) { DownloadMock: &mock.DownloadMock{}, } - t.Run("it creates an order object", func(t *testing.T) { + t.Run("successfully downloads a buildpack", func(t *testing.T) { fakeImg := &fakeImage.FakeImage{} + fakeImg.DigestReturns(v1.NewHash("sha256:2cf24dba5fb0a30e26e83b2ac5b9e29e1b161e5c1fa7425e73043362938b9824")) fakeImg.ConfigFileReturns(&v1.ConfigFile{ Config: v1.Config{ Labels: map[string]string{ @@ -32,9 +35,50 @@ func TestBuildpackDownload(t *testing.T) { mockUtils.ReturnImage = fakeImg mockUtils.RemoteImageInfo = fakeImg - order, err := cnbutils.DownloadBuildpacks("/destination", []string{"buildpack"}, "/tmp/config.json", mockUtils) + err := cnbutils.DownloadBuildpacks("/destination", []string{"buildpack"}, "/tmp/config.json", mockUtils) + assert.NoError(t, err) + }) +} + +func TestGetMetadata(t *testing.T) { + var mockUtils = &cnbutils.MockUtils{ + ExecMockRunner: &mock.ExecMockRunner{}, + FilesMock: &mock.FilesMock{}, + DownloadMock: &mock.DownloadMock{ + ImageInfoStub: func(imageRef string) (v1.Image, error) { + return &fake.FakeImage{ + ConfigFileStub: func() (*v1.ConfigFile, error) { + return &v1.ConfigFile{ + Config: v1.Config{ + Labels: map[string]string{ + "io.buildpacks.buildpackage.metadata": fmt.Sprintf("{\"id\": \"%s\", \"version\": \"0.0.1\"}", imageRef), + }, + }, + }, nil + }, + }, nil + }, + }, + } + t.Run("returns empty metadata", func(t *testing.T) { + meta, err := cnbutils.GetMetadata(nil, mockUtils) assert.NoError(t, err) - assert.Equal(t, 1, len(order.Order)) + assert.Empty(t, meta) + }) + + t.Run("returns metadata of the provided buildpacks", func(t *testing.T) { + meta, err := cnbutils.GetMetadata([]string{"buildpack1", "buildpack2"}, mockUtils) + assert.NoError(t, err) + assert.Equal(t, []cnbutils.BuildPackMetadata{ + { + ID: "buildpack1", + Version: "0.0.1", + }, + { + ID: "buildpack2", + Version: "0.0.1", + }, + }, meta) }) } diff --git a/pkg/cnbutils/order.go b/pkg/cnbutils/order.go index 8c9213e23c..a7f9d0dd1d 100644 --- a/pkg/cnbutils/order.go +++ b/pkg/cnbutils/order.go @@ -2,10 +2,14 @@ package cnbutils import ( "bytes" + "os" + "path/filepath" - "github.com/pelletier/go-toml" + "github.com/BurntSushi/toml" ) +const DefaultOrderPath = "/cnb/order.toml" + type Order struct { Order []OrderEntry `toml:"order"` Utils BuildUtils `toml:"-"` @@ -30,3 +34,73 @@ func (o Order) Save(path string) error { return nil } + +func loadExistingOrder(utils BuildUtils) (Order, error) { + order := Order{ + Utils: utils, + } + + orderReader, err := utils.Open(DefaultOrderPath) + if err != nil { + return Order{}, err + } + defer orderReader.Close() + + _, err = toml.NewDecoder(orderReader).Decode(&order) + if err != nil { + return Order{}, err + } + + return order, nil +} + +func newOrder(bpacks []string, utils BuildUtils) (Order, error) { + buildpacksMeta, err := GetMetadata(bpacks, utils) + if err != nil { + return Order{}, err + } + + return Order{ + Utils: utils, + Order: []OrderEntry{{ + Group: buildpacksMeta, + }}, + }, nil +} + +func CreateOrder(bpacks, preBpacks, postBpacks []string, dockerCreds string, utils BuildUtils) (Order, error) { + if dockerCreds != "" { + os.Setenv("DOCKER_CONFIG", filepath.Dir(dockerCreds)) + } + + var order Order + var err error + if len(bpacks) == 0 { + order, err = loadExistingOrder(utils) + if err != nil { + return Order{}, err + } + } else { + order, err = newOrder(bpacks, utils) + if err != nil { + return Order{}, err + } + } + + for idx := range order.Order { + preMetadata, err := GetMetadata(preBpacks, utils) + if err != nil { + return Order{}, err + } + + postMetadata, err := GetMetadata(postBpacks, utils) + if err != nil { + return Order{}, err + } + + order.Order[idx].Group = append(preMetadata, order.Order[idx].Group...) + order.Order[idx].Group = append(order.Order[idx].Group, postMetadata...) + } + + return order, nil +} diff --git a/pkg/cnbutils/order_test.go b/pkg/cnbutils/order_test.go index fa75e1170b..3b6968428e 100644 --- a/pkg/cnbutils/order_test.go +++ b/pkg/cnbutils/order_test.go @@ -9,6 +9,8 @@ import ( "github.com/SAP/jenkins-library/pkg/cnbutils" "github.com/SAP/jenkins-library/pkg/mock" + v1 "github.com/google/go-containerregistry/pkg/v1" + "github.com/google/go-containerregistry/pkg/v1/fake" "github.com/stretchr/testify/assert" ) @@ -43,7 +45,7 @@ func TestOrderSave(t *testing.T) { assert.True(t, mockUtils.HasWrittenFile("/tmp/order.toml")) result, err := mockUtils.FileRead("/tmp/order.toml") assert.NoError(t, err) - assert.Equal(t, "\n[[order]]\n\n [[order.group]]\n id = \"paketo-buildpacks/sap-machine\"\n version = \"1.1.1\"\n\n [[order.group]]\n id = \"paketo-buildpacks/java\"\n version = \"2.2.2\"\n", string(result)) + assert.Equal(t, "[[order]]\n\n [[order.group]]\n id = \"paketo-buildpacks/sap-machine\"\n version = \"1.1.1\"\n\n [[order.group]]\n id = \"paketo-buildpacks/java\"\n version = \"2.2.2\"\n", string(result)) }) t.Run("raises an error if unable to write the file", func(t *testing.T) { @@ -64,3 +66,155 @@ func TestOrderSave(t *testing.T) { assert.False(t, mockUtils.HasWrittenFile("/tmp/order.toml")) }) } + +func TestCreateOrder(t *testing.T) { + imageStub := func(imageRef, target string) (v1.Image, error) { + fakeImage := &fake.FakeImage{} + var imageConfig v1.Config + switch imageRef { + case "pre-buildpack": + imageConfig = v1.Config{ + Labels: map[string]string{ + "io.buildpacks.buildpackage.metadata": "{\"id\": \"pre-testbuildpack\", \"version\": \"0.0.1\"}", + }, + } + case "post-buildpack": + imageConfig = v1.Config{ + Labels: map[string]string{ + "io.buildpacks.buildpackage.metadata": "{\"id\": \"post-testbuildpack\", \"version\": \"0.0.1\"}", + }, + } + default: + imageConfig = v1.Config{ + Labels: map[string]string{ + "io.buildpacks.buildpackage.metadata": "{\"id\": \"testbuildpack\", \"version\": \"0.0.1\"}", + }, + } + } + + fakeImage.ConfigFileReturns(&v1.ConfigFile{ + Config: imageConfig, + }, nil) + + return fakeImage, nil + } + + mockUtils := &cnbutils.MockUtils{ + FilesMock: &mock.FilesMock{}, + DownloadMock: &mock.DownloadMock{ + ImageContentStub: imageStub, + ImageInfoStub: func(imageRef string) (v1.Image, error) { + return imageStub(imageRef, "") + }, + }, + } + + mockUtils.AddFile(cnbutils.DefaultOrderPath, []byte(`[[order]] + [[order.group]] + id = "buildpacks/java" + version = "1.8.0" +[[order]] + [[order.group]] + id = "buildpacks/nodejs" + version = "1.6.0"`)) + + t.Run("successfully loads baked in order.toml", func(t *testing.T) { + order, err := cnbutils.CreateOrder(nil, nil, nil, "", mockUtils) + assert.NoError(t, err) + assert.Equal(t, []cnbutils.OrderEntry{ + { + Group: []cnbutils.BuildPackMetadata{ + { + ID: "buildpacks/java", + Version: "1.8.0", + }, + }, + }, + { + Group: []cnbutils.BuildPackMetadata{ + { + ID: "buildpacks/nodejs", + Version: "1.6.0", + }, + }, + }, + }, order.Order) + }) + + t.Run("successfully loads baked in order.toml and adds pre/post buildpacks", func(t *testing.T) { + order, err := cnbutils.CreateOrder(nil, []string{"pre-buildpack"}, []string{"post-buildpack"}, "", mockUtils) + assert.NoError(t, err) + assert.Equal(t, []cnbutils.OrderEntry{ + { + Group: []cnbutils.BuildPackMetadata{ + { + ID: "pre-testbuildpack", + Version: "0.0.1", + }, + { + ID: "buildpacks/java", + Version: "1.8.0", + }, + { + ID: "post-testbuildpack", + Version: "0.0.1", + }, + }, + }, + { + Group: []cnbutils.BuildPackMetadata{ + { + ID: "pre-testbuildpack", + Version: "0.0.1", + }, + { + ID: "buildpacks/nodejs", + Version: "1.6.0", + }, + { + ID: "post-testbuildpack", + Version: "0.0.1", + }, + }, + }, + }, order.Order) + }) + + t.Run("successfully creates new order with custom buildpacks", func(t *testing.T) { + order, err := cnbutils.CreateOrder([]string{"testbuildpack"}, nil, nil, "", mockUtils) + assert.NoError(t, err) + assert.Equal(t, []cnbutils.OrderEntry{ + { + Group: []cnbutils.BuildPackMetadata{ + { + ID: "testbuildpack", + Version: "0.0.1", + }, + }, + }, + }, order.Order) + }) + + t.Run("successfully creates new order with custom buildpacks and adds pre/post buildpacks", func(t *testing.T) { + order, err := cnbutils.CreateOrder([]string{"testbuildpack"}, []string{"pre-buildpack"}, []string{"post-buildpack"}, "", mockUtils) + assert.NoError(t, err) + assert.Equal(t, []cnbutils.OrderEntry{ + { + Group: []cnbutils.BuildPackMetadata{ + { + ID: "pre-testbuildpack", + Version: "0.0.1", + }, + { + ID: "testbuildpack", + Version: "0.0.1", + }, + { + ID: "post-testbuildpack", + Version: "0.0.1", + }, + }, + }, + }, order.Order) + }) +} diff --git a/pkg/cnbutils/project/descriptor.go b/pkg/cnbutils/project/descriptor.go index 71a544d6a6..620b27c79c 100644 --- a/pkg/cnbutils/project/descriptor.go +++ b/pkg/cnbutils/project/descriptor.go @@ -2,56 +2,40 @@ package project import ( - "errors" + "github.com/pkg/errors" + "github.com/BurntSushi/toml" "github.com/SAP/jenkins-library/pkg/cnbutils" + "github.com/SAP/jenkins-library/pkg/cnbutils/project/types" + v01 "github.com/SAP/jenkins-library/pkg/cnbutils/project/v01" + v02 "github.com/SAP/jenkins-library/pkg/cnbutils/project/v02" "github.com/SAP/jenkins-library/pkg/cnbutils/registry" piperhttp "github.com/SAP/jenkins-library/pkg/http" "github.com/SAP/jenkins-library/pkg/log" - "github.com/pelletier/go-toml" ignore "github.com/sabhiram/go-gitignore" ) -type script struct { - API string `toml:"api"` - Inline string `toml:"inline"` - Shell string `toml:"shell"` -} -type buildpack struct { - ID string `toml:"id"` - Version string `toml:"version"` - URI string `toml:"uri"` - Script script `toml:"script"` -} - -type envVar struct { - Name string `toml:"name"` - Value string `toml:"value"` +type project struct { + Version string `toml:"schema-version"` } -type build struct { - Include []string `toml:"include"` - Exclude []string `toml:"exclude"` - Buildpacks []buildpack `toml:"buildpacks"` - Env []envVar `toml:"env"` +type versionDescriptor struct { + Project project `toml:"_"` } -type project struct { - ID string `toml:"id"` -} - -type projectDescriptor struct { - Build build `toml:"build"` - Project project `toml:"project"` - Metadata map[string]interface{} `toml:"metadata"` +var parsers = map[string]func(string) (types.Descriptor, error){ + "0.1": v01.NewDescriptor, + "0.2": v02.NewDescriptor, } type Descriptor struct { - Exclude *ignore.GitIgnore - Include *ignore.GitIgnore - EnvVars map[string]interface{} - Buildpacks []string - ProjectID string + Exclude *ignore.GitIgnore + Include *ignore.GitIgnore + EnvVars map[string]interface{} + Buildpacks []string + PreBuildpacks []string + PostBuildpacks []string + ProjectID string } func ParseDescriptor(descriptorPath string, utils cnbutils.BuildUtils, httpClient piperhttp.Sender) (*Descriptor, error) { @@ -62,23 +46,45 @@ func ParseDescriptor(descriptorPath string, utils cnbutils.BuildUtils, httpClien return nil, err } - rawDescriptor := projectDescriptor{} - err = toml.Unmarshal(descriptorContent, &rawDescriptor) + var versionDescriptor versionDescriptor + _, err = toml.Decode(string(descriptorContent), &versionDescriptor) if err != nil { - return nil, err + return &Descriptor{}, errors.Wrapf(err, "parsing schema version") + } + + version := versionDescriptor.Project.Version + if version == "" { + version = "0.1" + } + + rawDescriptor, err := parsers[version](string(descriptorContent)) + if err != nil { + return &Descriptor{}, err } - if rawDescriptor.Build.Buildpacks != nil && len(rawDescriptor.Build.Buildpacks) > 0 { - buildpacksImg, err := rawDescriptor.Build.searchBuildpacks(httpClient) + if len(rawDescriptor.Build.Buildpacks) > 0 { + descriptor.Buildpacks, err = searchBuildpacks(rawDescriptor.Build.Buildpacks, httpClient) if err != nil { return nil, err } + } - descriptor.Buildpacks = buildpacksImg + if len(rawDescriptor.Build.Pre.Buildpacks) > 0 { + descriptor.PreBuildpacks, err = searchBuildpacks(rawDescriptor.Build.Pre.Buildpacks, httpClient) + if err != nil { + return nil, err + } + } + + if len(rawDescriptor.Build.Post.Buildpacks) > 0 { + descriptor.PostBuildpacks, err = searchBuildpacks(rawDescriptor.Build.Post.Buildpacks, httpClient) + if err != nil { + return nil, err + } } - if rawDescriptor.Build.Env != nil && len(rawDescriptor.Build.Env) > 0 { - descriptor.EnvVars = rawDescriptor.Build.envToMap() + if len(rawDescriptor.Build.Env) > 0 { + descriptor.EnvVars = envToMap(rawDescriptor.Build.Env) } if len(rawDescriptor.Build.Exclude) > 0 && len(rawDescriptor.Build.Include) > 0 { @@ -100,10 +106,10 @@ func ParseDescriptor(descriptorPath string, utils cnbutils.BuildUtils, httpClien return descriptor, nil } -func (b *build) envToMap() map[string]interface{} { +func envToMap(env []types.EnvVar) map[string]interface{} { envMap := map[string]interface{}{} - for _, e := range b.Env { + for _, e := range env { if len(e.Name) == 0 { continue } @@ -114,11 +120,11 @@ func (b *build) envToMap() map[string]interface{} { return envMap } -func (b *build) searchBuildpacks(httpClient piperhttp.Sender) ([]string, error) { +func searchBuildpacks(buildpacks []types.Buildpack, httpClient piperhttp.Sender) ([]string, error) { var bpackImg []string - for _, bpack := range b.Buildpacks { - if bpack.Script != (script{}) { + for _, bpack := range buildpacks { + if bpack.Script != (types.Script{}) { return nil, errors.New("inline buildpacks are not supported") } diff --git a/pkg/cnbutils/project/descriptor_test.go b/pkg/cnbutils/project/descriptor_test.go index 88cdbe17bb..33d03cb137 100644 --- a/pkg/cnbutils/project/descriptor_test.go +++ b/pkg/cnbutils/project/descriptor_test.go @@ -16,7 +16,7 @@ import ( ) func TestParseDescriptor(t *testing.T) { - t.Run("parses the project.toml file", func(t *testing.T) { + t.Run("parses the project.toml file v01", func(t *testing.T) { projectToml := `[project] id = "io.buildpacks.my-app" version = "0.1" @@ -41,6 +41,14 @@ value = "VAL2" name = "EMPTY" value = "" +[[build.pre.group]] +id = "paketo-buildpacks/java" +version = "5.9.1" + +[[build.post.group]] +id = "paketo-buildpacks/java" +version = "5.9.1" + [[build.buildpacks]] id = "paketo-buildpacks/java" version = "5.9.1" @@ -78,6 +86,94 @@ id = "paketo-buildpacks/nodejs" assert.Contains(t, descriptor.Buildpacks, "index.docker.io/test-java@5.9.1") assert.Contains(t, descriptor.Buildpacks, "index.docker.io/test-nodejs@1.1.1") + assert.Contains(t, descriptor.PreBuildpacks, "index.docker.io/test-java@5.9.1") + assert.Contains(t, descriptor.PostBuildpacks, "index.docker.io/test-java@5.9.1") + + assert.NotNil(t, descriptor.Include) + + t3 := descriptor.Include.MatchesPath("cmd/cobra.go") + assert.True(t, t3) + + t4 := descriptor.Include.MatchesPath("pkg/test/main.go") + assert.True(t, t4) + + t5 := descriptor.Include.MatchesPath("Makefile") + assert.False(t, t5) + }) + + t.Run("parses the project.toml file v02", func(t *testing.T) { + projectToml := `[_] +id = "io.buildpacks.my-app" +version = "0.1" +schema-version = "0.2" + +[io.buildpacks] +include = [ + "cmd/", + "go.mod", + "go.sum", + "*.go" +] + +[[io.buildpacks.build.env]] +name = "VAR1" +value = "VAL1" + +[[io.buildpacks.build.env]] +name = "VAR2" +value = "VAL2" + +[[io.buildpacks.build.env]] +name = "EMPTY" +value = "" + +[[io.buildpacks.pre.group]] +id = "paketo-buildpacks/java" +version = "5.9.1" + +[[io.buildpacks.post.group]] +id = "paketo-buildpacks/java" +version = "5.9.1" + +[[io.buildpacks.group]] +id = "paketo-buildpacks/java" +version = "5.9.1" + +[[io.buildpacks.group]] +id = "paketo-buildpacks/nodejs" +` + utils := &cnbutils.MockUtils{ + FilesMock: &mock.FilesMock{}, + } + + fakeJavaResponse := "{\"latest\":{\"version\":\"1.1.1\",\"namespace\":\"test\",\"name\":\"test\",\"description\":\"\",\"homepage\":\"\",\"licenses\":null,\"stacks\":[\"test\",\"test\"],\"id\":\"test\"},\"versions\":[{\"version\":\"5.9.1\",\"_link\":\"https://test-java/5.9.1\"}]}" + fakeNodeJsResponse := "{\"latest\":{\"version\":\"1.1.1\",\"namespace\":\"test\",\"name\":\"test\",\"description\":\"\",\"homepage\":\"\",\"licenses\":null,\"stacks\":[\"test\",\"test\"],\"id\":\"test\"},\"versions\":[{\"version\":\"1.1.1\",\"_link\":\"https://test-nodejs/1.1.1\"}]}" + + utils.AddFile("project.toml", []byte(projectToml)) + httpmock.Activate() + defer httpmock.DeactivateAndReset() + httpmock.RegisterResponder(http.MethodGet, "https://registry.buildpacks.io/api/v1/buildpacks/paketo-buildpacks/java", httpmock.NewStringResponder(200, fakeJavaResponse)) + httpmock.RegisterResponder(http.MethodGet, "https://registry.buildpacks.io/api/v1/buildpacks/paketo-buildpacks/nodejs", httpmock.NewStringResponder(200, fakeNodeJsResponse)) + + httpmock.RegisterResponder(http.MethodGet, "https://test-java/5.9.1", httpmock.NewStringResponder(200, "{\"addr\": \"index.docker.io/test-java@5.9.1\"}")) + httpmock.RegisterResponder(http.MethodGet, "https://test-nodejs/1.1.1", httpmock.NewStringResponder(200, "{\"addr\": \"index.docker.io/test-nodejs@1.1.1\"}")) + client := &piperhttp.Client{} + client.SetOptions(piperhttp.ClientOptions{MaxRetries: -1, UseDefaultTransport: true}) + + descriptor, err := ParseDescriptor("project.toml", utils, client) + + assert.NoError(t, err) + assert.Equal(t, "VAL1", descriptor.EnvVars["VAR1"]) + assert.Equal(t, "VAL2", descriptor.EnvVars["VAR2"]) + assert.Equal(t, "", descriptor.EnvVars["EMPTY"]) + + assert.Equal(t, "io.buildpacks.my-app", descriptor.ProjectID) + + assert.Contains(t, descriptor.Buildpacks, "index.docker.io/test-java@5.9.1") + assert.Contains(t, descriptor.Buildpacks, "index.docker.io/test-nodejs@1.1.1") + assert.Contains(t, descriptor.PreBuildpacks, "index.docker.io/test-java@5.9.1") + assert.Contains(t, descriptor.PostBuildpacks, "index.docker.io/test-java@5.9.1") + assert.NotNil(t, descriptor.Include) t3 := descriptor.Include.MatchesPath("cmd/cobra.go") @@ -160,6 +256,6 @@ exclude = [ _, err := ParseDescriptor("project.toml", utils, &piperhttp.Client{}) assert.Error(t, err) - assert.Equal(t, "(1, 8): was expecting token =, but got EOF instead", err.Error()) + assert.Equal(t, "parsing schema version: toml: line 0: unexpected EOF; expected key separator '='", err.Error()) }) } diff --git a/pkg/cnbutils/project/metadata/metadata.go b/pkg/cnbutils/project/metadata/metadata.go index 167469adac..abbb3498b4 100644 --- a/pkg/cnbutils/project/metadata/metadata.go +++ b/pkg/cnbutils/project/metadata/metadata.go @@ -5,11 +5,11 @@ import ( "bytes" "path/filepath" + "github.com/BurntSushi/toml" "github.com/SAP/jenkins-library/pkg/cnbutils" "github.com/SAP/jenkins-library/pkg/log" "github.com/SAP/jenkins-library/pkg/piperenv" "github.com/buildpacks/lifecycle/platform" - "github.com/pelletier/go-toml" ) var metadataFilePath = "/layers/project-metadata.toml" diff --git a/pkg/cnbutils/project/metadata/metadata_test.go b/pkg/cnbutils/project/metadata/metadata_test.go index 6310d8067b..cf36c9eaaa 100644 --- a/pkg/cnbutils/project/metadata/metadata_test.go +++ b/pkg/cnbutils/project/metadata/metadata_test.go @@ -15,16 +15,13 @@ import ( ) func TestWriteProjectMetadata(t *testing.T) { - expectedResult := ` -[source] + expectedResult := `[source] type = "git" - - [source.metadata] - refs = ["main"] - [source.version] commit = "012548" describe = "test-commit" + [source.metadata] + refs = ["main"] ` mockUtils := &cnbutils.MockUtils{ ExecMockRunner: &mock.ExecMockRunner{}, diff --git a/pkg/cnbutils/project/types/types.go b/pkg/cnbutils/project/types/types.go new file mode 100644 index 0000000000..a4f9521392 --- /dev/null +++ b/pkg/cnbutils/project/types/types.go @@ -0,0 +1,58 @@ +// Source: https://github.com/buildpacks/pack/blob/main/pkg/project/types/types.go +package types + +import ( + "github.com/buildpacks/lifecycle/api" +) + +type Script struct { + API string `toml:"api"` + Inline string `toml:"inline"` + Shell string `toml:"shell"` +} + +type Buildpack struct { + ID string `toml:"id"` + Version string `toml:"version"` + URI string `toml:"uri"` + Script Script `toml:"script"` +} + +type EnvVar struct { + Name string `toml:"name"` + Value string `toml:"value"` +} + +type Build struct { + Include []string `toml:"include"` + Exclude []string `toml:"exclude"` + Buildpacks []Buildpack `toml:"buildpacks"` + Env []EnvVar `toml:"env"` + Builder string `toml:"builder"` + Pre GroupAddition + Post GroupAddition +} + +type Project struct { + ID string `toml:"id"` + Name string `toml:"name"` + Version string `toml:"version"` + SourceURL string `toml:"source-url"` + Licenses []License `toml:"licenses"` +} + +type License struct { + Type string `toml:"type"` + URI string `toml:"uri"` +} + +type Descriptor struct { + Project Project `toml:"project"` + Build Build `toml:"build"` + Metadata map[string]interface{} `toml:"metadata"` + SchemaVersion *api.Version +} + +type GroupAddition struct { + Buildpacks []Buildpack `toml:"group"` +} diff --git a/pkg/cnbutils/project/v01/project.go b/pkg/cnbutils/project/v01/project.go new file mode 100644 index 0000000000..da41b98ad5 --- /dev/null +++ b/pkg/cnbutils/project/v01/project.go @@ -0,0 +1,30 @@ +// Source: https://github.com/buildpacks/pack/blob/main/pkg/project/v01/project.go +package v01 + +import ( + "github.com/BurntSushi/toml" + "github.com/SAP/jenkins-library/pkg/cnbutils/project/types" + "github.com/buildpacks/lifecycle/api" +) + +type Descriptor struct { + Project types.Project `toml:"project"` + Build types.Build `toml:"build"` + Metadata map[string]interface{} `toml:"metadata"` +} + +func NewDescriptor(projectTomlContents string) (types.Descriptor, error) { + versionedDescriptor := &Descriptor{} + + _, err := toml.Decode(projectTomlContents, versionedDescriptor) + if err != nil { + return types.Descriptor{}, err + } + + return types.Descriptor{ + Project: versionedDescriptor.Project, + Build: versionedDescriptor.Build, + Metadata: versionedDescriptor.Metadata, + SchemaVersion: api.MustParse("0.1"), + }, nil +} diff --git a/pkg/cnbutils/project/v02/project.go b/pkg/cnbutils/project/v02/project.go new file mode 100644 index 0000000000..96bf3c6861 --- /dev/null +++ b/pkg/cnbutils/project/v02/project.go @@ -0,0 +1,78 @@ +// Source: https://github.com/buildpacks/pack/blob/main/pkg/project/v02/project.go +package v02 + +import ( + "github.com/BurntSushi/toml" + "github.com/SAP/jenkins-library/pkg/cnbutils/project/types" + "github.com/buildpacks/lifecycle/api" +) + +type Buildpacks struct { + Include []string `toml:"include"` + Exclude []string `toml:"exclude"` + Group []types.Buildpack `toml:"group"` + Env Env `toml:"env"` + Build Build `toml:"build"` + Builder string `toml:"builder"` + Pre types.GroupAddition `toml:"pre"` + Post types.GroupAddition `toml:"post"` +} + +type Build struct { + Env []types.EnvVar `toml:"env"` +} + +// Env is deprecated: use `[[io.buildpacks.build.env]]` instead. see https://github.com/buildpacks/pack/pull/1479 +type Env struct { + Build []types.EnvVar `toml:"build"` +} + +type Project struct { + ID string `toml:"id"` + Name string `toml:"name"` + Licenses []types.License `toml:"licenses"` + Metadata map[string]interface{} `toml:"metadata"` + SchemaVersion string `toml:"schema-version"` +} + +type IO struct { + Buildpacks Buildpacks `toml:"buildpacks"` +} + +type Descriptor struct { + Project Project `toml:"_"` + IO IO `toml:"io"` +} + +func NewDescriptor(projectTomlContents string) (types.Descriptor, error) { + versionedDescriptor := &Descriptor{} + _, err := toml.Decode(projectTomlContents, &versionedDescriptor) + if err != nil { + return types.Descriptor{}, err + } + + // backward compatibility for incorrect key + env := versionedDescriptor.IO.Buildpacks.Build.Env + if env == nil { + env = versionedDescriptor.IO.Buildpacks.Env.Build + } + + return types.Descriptor{ + Project: types.Project{ + ID: versionedDescriptor.Project.ID, + Name: versionedDescriptor.Project.Name, + Licenses: versionedDescriptor.Project.Licenses, + }, + Build: types.Build{ + Include: versionedDescriptor.IO.Buildpacks.Include, + Exclude: versionedDescriptor.IO.Buildpacks.Exclude, + Buildpacks: versionedDescriptor.IO.Buildpacks.Group, + Env: env, + Builder: versionedDescriptor.IO.Buildpacks.Builder, + Pre: versionedDescriptor.IO.Buildpacks.Pre, + Post: versionedDescriptor.IO.Buildpacks.Post, + }, + Metadata: versionedDescriptor.Project.Metadata, + SchemaVersion: api.MustParse("0.2"), + }, nil +} diff --git a/pkg/cnbutils/report.go b/pkg/cnbutils/report.go index 0f5e99f491..8b0fe37280 100644 --- a/pkg/cnbutils/report.go +++ b/pkg/cnbutils/report.go @@ -3,8 +3,8 @@ package cnbutils import ( "fmt" + "github.com/BurntSushi/toml" "github.com/buildpacks/lifecycle/platform" - "github.com/pelletier/go-toml" ) const reportFile = "/layers/report.toml" diff --git a/pkg/cnbutils/report_test.go b/pkg/cnbutils/report_test.go index c82eee561e..4ac2711099 100644 --- a/pkg/cnbutils/report_test.go +++ b/pkg/cnbutils/report_test.go @@ -46,6 +46,6 @@ digest = "sha256:52eac630560210e5ae13eb10797c4246d6f02d425f32b9430ca00bde697c79e digest, err := cnbutils.DigestFromReport(mockUtils) assert.Empty(t, digest) - assert.EqualError(t, err, "(1, 1): parsing error: keys cannot contain { character") + assert.EqualError(t, err, "toml: line 1: expected '.' or '=', but got '{' instead") }) } diff --git a/pkg/mock/dockerClient.go b/pkg/mock/dockerClient.go index f2fb0689c8..2f1f972715 100644 --- a/pkg/mock/dockerClient.go +++ b/pkg/mock/dockerClient.go @@ -17,7 +17,9 @@ type DownloadMock struct { RemoteImageInfo v1.Image ReturnError string - Stub func(imageRef, targetDir string) (v1.Image, error) + Stub func(imageRef, targetDir string) (v1.Image, error) + ImageContentStub func(imageRef, targetFile string) (v1.Image, error) + ImageInfoStub func(imageRef string) (v1.Image, error) } // DownloadImage . @@ -40,6 +42,10 @@ func (c *DownloadMock) DownloadImageContent(imageRef, targetFile string) (v1.Ima c.ImageRef = imageRef c.FilePath = targetFile + if c.ImageContentStub != nil { + return c.ImageContentStub(imageRef, targetFile) + } + if len(c.ReturnError) > 0 { return nil, fmt.Errorf(c.ReturnError) } @@ -50,6 +56,10 @@ func (c *DownloadMock) DownloadImageContent(imageRef, targetFile string) (v1.Ima func (c *DownloadMock) GetRemoteImageInfo(imageRef string) (v1.Image, error) { c.RemoteImageRef = imageRef + if c.ImageInfoStub != nil { + return c.ImageInfoStub(imageRef) + } + if len(c.ReturnError) > 0 { return nil, fmt.Errorf(c.ReturnError) } diff --git a/resources/metadata/cnbBuild.yaml b/resources/metadata/cnbBuild.yaml index 9fe08524f7..ffd5c44fb8 100644 --- a/resources/metadata/cnbBuild.yaml +++ b/resources/metadata/cnbBuild.yaml @@ -97,7 +97,7 @@ spec: param: container/registryUrl - name: buildpacks type: "[]string" - description: List of custom buildpacks to use in the form of `$HOSTNAME/$REPO[:$TAG]`. + description: List of custom buildpacks to use in the form of `$HOSTNAME/$REPO[:$TAG]`. When this property is specified, buildpacks which are part of the builder will be ignored. scope: - PARAMETERS - STAGES @@ -105,6 +105,26 @@ spec: resourceRef: - name: commonPipelineEnvironment param: container/buildpacks + - name: preBuildpacks + type: "[]string" + description: Buildpacks to prepend to the groups in the builder's order. + scope: + - PARAMETERS + - STAGES + - STEPS + resourceRef: + - name: commonPipelineEnvironment + param: container/preBuildpacks + - name: postBuildpacks + type: "[]string" + description: Buildpacks to append to the groups in the builder's order. + scope: + - PARAMETERS + - STAGES + - STEPS + resourceRef: + - name: commonPipelineEnvironment + param: container/postBuildpacks - name: buildEnvVars type: "map[string]interface{}" description: |