feat(cnbBuild) Add support for pre and post-buildpacks (SAP#4448)

* Add pre and post buildpacks Co-authored-by: Johannes Dillmann <[email protected]> Co-authored-by: Ralf Pannemans <[email protected]> Co-authored-by: Pavel Busko <[email protected]> * fix integration tests Co-authored-by: Pavel Busko <[email protected]> Co-authored-by: Ralf Pannemans <[email protected]> * simplify if clauses Co-authored-by: Pavel Busko <[email protected]> --------- Co-authored-by: Johannes Dillmann <[email protected]> Co-authored-by: Ralf Pannemans <[email protected]>
maxatsap · Jul 23, 2024 · 757e2e7 · 757e2e7
1 parent aff8633
commit 757e2e7
Show file tree

Hide file tree

Showing 24 changed files with 913 additions and 116 deletions.
diff --git a/cmd/cnbBuild.go b/cmd/cnbBuild.go
@@ -112,10 +112,29 @@ func processConfigs(main cnbBuildOptions, multipleImages []map[string]interface{
 	return result, nil
 }
 
-func setCustomBuildpacks(bpacks []string, dockerCreds string, utils cnbutils.BuildUtils) (string, string, error) {
+func setCustomBuildpacks(bpacks, preBuildpacks, postBuildpacks []string, dockerCreds string, utils cnbutils.BuildUtils) (string, string, error) {
 	buildpacksPath := "/tmp/buildpacks"
 	orderPath := "/tmp/buildpacks/order.toml"
-	newOrder, err := cnbutils.DownloadBuildpacks(buildpacksPath, bpacks, dockerCreds, utils)
+	err := cnbutils.DownloadBuildpacks(buildpacksPath, append(bpacks, append(preBuildpacks, postBuildpacks...)...), dockerCreds, utils)
+	if err != nil {
+		return "", "", err
+	}
+
+	if len(bpacks) == 0 && (len(postBuildpacks) > 0 || len(preBuildpacks) > 0) {
+		matches, err := utils.Glob("/cnb/buildpacks/*")
+		if err != nil {
+			return "", "", err
+		}
+
+		for _, match := range matches {
+			err = cnbutils.CreateVersionSymlinks(buildpacksPath, match, utils)
+			if err != nil {
+				return "", "", err
+			}
+		}
+	}
+
+	newOrder, err := cnbutils.CreateOrder(bpacks, preBuildpacks, postBuildpacks, dockerCreds, utils)
 	if err != nil {
 		return "", "", err
 	}
@@ -475,10 +494,18 @@ func runCnbBuild(config *cnbBuildOptions, cnbTelemetry *cnbBuildTelemetry, utils
 
 		config.mergeEnvVars(descriptor.EnvVars)
 
-		if (config.Buildpacks == nil || len(config.Buildpacks) == 0) && len(descriptor.Buildpacks) > 0 {
+		if len(config.Buildpacks) == 0 {
 			config.Buildpacks = descriptor.Buildpacks
 		}
 
+		if len(config.PreBuildpacks) == 0 {
+			config.PreBuildpacks = descriptor.PreBuildpacks
+		}
+
+		if len(config.PostBuildpacks) == 0 {
+			config.PostBuildpacks = descriptor.PostBuildpacks
+		}
+
 		if descriptor.Exclude != nil {
 			exclude = descriptor.Exclude
 		}
@@ -563,11 +590,13 @@ func runCnbBuild(config *cnbBuildOptions, cnbTelemetry *cnbBuildTelemetry, utils
 	metadata.WriteProjectMetadata(GeneralConfig.EnvRootPath, utils)
 
 	var buildpacksPath = "/cnb/buildpacks"
-	var orderPath = "/cnb/order.toml"
+	var orderPath = cnbutils.DefaultOrderPath
 
-	if config.Buildpacks != nil && len(config.Buildpacks) > 0 {
+	if len(config.Buildpacks) > 0 || len(config.PreBuildpacks) > 0 || len(config.PostBuildpacks) > 0 {
 		log.Entry().Infof("Setting custom buildpacks: '%v'", config.Buildpacks)
-		buildpacksPath, orderPath, err = setCustomBuildpacks(config.Buildpacks, config.DockerConfigJSON, utils)
+		log.Entry().Infof("Pre-buildpacks: '%v'", config.PreBuildpacks)
+		log.Entry().Infof("Post-buildpacks: '%v'", config.PostBuildpacks)
+		buildpacksPath, orderPath, err = setCustomBuildpacks(config.Buildpacks, config.PreBuildpacks, config.PostBuildpacks, config.DockerConfigJSON, utils)
 		defer func() { _ = utils.RemoveAll(buildpacksPath) }()
 		defer func() { _ = utils.RemoveAll(orderPath) }()
 		if err != nil {

diff --git a/cmd/cnbBuild_generated.go b/cmd/cnbBuild_generated.go
diff --git a/cmd/cnbBuild_test.go b/cmd/cnbBuild_test.go
@@ -25,23 +25,56 @@ import (
 const imageRegistry = "some-registry"
 
 func newCnbBuildTestsUtils() cnbutils.MockUtils {
+	imageStub := func(imageRef, target string) (v1.Image, error) {
+		fakeImage := &fake.FakeImage{}
+		var imageConfig v1.Config
+		switch imageRef {
+		case "pre-test":
+			imageConfig = v1.Config{
+				Labels: map[string]string{
+					"io.buildpacks.buildpackage.metadata": "{\"id\": \"pre-testbuildpack\", \"version\": \"0.0.1\"}",
+				},
+			}
+		case "post-test":
+			imageConfig = v1.Config{
+				Labels: map[string]string{
+					"io.buildpacks.buildpackage.metadata": "{\"id\": \"post-testbuildpack\", \"version\": \"0.0.1\"}",
+				},
+			}
+		default:
+			imageConfig = v1.Config{
+				Labels: map[string]string{
+					"io.buildpacks.buildpackage.metadata": "{\"id\": \"testbuildpack\", \"version\": \"0.0.1\"}",
+				},
+			}
+		}
+
+		fakeImage.ConfigFileReturns(&v1.ConfigFile{
+			Config: imageConfig,
+		}, nil)
+
+		return fakeImage, nil
+	}
+
 	utils := cnbutils.MockUtils{
 		ExecMockRunner: &mock.ExecMockRunner{},
 		FilesMock:      &mock.FilesMock{},
-		DownloadMock:   &mock.DownloadMock{},
-	}
-
-	fakeImage := &fake.FakeImage{}
-	fakeImage.ConfigFileReturns(&v1.ConfigFile{
-		Config: v1.Config{
-			Labels: map[string]string{
-				"io.buildpacks.buildpackage.metadata": "{\"id\": \"testbuildpack\", \"version\": \"0.0.1\"}",
+		DownloadMock: &mock.DownloadMock{
+			ImageContentStub: imageStub,
+			ImageInfoStub: func(imageRef string) (v1.Image, error) {
+				return imageStub(imageRef, "")
 			},
 		},
-	}, nil)
+	}
 
-	utils.RemoteImageInfo = fakeImage
-	utils.ReturnImage = fakeImage
+	utils.AddFile("/cnb/order.toml", []byte(`[[order]]
+    [[order.group]]
+      id = "buildpacks/java"
+      version = "1.8.0"
+[[order]]
+    [[order.group]]
+      id = "buildpacks/nodejs"
+      version = "1.6.0"`))
 	utils.AddFile("/layers/report.toml", []byte(`[build]
 [image]
 tags = ["localhost:5000/not-found:0.0.1"]
@@ -234,6 +267,75 @@ func TestRunCnbBuild(t *testing.T) {
 		assert.True(t, copiedFileExists)
 	})
 
+	t.Run("success case (custom buildpacks, pre and post buildpacks and custom env variables, renaming docker conf file, additional tag)", func(t *testing.T) {
+		t.Parallel()
+		config := cnbBuildOptions{
+			ContainerImageName:   "my-image",
+			ContainerImageTag:    "0.0.1",
+			ContainerRegistryURL: imageRegistry,
+			DockerConfigJSON:     "/path/to/test.json",
+			PreBuildpacks:        []string{"pre-test"},
+			PostBuildpacks:       []string{"post-test"},
+			Buildpacks:           []string{"test"},
+			BuildEnvVars: map[string]interface{}{
+				"FOO": "BAR",
+			},
+			AdditionalTags: []string{"latest"},
+		}
+
+		utils := newCnbBuildTestsUtils()
+		utils.FilesMock.AddFile(config.DockerConfigJSON, []byte(`{"auths":{"my-registry":{"auth":"dXNlcjpwYXNz"}}}`))
+		addBuilderFiles(&utils)
+
+		err := callCnbBuild(&config, &telemetry.CustomData{}, &utils, &cnbBuildCommonPipelineEnvironment{}, &piperhttp.Client{})
+
+		require.NoError(t, err)
+		runner := utils.ExecMockRunner
+		assert.Contains(t, runner.Env, "CNB_REGISTRY_AUTH={\"my-registry\":\"Basic dXNlcjpwYXNz\"}")
+		assert.Equal(t, creatorPath, runner.Calls[0].Exec)
+		assert.Contains(t, runner.Calls[0].Params, "/tmp/buildpacks")
+		assert.Contains(t, runner.Calls[0].Params, "/tmp/buildpacks/order.toml")
+		assert.Contains(t, runner.Calls[0].Params, fmt.Sprintf("%s/%s:%s", config.ContainerRegistryURL, config.ContainerImageName, config.ContainerImageTag))
+		assert.Contains(t, runner.Calls[0].Params, fmt.Sprintf("%s/%s:latest", config.ContainerRegistryURL, config.ContainerImageName))
+
+		copiedFileExists, _ := utils.FileExists("/tmp/config.json")
+		assert.True(t, copiedFileExists)
+	})
+
+	t.Run("success case (custom pre and post buildpacks and custom env variables, renaming docker conf file, additional tag)", func(t *testing.T) {
+		t.Parallel()
+		config := cnbBuildOptions{
+			ContainerImageName:   "my-image",
+			ContainerImageTag:    "0.0.1",
+			ContainerRegistryURL: imageRegistry,
+			DockerConfigJSON:     "/path/to/test.json",
+			PostBuildpacks:       []string{"post-test"},
+			PreBuildpacks:        []string{"pre-test"},
+			BuildEnvVars: map[string]interface{}{
+				"FOO": "BAR",
+			},
+			AdditionalTags: []string{"latest"},
+		}
+
+		utils := newCnbBuildTestsUtils()
+		utils.FilesMock.AddFile(config.DockerConfigJSON, []byte(`{"auths":{"my-registry":{"auth":"dXNlcjpwYXNz"}}}`))
+		addBuilderFiles(&utils)
+
+		err := callCnbBuild(&config, &telemetry.CustomData{}, &utils, &cnbBuildCommonPipelineEnvironment{}, &piperhttp.Client{})
+
+		require.NoError(t, err)
+		runner := utils.ExecMockRunner
+		assert.Contains(t, runner.Env, "CNB_REGISTRY_AUTH={\"my-registry\":\"Basic dXNlcjpwYXNz\"}")
+		assert.Equal(t, creatorPath, runner.Calls[0].Exec)
+		assert.Contains(t, runner.Calls[0].Params, "/tmp/buildpacks")
+		assert.Contains(t, runner.Calls[0].Params, "/tmp/buildpacks/order.toml")
+		assert.Contains(t, runner.Calls[0].Params, fmt.Sprintf("%s/%s:%s", config.ContainerRegistryURL, config.ContainerImageName, config.ContainerImageTag))
+		assert.Contains(t, runner.Calls[0].Params, fmt.Sprintf("%s/%s:latest", config.ContainerRegistryURL, config.ContainerImageName))
+
+		copiedFileExists, _ := utils.FileExists("/tmp/config.json")
+		assert.True(t, copiedFileExists)
+	})
+
 	t.Run("success case (customTlsCertificates)", func(t *testing.T) {
 		t.Parallel()
 		httpmock.Activate()

diff --git a/go.mod b/go.mod
@@ -9,6 +9,7 @@ replace golang.org/x/crypto v0.0.0-20220622213112-05595931fe9d => golang.org/x/c
 require (
 	cloud.google.com/go/storage v1.22.1
 	github.com/Azure/azure-sdk-for-go/sdk/storage/azblob v0.4.0
+	github.com/BurntSushi/toml v1.1.0
 	github.com/Jeffail/gabs/v2 v2.6.1
 	github.com/Masterminds/sprig v2.22.0+incompatible
 	github.com/antchfx/htmlquery v1.2.4
@@ -46,7 +47,6 @@ require (
 	github.com/mitchellh/mapstructure v1.5.0
 	github.com/motemen/go-nuts v0.0.0-20210915132349-615a782f2c69
 	github.com/package-url/packageurl-go v0.1.0
-	github.com/pelletier/go-toml v1.9.5
 	github.com/piper-validation/fortify-client-go v0.0.0-20220126145513-7b3e9a72af01
 	github.com/pkg/errors v0.9.1
 	github.com/sabhiram/go-gitignore v0.0.0-20210923224102-525f6e181f06
@@ -87,7 +87,6 @@ require (
 	github.com/Azure/go-autorest/autorest/validation v0.3.1 // indirect
 	github.com/Azure/go-autorest/logger v0.2.1 // indirect
 	github.com/Azure/go-autorest/tracing v0.6.0 // indirect
-	github.com/BurntSushi/toml v1.1.0 // indirect
 	github.com/CycloneDX/cyclonedx-go v0.6.0
 	github.com/DataDog/datadog-go v3.2.0+incompatible // indirect
 	github.com/Jeffail/gabs v1.1.1 // indirect

diff --git a/go.sum b/go.sum
@@ -1707,8 +1707,6 @@ github.com/pelletier/go-toml v1.7.0/go.mod h1:vwGMzjaWMwyfHwgIBhI2YUM4fB6nL6lVAv
 github.com/pelletier/go-toml v1.8.1/go.mod h1:T2/BmBdy8dvIRq1a/8aqjN41wvWlN4lrapLU/GW4pbc=
 github.com/pelletier/go-toml v1.9.3/go.mod h1:u1nR/EPcESfeI/szUZKdtJ0xRNbUoANCkoOuaOx1Y+c=
 github.com/pelletier/go-toml v1.9.4/go.mod h1:u1nR/EPcESfeI/szUZKdtJ0xRNbUoANCkoOuaOx1Y+c=
-github.com/pelletier/go-toml v1.9.5 h1:4yBQzkHv+7BHq2PQUZF3Mx0IYxG7LsP222s7Agd3ve8=
-github.com/pelletier/go-toml v1.9.5/go.mod h1:u1nR/EPcESfeI/szUZKdtJ0xRNbUoANCkoOuaOx1Y+c=
 github.com/peterbourgon/diskv v2.0.1+incompatible h1:UBdAOUP5p4RWqPBg048CAvpKN+vxiaj6gdUUzhl4XmI=
 github.com/peterbourgon/diskv v2.0.1+incompatible/go.mod h1:uqqh8zWWbv1HBMNONnaR/tNboyR3/BZd58JJSHlUSCU=
 github.com/petermattis/goid v0.0.0-20180202154549-b0b1615b78e5 h1:q2e307iGHPdTGp0hoxKjt1H5pDo6utceo3dQVK3I5XQ=

diff --git a/integration/github_actions_integration_test_list.yml b/integration/github_actions_integration_test_list.yml
@@ -9,6 +9,7 @@ run:
   - '"TestCNBIntegrationNPMCustomBuildpacksBuildpacklessProject"'
   - '"TestCNBIntegrationNPMCustomBuildpacksFullProject"'
   - '"TestCNBIntegrationProjectDescriptor"'
+  - '"TestCNBIntegrationPrePostBuildpacks"'
 
   - '"TestGolangIntegration"'
   - '"TestGradleIntegration"'

diff --git a/integration/integration_cnb_test.go b/integration/integration_cnb_test.go
@@ -17,7 +17,7 @@ import (
 
 const (
 	registryURL = "localhost:5000"
-	baseBuilder = "paketobuildpacks/builder:0.3.26-base"
+	baseBuilder = "paketobuildpacks/builder:0.3.280-base"
 )
 
 func setupDockerRegistry(t *testing.T, ctx context.Context) testcontainers.Container {
@@ -135,7 +135,7 @@ func TestCNBIntegrationZipPath(t *testing.T) {
 	container.assertHasOutput(t,
 		"running command: /cnb/lifecycle/creator",
 		"Installing Go",
-		"Paketo Go Build Buildpack",
+		"Paketo Buildpack for Go Build",
 		fmt.Sprintf("Saving %s/not-found:0.0.1", registryURL),
 		"*** Images (sha256:",
 		"SUCCESS",
@@ -279,9 +279,9 @@ func TestCNBIntegrationMultiImage(t *testing.T) {
 	assert.NoError(t, err)
 
 	container.assertHasOutput(t,
-		"Previous image with name \"localhost:5000/io-buildpacks-my-app:latest\" not found",
+		"Image with name \"localhost:5000/io-buildpacks-my-app:latest\" not found",
 		"Saving localhost:5000/io-buildpacks-my-app:latest...",
-		"Previous image with name \"localhost:5000/go-app:v1.0.0\" not found",
+		"Image with name \"localhost:5000/go-app:v1.0.0\" not found",
 		"Saving localhost:5000/go-app:v1.0.0...",
 		"Using cached buildpack",
 		"Saving localhost:5000/my-app2:latest...",
@@ -334,3 +334,27 @@ func TestCNBIntegrationPreserveFilesIgnored(t *testing.T) {
 	container.assertHasOutput(t, "skipping preserving files because the source")
 	container.terminate(t)
 }
+
+func TestCNBIntegrationPrePostBuildpacks(t *testing.T) {
+	t.Parallel()
+	ctx := context.Background()
+	registryContainer := setupDockerRegistry(t, ctx)
+	defer registryContainer.Terminate(ctx)
+
+	container := givenThisContainer(t, IntegrationTestDockerExecRunnerBundle{
+		Image:   baseBuilder,
+		User:    "cnb",
+		TestDir: []string{"testdata", "TestCnbIntegration"},
+		Network: fmt.Sprintf("container:%s", registryContainer.GetContainerID()),
+		Environment: map[string]string{
+			"PIPER_VAULTCREDENTIAL_DYNATRACE_API_KEY": "api-key-content",
+		},
+	})
+
+	err := container.whenRunningPiperCommand("cnbBuild", "--noTelemetry", "--verbose", "--projectDescriptor", "", "--path", "project", "--customConfig", "config.yml", "--containerImageTag", "0.0.1", "--containerImageName", "not-found", "--containerRegistryUrl", registryURL, "--postBuildpacks", "paketobuildpacks/datadog")
+	assert.NoError(t, err)
+	container.assertHasOutput(t, "Setting custom buildpacks: '[]'")
+	container.assertHasOutput(t, "Pre-buildpacks: '[]'")
+	container.assertHasOutput(t, "Post-buildpacks: '[paketobuildpacks/datadog]'")
+	container.terminate(t)
+}
diff --git a/integration/testdata/TestCnbIntegration/config.yml b/integration/testdata/TestCnbIntegration/config.yml
@@ -3,6 +3,9 @@ general:
   collectTelemetryData: false
 steps:
   cnbBuild:
+    buildEnvVars:
+      BP_DATADOG_ENABLED: true
+      BP_EAR_KEY: 74657374
     bindings:
       maven-settings:
         type: maven

diff --git a/integration/testdata/TestCnbIntegration/project/package.json b/integration/testdata/TestCnbIntegration/project/package.json
@@ -1,6 +1,7 @@
 {
     "name": "test-mta-js",
     "version": "1.0.0",
+    "main": "srv/hello.js",
     "dependencies": {
         "jest": "^26.0.1",
         "jest-jenkins-reporter": "^1.0.2"