From 4d863084338b186cafa1dbac6cf260749b477551 Mon Sep 17 00:00:00 2001 From: Pavel Busko Date: Thu, 14 Mar 2024 15:39:30 +0100 Subject: [PATCH] chore(cnbBuild): simplify telemetry data (#4864) Co-authored-by: Johannes Dillmann --- cmd/cnbBuild.go | 26 ++---- cmd/cnbBuild_test.go | 126 +++----------------------- pkg/buildpacks/telemetry.go | 151 +++----------------------------- pkg/telemetry/data.go | 3 +- pkg/telemetry/telemetry_test.go | 3 +- 5 files changed, 31 insertions(+), 278 deletions(-) diff --git a/cmd/cnbBuild.go b/cmd/cnbBuild.go index ecf6581551..f5d6a3e76a 100644 --- a/cmd/cnbBuild.go +++ b/cmd/cnbBuild.go @@ -296,8 +296,10 @@ func callCnbBuild(config *cnbBuildOptions, telemetryData *telemetry.CustomData, if err != nil { log.Entry().Warnf("failed to retrieve dockerImage configuration: '%v'", err) } + telemetry.WithBuilder(dockerImage) - telemetry.WithImage(dockerImage) + buildTool, _ := getBuildToolFromStageConfig("cnbBuild") + telemetry.WithBuildTool(buildTool) cnbBuildConfig := buildsettings.BuildOptions{ CreateBOM: config.CreateBOM, @@ -351,10 +353,12 @@ func callCnbBuild(config *cnbBuildOptions, telemetryData *telemetry.CustomData, } } - return telemetry.Export() + return nil } func runCnbBuild(config *cnbBuildOptions, telemetry *buildpacks.Telemetry, imageSummary *cnbutils.ImageSummary, utils cnbutils.BuildUtils, commonPipelineEnvironment *cnbBuildCommonPipelineEnvironment, httpClient piperhttp.Sender) error { + telemetry.WithRunImage(config.RunImage) + err := cleanDir("/layers", utils) if err != nil { log.SetErrorCategory(log.ErrorBuild) @@ -388,8 +392,6 @@ func runCnbBuild(config *cnbBuildOptions, telemetry *buildpacks.Telemetry, image } config.BuildEnvVars["TMPDIR"] = tempdir - telemetrySegment := createInitialTelemetrySegment(config, utils) - include := ignore.CompileIgnoreLines("**/*") exclude := ignore.CompileIgnoreLines("piper", ".pipeline", ".git") @@ -407,7 +409,6 @@ func runCnbBuild(config *cnbBuildOptions, telemetry *buildpacks.Telemetry, image log.SetErrorCategory(log.ErrorConfiguration) return errors.Wrapf(err, "failed to parse %s", projDescPath) } - telemetrySegment.WithProjectDescriptor(descriptor) config.mergeEnvVars(descriptor.EnvVars) @@ -440,8 +441,6 @@ func runCnbBuild(config *cnbBuildOptions, telemetry *buildpacks.Telemetry, image return errors.Wrap(err, "failed to retrieve target image configuration") } - telemetry.AddSegment(telemetrySegment.WithBuildpacksOverall(config.Buildpacks).WithKeyValues(config.BuildEnvVars)) - if commonPipelineEnvironment.container.imageNameTag == "" { commonPipelineEnvironment.container.registryURL = fmt.Sprintf("%s://%s", targetImage.ContainerRegistry.Scheme, targetImage.ContainerRegistry.Host) commonPipelineEnvironment.container.imageNameTag = fmt.Sprintf("%v:%v", targetImage.ContainerImageName, targetImage.ContainerImageTag) @@ -627,16 +626,3 @@ func expandEnvVars(envVars map[string]any) map[string]any { } return expandedEnvVars } - -func createInitialTelemetrySegment(config *cnbBuildOptions, utils cnbutils.BuildUtils) *buildpacks.Segment { - telemetrySegment := buildpacks.NewSegment() - projectPath, _, _ := config.resolvePath(utils) // ignore error here, telemetry problems should not fail the build - buildTool, _ := getBuildToolFromStageConfig("cnbBuild") // ignore error here, telemetry problems should not fail the build - - return telemetrySegment.WithBindings(config.Bindings). - WithTags(config.ContainerImageTag, config.AdditionalTags). - WithPath(projectPath). - WithEnv(config.BuildEnvVars). - WithBuildTool(buildTool). - WithBuildpacksFromConfig(config.Buildpacks) -} diff --git a/cmd/cnbBuild_test.go b/cmd/cnbBuild_test.go index dfabbd47e0..0fe38a8c7a 100644 --- a/cmd/cnbBuild_test.go +++ b/cmd/cnbBuild_test.go @@ -4,13 +4,11 @@ package cmd import ( - "encoding/json" "fmt" "net/http" "path/filepath" "testing" - "github.com/SAP/jenkins-library/pkg/buildpacks" "github.com/SAP/jenkins-library/pkg/cnbutils" piperconf "github.com/SAP/jenkins-library/pkg/config" piperhttp "github.com/SAP/jenkins-library/pkg/http" @@ -179,13 +177,6 @@ func TestRunCnbBuild(t *testing.T) { assert.Equal(t, "sha256:52eac630560210e5ae13eb10797c4246d6f02d425f32b9430ca00bde697c79ec", commonPipelineEnvironment.container.imageDigest) assert.Contains(t, commonPipelineEnvironment.container.imageDigests, "sha256:52eac630560210e5ae13eb10797c4246d6f02d425f32b9430ca00bde697c79ec") - - customDataAsString := telemetryData.CnbBuildStepData - customData := &buildpacks.BuildpacksTelemetry{} - err = json.Unmarshal([]byte(customDataAsString), customData) - require.NoError(t, err) - assert.Equal(t, 1, len(customData.Data)) - assert.Equal(t, "root", string(customData.Data[0].Path)) }) t.Run("success case (registry with https)", func(t *testing.T) { @@ -584,58 +575,25 @@ func TestRunCnbBuild(t *testing.T) { ContainerImageName: "my-image", ContainerImageTag: "3.1.5", ContainerRegistryURL: registry, - DockerConfigJSON: "/path/to/config.json", - ProjectDescriptor: "project.toml", - AdditionalTags: []string{"latest"}, - Buildpacks: []string{"paketobuildpacks/java", "gcr.io/paketo-buildpacks/node"}, - Bindings: map[string]interface{}{"SECRET": map[string]string{"key": "KEY", "file": "a_file"}}, - Path: "target", + MultipleImages: []map[string]interface{}{ + { + "runImage": "foo", + }, + { + "runImage": "bar", + }, + }, } utils := newCnbBuildTestsUtils() - utils.FilesMock.AddFile(config.DockerConfigJSON, []byte(`{"auths":{"my-registry":{"auth":"dXNlcjpwYXNz"}}}`)) - utils.FilesMock.AddDir("target") - utils.FilesMock.AddFile("target/project.toml", []byte(`[project] -id = "test" -name = "test" -version = "1.0.0" - -[build] -include = [] -exclude = ["*.tar"] - -[[build.buildpacks]] -uri = "some-buildpack"`)) - utils.FilesMock.AddFile("a_file", []byte(`{}`)) - utils.FilesMock.AddFile("target/somelib.jar", []byte(`FFFFFF`)) - addBuilderFiles(&utils) telemetryData := &telemetry.CustomData{} err := callCnbBuild(&config, telemetryData, &utils, &cnbBuildCommonPipelineEnvironment{}, &piperhttp.Client{}) require.NoError(t, err) - customDataAsString := telemetryData.CnbBuildStepData - customData := &buildpacks.BuildpacksTelemetry{} - err = json.Unmarshal([]byte(customDataAsString), customData) - - require.NoError(t, err) - assert.Equal(t, 3, customData.Version) - require.Equal(t, 1, len(customData.Data)) - assert.Equal(t, "3.1.5", customData.Data[0].ImageTag) - assert.Equal(t, "folder", string(customData.Data[0].Path)) - assert.Contains(t, customData.Data[0].AdditionalTags, "latest") - assert.Contains(t, customData.Data[0].BindingKeys, "SECRET") - assert.Equal(t, "paketobuildpacks/builder-jammy-base:latest", customData.Data[0].Builder) - - assert.Contains(t, customData.Data[0].Buildpacks.FromConfig, "paketobuildpacks/java") - assert.NotContains(t, customData.Data[0].Buildpacks.FromProjectDescriptor, "paketobuildpacks/java") - assert.Contains(t, customData.Data[0].Buildpacks.FromProjectDescriptor, "bcc73ab1f0a0d3fb0d1bf2b6df5510a25ccd14a761dbc0f5044ea24ead30452b") - assert.Contains(t, customData.Data[0].Buildpacks.Overall, "paketobuildpacks/java") - - assert.True(t, customData.Data[0].ProjectDescriptor.Used) - assert.False(t, customData.Data[0].ProjectDescriptor.IncludeUsed) - assert.True(t, customData.Data[0].ProjectDescriptor.ExcludeUsed) + assert.Equal(t, "paketobuildpacks/builder-jammy-base:latest", telemetryData.CnbBuilder) + assert.Equal(t, "foo,bar", telemetryData.CnbRunImage) }) t.Run("error case, multiple artifacts in path", func(t *testing.T) { @@ -696,63 +654,6 @@ uri = "some-buildpack"`)) assert.Equal(t, "my-image:3.1.5", commonPipelineEnvironment.container.imageNameTag) }) - t.Run("success case (build env telemetry was added)", func(t *testing.T) { - t.Parallel() - registry := "some-registry" - config := cnbBuildOptions{ - ContainerImageName: "my-image", - ContainerImageTag: "3.1.5", - ContainerRegistryURL: registry, - ProjectDescriptor: "project.toml", - BuildEnvVars: map[string]interface{}{"CONFIG_KEY": "var", "BP_JVM_VERSION": "8"}, - } - - utils := newCnbBuildTestsUtils() - utils.FilesMock.AddFile("project.toml", []byte(`[project] -id = "test" - -[build] -include = [] - -[[build.env]] -name='PROJECT_KEY' -value='var' - -[[build.env]] -name='BP_NODE_VERSION' -value='11' - -[[build.buildpacks]] -uri = "some-buildpack" -`)) - - addBuilderFiles(&utils) - - telemetryData := &telemetry.CustomData{} - err := callCnbBuild(&config, telemetryData, &utils, &cnbBuildCommonPipelineEnvironment{}, &piperhttp.Client{}) - require.NoError(t, err) - - customDataAsString := telemetryData.CnbBuildStepData - customData := &buildpacks.BuildpacksTelemetry{} - err = json.Unmarshal([]byte(customDataAsString), customData) - - require.NoError(t, err) - require.Equal(t, 1, len(customData.Data)) - assert.Contains(t, customData.Data[0].BuildEnv.KeysFromConfig, "CONFIG_KEY") - assert.NotContains(t, customData.Data[0].BuildEnv.KeysFromProjectDescriptor, "CONFIG_KEY") - assert.Contains(t, customData.Data[0].BuildEnv.KeysOverall, "CONFIG_KEY") - - assert.NotContains(t, customData.Data[0].BuildEnv.KeysFromConfig, "PROJECT_KEY") - assert.Contains(t, customData.Data[0].BuildEnv.KeysFromProjectDescriptor, "PROJECT_KEY") - assert.Contains(t, customData.Data[0].BuildEnv.KeysOverall, "PROJECT_KEY") - - assert.Equal(t, "8", customData.Data[0].BuildEnv.KeyValues["BP_JVM_VERSION"]) - assert.Equal(t, "11", customData.Data[0].BuildEnv.KeyValues["BP_NODE_VERSION"]) - assert.NotContains(t, customData.Data[0].BuildEnv.KeyValues, "PROJECT_KEY") - - assert.Contains(t, customData.Data[0].Buildpacks.Overall, "bcc73ab1f0a0d3fb0d1bf2b6df5510a25ccd14a761dbc0f5044ea24ead30452b") - }) - t.Run("success case (multiple images configured)", func(t *testing.T) { t.Parallel() commonPipelineEnvironment := cnbBuildCommonPipelineEnvironment{} @@ -774,12 +675,6 @@ uri = "some-buildpack" err := callCnbBuild(&config, telemetryData, &utils, &commonPipelineEnvironment, &piperhttp.Client{}) require.NoError(t, err) - customDataAsString := telemetryData.CnbBuildStepData - customData := &buildpacks.BuildpacksTelemetry{} - err = json.Unmarshal([]byte(customDataAsString), customData) - assert.NoError(t, err) - require.Equal(t, expectedImageCount, len(customData.Data)) - runner := utils.ExecMockRunner require.Equal(t, expectedImageCount, len(runner.Calls)-1) for i, call := range runner.Calls { @@ -787,7 +682,6 @@ uri = "some-buildpack" continue } lifecycleCall := i - 1 - assert.Equal(t, 4, len(customData.Data[lifecycleCall].AdditionalTags)) assertLifecycleCalls(t, runner, i+1) containerImageName := fmt.Sprintf("my-image-%d", lifecycleCall) assert.Contains(t, call.Params, fmt.Sprintf("%s/%s:%s", config.ContainerRegistryURL, containerImageName, config.ContainerImageTag)) diff --git a/pkg/buildpacks/telemetry.go b/pkg/buildpacks/telemetry.go index c60cd70cd6..9ef7271326 100644 --- a/pkg/buildpacks/telemetry.go +++ b/pkg/buildpacks/telemetry.go @@ -1,163 +1,34 @@ package buildpacks import ( - "encoding/json" + "fmt" - "github.com/SAP/jenkins-library/pkg/cnbutils/privacy" - "github.com/SAP/jenkins-library/pkg/cnbutils/project" "github.com/SAP/jenkins-library/pkg/telemetry" - "github.com/pkg/errors" ) -const version = 3 - type Telemetry struct { customData *telemetry.CustomData - data *BuildpacksTelemetry + runImages []string } func NewTelemetry(customData *telemetry.CustomData) *Telemetry { return &Telemetry{ customData: customData, - data: &BuildpacksTelemetry{ - Version: version, - }, } } -func (d *Telemetry) Export() error { - customData, err := json.Marshal(d.data) - if err != nil { - return errors.Wrap(err, "failed to marshal custom telemetry data") - } - d.customData.CnbBuildStepData = string(customData) - return nil -} - -func (d *Telemetry) WithImage(image string) { - d.data.builder = image -} - -func (d *Telemetry) AddSegment(segment *Segment) { - segment.data.Builder = d.data.builder - d.data.Data = append(d.data.Data, segment.data) -} - -type BuildpacksTelemetry struct { - builder string - Version int `json:"version"` - Data []*cnbBuildTelemetryData `json:"data"` -} - -type cnbBuildTelemetryData struct { - ImageTag string `json:"imageTag"` - AdditionalTags []string `json:"additionalTags"` - BindingKeys []string `json:"bindingKeys"` - Path PathEnum `json:"path"` - BuildEnv cnbBuildTelemetryDataBuildEnv `json:"buildEnv"` - Buildpacks cnbBuildTelemetryDataBuildpacks `json:"buildpacks"` - ProjectDescriptor cnbBuildTelemetryDataProjectDescriptor `json:"projectDescriptor"` - BuildTool string `json:"buildTool"` - Builder string `json:"builder"` -} - -type cnbBuildTelemetryDataBuildEnv struct { - KeysFromConfig []string `json:"keysFromConfig"` - KeysFromProjectDescriptor []string `json:"keysFromProjectDescriptor"` - KeysOverall []string `json:"keysOverall"` - JVMVersion string `json:"jvmVersion"` - KeyValues map[string]interface{} `json:"keyValues"` -} - -type cnbBuildTelemetryDataBuildpacks struct { - FromConfig []string `json:"FromConfig"` - FromProjectDescriptor []string `json:"FromProjectDescriptor"` - Overall []string `json:"overall"` -} - -type cnbBuildTelemetryDataProjectDescriptor struct { - Used bool `json:"used"` - IncludeUsed bool `json:"includeUsed"` - ExcludeUsed bool `json:"excludeUsed"` -} - -type Segment struct { - data *cnbBuildTelemetryData -} - -func NewSegment() *Segment { - return &Segment{ - data: &cnbBuildTelemetryData{}, - } -} - -func (s *Segment) WithBindings(bindings map[string]interface{}) *Segment { - var bindingKeys []string - for k := range bindings { - bindingKeys = append(bindingKeys, k) - } - s.data.BindingKeys = bindingKeys - return s -} - -func (s *Segment) WithEnv(env map[string]interface{}) *Segment { - s.data.BuildEnv.KeysFromConfig = []string{} - s.data.BuildEnv.KeysOverall = []string{} - for key := range env { - s.data.BuildEnv.KeysFromConfig = append(s.data.BuildEnv.KeysFromConfig, key) - s.data.BuildEnv.KeysOverall = append(s.data.BuildEnv.KeysOverall, key) - } - return s -} - -func (s *Segment) WithTags(tag string, additionalTags []string) *Segment { - s.data.ImageTag = tag - s.data.AdditionalTags = additionalTags - return s -} - -func (s *Segment) WithPath(path PathEnum) *Segment { - s.data.Path = path - return s -} - -func (s *Segment) WithBuildTool(buildTool string) *Segment { - s.data.BuildTool = buildTool - return s -} - -func (s *Segment) WithBuilder(builder string) *Segment { - s.data.Builder = privacy.FilterBuilder(builder) - return s -} - -func (s *Segment) WithBuildpacksFromConfig(buildpacks []string) *Segment { - s.data.Buildpacks.FromConfig = privacy.FilterBuildpacks(buildpacks) - return s -} - -func (s *Segment) WithBuildpacksOverall(buildpacks []string) *Segment { - s.data.Buildpacks.Overall = privacy.FilterBuildpacks(buildpacks) - return s +func (d *Telemetry) WithBuilder(builder string) { + d.customData.CnbBuilder = builder } -func (s *Segment) WithKeyValues(env map[string]interface{}) *Segment { - s.data.BuildEnv.KeyValues = privacy.FilterEnv(env) - return s +func (d *Telemetry) WithBuildTool(buildTool string) { + d.customData.BuildTool = buildTool } -func (s *Segment) WithProjectDescriptor(descriptor *project.Descriptor) *Segment { - descriptorKeys := s.data.BuildEnv.KeysFromProjectDescriptor - overallKeys := s.data.BuildEnv.KeysOverall - for key := range descriptor.EnvVars { - descriptorKeys = append(descriptorKeys, key) - overallKeys = append(overallKeys, key) +func (d *Telemetry) WithRunImage(runImage string) { + if d.customData.CnbRunImage == "" { + d.customData.CnbRunImage = runImage + } else { + d.customData.CnbRunImage += fmt.Sprintf(",%s", runImage) } - s.data.BuildEnv.KeysFromProjectDescriptor = descriptorKeys - s.data.BuildEnv.KeysOverall = overallKeys - s.data.Buildpacks.FromProjectDescriptor = privacy.FilterBuildpacks(descriptor.Buildpacks) - s.data.ProjectDescriptor.Used = true - s.data.ProjectDescriptor.IncludeUsed = descriptor.Include != nil - s.data.ProjectDescriptor.ExcludeUsed = descriptor.Exclude != nil - return s } diff --git a/pkg/telemetry/data.go b/pkg/telemetry/data.go index 7edc63e377..c0bc3aee43 100644 --- a/pkg/telemetry/data.go +++ b/pkg/telemetry/data.go @@ -33,7 +33,8 @@ type CustomData struct { LegacyJobNameTemplate string `json:"legacyJobNameTemplate,omitempty"` LegacyJobName string `json:"legacyJobName,omitempty"` DeployType string `json:"deployType,omitempty"` - CnbBuildStepData string `json:"cnbBuildStepData,omitempty"` + CnbBuilder string `json:"cnbBuilder,omitempty"` + CnbRunImage string `json:"cnbRunImage,omitempty"` ServerURL string `json:"serverURL,omitempty"` ECCNMessageStatus string `json:"eccnMessageStatus,omitempty"` ChangeRequestUpload string `json:"changeRequestUpload,omitempty"` diff --git a/pkg/telemetry/telemetry_test.go b/pkg/telemetry/telemetry_test.go index 839e2ffdd2..b8d8411e7d 100644 --- a/pkg/telemetry/telemetry_test.go +++ b/pkg/telemetry/telemetry_test.go @@ -231,7 +231,8 @@ func TestSetData(t *testing.T) { LegacyJobNameTemplate: "", LegacyJobName: "", DeployType: "", - CnbBuildStepData: "", + CnbBuilder: "", + CnbRunImage: "", IsScheduled: false, IsOptimized: false, },