From 6cc6a4e80a383f9efd28ad464e171d749549cff3 Mon Sep 17 00:00:00 2001 From: Dmitrii Pavlukhin Date: Fri, 5 Jan 2024 18:23:55 +0300 Subject: [PATCH] Feature - whitesourceExecuteScan - adding ability to scan multiple docker images (#4755) * added-multiple-images-scan-logic * amended-description * added-reference-to-common-pipeline-env --- cmd/whitesourceExecuteScan.go | 117 +++++++++++------- cmd/whitesourceExecuteScan_generated.go | 27 ++++ pkg/whitesource/configHelper.go | 9 +- pkg/whitesource/scanOptions.go | 2 + pkg/whitesource/scanUA.go | 6 +- .../metadata/whitesourceExecuteScan.yaml | 19 +++ 6 files changed, 131 insertions(+), 49 deletions(-) diff --git a/cmd/whitesourceExecuteScan.go b/cmd/whitesourceExecuteScan.go index 7a195f0d0a..ff388f235b 100644 --- a/cmd/whitesourceExecuteScan.go +++ b/cmd/whitesourceExecuteScan.go @@ -196,28 +196,25 @@ func runWhitesourceExecuteScan(ctx context.Context, config *ScanOptions, scan *w } func runWhitesourceScan(ctx context.Context, config *ScanOptions, scan *ws.Scan, utils whitesourceUtils, sys whitesource, commonPipelineEnvironment *whitesourceExecuteScanCommonPipelineEnvironment, influx *whitesourceExecuteScanInflux) error { + // Download Docker image for container scan // ToDo: move it to improve testability if config.BuildTool == "docker" { - saveImageOptions := containerSaveImageOptions{ - ContainerImage: config.ScanImage, - ContainerRegistryURL: config.ScanImageRegistryURL, - ContainerRegistryUser: config.ContainerRegistryUser, - ContainerRegistryPassword: config.ContainerRegistryPassword, - DockerConfigJSON: config.DockerConfigJSON, - FilePath: config.ProjectName, - ImageFormat: "legacy", // keep the image format legacy or whitesource is not able to read layers - } - dClientOptions := piperDocker.ClientOptions{ImageName: saveImageOptions.ContainerImage, RegistryURL: saveImageOptions.ContainerRegistryURL, LocalPath: "", ImageFormat: "legacy"} - dClient := &piperDocker.Client{} - dClient.SetOptions(dClientOptions) - if _, err := runContainerSaveImage(&saveImageOptions, &telemetry.CustomData{}, "./cache", "", dClient, utils); err != nil { - if strings.Contains(fmt.Sprint(err), "no image found") { - log.SetErrorCategory(log.ErrorConfiguration) + if len(config.ScanImages) != 0 { + for _, image := range config.ScanImages { + config.ScanImage = image + err := downloadDockerImageAsTar(config, utils) + if err != nil { + return errors.Wrapf(err, "failed to download docker image") + } } - return errors.Wrapf(err, "failed to download Docker image %v", config.ScanImage) - } + } else { + err := downloadDockerImageAsTar(config, utils) + if err != nil { + return errors.Wrapf(err, "failed to download docker image") + } + } } // Start the scan @@ -375,8 +372,11 @@ func resolveProjectIdentifiers(config *ScanOptions, scan *ws.Scan, utils whiteso if err := resolveProductToken(config, sys); err != nil { return errors.Wrap(err, "error resolving product token") } - if err := resolveAggregateProjectToken(config, sys); err != nil { - return errors.Wrap(err, "error resolving aggregate project token") + + if !config.SkipParentProjectResolution { + if err := resolveAggregateProjectToken(config, sys); err != nil { + return errors.Wrap(err, "error resolving aggregate project token") + } } scan.ProductToken = config.ProductToken @@ -465,33 +465,34 @@ func validateProductVersion(version string) string { func wsScanOptions(config *ScanOptions) *ws.ScanOptions { return &ws.ScanOptions{ - BuildTool: config.BuildTool, - ScanType: "", // no longer provided via config - OrgToken: config.OrgToken, - UserToken: config.UserToken, - ProductName: config.ProductName, - ProductToken: config.ProductToken, - ProductVersion: config.Version, - ProjectName: config.ProjectName, - BuildDescriptorFile: config.BuildDescriptorFile, - BuildDescriptorExcludeList: config.BuildDescriptorExcludeList, - PomPath: config.BuildDescriptorFile, - M2Path: config.M2Path, - GlobalSettingsFile: config.GlobalSettingsFile, - ProjectSettingsFile: config.ProjectSettingsFile, - InstallArtifacts: config.InstallArtifacts, - DefaultNpmRegistry: config.DefaultNpmRegistry, - AgentDownloadURL: config.AgentDownloadURL, - AgentFileName: config.AgentFileName, - ConfigFilePath: config.ConfigFilePath, - Includes: config.Includes, - Excludes: config.Excludes, - JreDownloadURL: config.JreDownloadURL, - AgentURL: config.AgentURL, - ServiceURL: config.ServiceURL, - ScanPath: config.ScanPath, - InstallCommand: config.InstallCommand, - Verbose: GeneralConfig.Verbose, + BuildTool: config.BuildTool, + ScanType: "", // no longer provided via config + OrgToken: config.OrgToken, + UserToken: config.UserToken, + ProductName: config.ProductName, + ProductToken: config.ProductToken, + ProductVersion: config.Version, + ProjectName: config.ProjectName, + BuildDescriptorFile: config.BuildDescriptorFile, + BuildDescriptorExcludeList: config.BuildDescriptorExcludeList, + PomPath: config.BuildDescriptorFile, + M2Path: config.M2Path, + GlobalSettingsFile: config.GlobalSettingsFile, + ProjectSettingsFile: config.ProjectSettingsFile, + InstallArtifacts: config.InstallArtifacts, + DefaultNpmRegistry: config.DefaultNpmRegistry, + AgentDownloadURL: config.AgentDownloadURL, + AgentFileName: config.AgentFileName, + ConfigFilePath: config.ConfigFilePath, + Includes: config.Includes, + Excludes: config.Excludes, + JreDownloadURL: config.JreDownloadURL, + AgentURL: config.AgentURL, + ServiceURL: config.ServiceURL, + ScanPath: config.ScanPath, + InstallCommand: config.InstallCommand, + Verbose: GeneralConfig.Verbose, + SkipParentProjectResolution: config.SkipParentProjectResolution, } } @@ -1086,3 +1087,27 @@ func createToolRecordWhitesource(utils whitesourceUtils, workspace string, confi } return record.GetFileName(), nil } + +func downloadDockerImageAsTar(config *ScanOptions, utils whitesourceUtils) error { + + saveImageOptions := containerSaveImageOptions{ + ContainerImage: config.ScanImage, + ContainerRegistryURL: config.ScanImageRegistryURL, + ContainerRegistryUser: config.ContainerRegistryUser, + ContainerRegistryPassword: config.ContainerRegistryPassword, + DockerConfigJSON: config.DockerConfigJSON, + FilePath: config.ScanPath + "/" + config.ScanImage, // previously was config.ProjectName + ImageFormat: "legacy", // keep the image format legacy or whitesource is not able to read layers + } + dClientOptions := piperDocker.ClientOptions{ImageName: saveImageOptions.ContainerImage, RegistryURL: saveImageOptions.ContainerRegistryURL, LocalPath: "", ImageFormat: "legacy"} + dClient := &piperDocker.Client{} + dClient.SetOptions(dClientOptions) + if _, err := runContainerSaveImage(&saveImageOptions, &telemetry.CustomData{}, "./cache", "", dClient, utils); err != nil { + if strings.Contains(fmt.Sprint(err), "no image found") { + log.SetErrorCategory(log.ErrorConfiguration) + } + return errors.Wrapf(err, "failed to download Docker image %v", config.ScanImage) + } + + return nil +} diff --git a/cmd/whitesourceExecuteScan_generated.go b/cmd/whitesourceExecuteScan_generated.go index d034bf7f66..df6ff9227b 100644 --- a/cmd/whitesourceExecuteScan_generated.go +++ b/cmd/whitesourceExecuteScan_generated.go @@ -54,6 +54,8 @@ type whitesourceExecuteScanOptions struct { ProjectToken string `json:"projectToken,omitempty"` Reporting bool `json:"reporting,omitempty"` ScanImage string `json:"scanImage,omitempty"` + ScanImages []string `json:"scanImages,omitempty"` + SkipParentProjectResolution bool `json:"skipParentProjectResolution,omitempty"` ScanImageRegistryURL string `json:"scanImageRegistryUrl,omitempty"` SecurityVulnerabilities bool `json:"securityVulnerabilities,omitempty"` ServiceURL string `json:"serviceUrl,omitempty"` @@ -349,6 +351,8 @@ func addWhitesourceExecuteScanFlags(cmd *cobra.Command, stepConfig *whitesourceE cmd.Flags().StringVar(&stepConfig.ProjectToken, "projectToken", os.Getenv("PIPER_projectToken"), "Project token to execute scan on. Ignored for scan types `maven`, `mta` and `npm`. Used for project aggregation when scanning with the Unified Agent and can be provided as an alternative to `projectName`.") cmd.Flags().BoolVar(&stepConfig.Reporting, "reporting", true, "Whether assessment is being done at all, defaults to `true`") cmd.Flags().StringVar(&stepConfig.ScanImage, "scanImage", os.Getenv("PIPER_scanImage"), "For `buildTool: docker`: Defines the docker image which should be scanned.") + cmd.Flags().StringSliceVar(&stepConfig.ScanImages, "scanImages", []string{}, "For `buildTool: docker`: Allowing to scan multiple docker images. In case parent project will not contain any dependecies, use skipParentProjectResolution parameter") + cmd.Flags().BoolVar(&stepConfig.SkipParentProjectResolution, "skipParentProjectResolution", false, "Parameter for multi-module, multi-images projects to skip the parent project resolution for reporing purpose Could be used if parent project is set as just a placeholder for scan and doesn't contain any dependencies.") cmd.Flags().StringVar(&stepConfig.ScanImageRegistryURL, "scanImageRegistryUrl", os.Getenv("PIPER_scanImageRegistryUrl"), "For `buildTool: docker`: Defines the registry where the scanImage is located.") cmd.Flags().BoolVar(&stepConfig.SecurityVulnerabilities, "securityVulnerabilities", true, "Whether security compliance is considered and reported as part of the assessment.") cmd.Flags().StringVar(&stepConfig.ServiceURL, "serviceUrl", `https://saas.whitesourcesoftware.com/api`, "URL to the WhiteSource API endpoint.") @@ -751,6 +755,29 @@ func whitesourceExecuteScanMetadata() config.StepData { Aliases: []config.Alias{}, Default: os.Getenv("PIPER_scanImage"), }, + { + Name: "scanImages", + ResourceRef: []config.ResourceReference{ + { + Name: "commonPipelineEnvironment", + Param: "container/imageNameTags", + }, + }, + Scope: []string{"PARAMETERS", "STAGES", "STEPS"}, + Type: "[]string", + Mandatory: false, + Aliases: []config.Alias{}, + Default: []string{}, + }, + { + Name: "skipParentProjectResolution", + ResourceRef: []config.ResourceReference{}, + Scope: []string{"PARAMETERS", "STAGES", "STEPS"}, + Type: "bool", + Mandatory: false, + Aliases: []config.Alias{}, + Default: false, + }, { Name: "scanImageRegistryUrl", ResourceRef: []config.ResourceReference{ diff --git a/pkg/whitesource/configHelper.go b/pkg/whitesource/configHelper.go index f7752f575f..e28831a142 100644 --- a/pkg/whitesource/configHelper.go +++ b/pkg/whitesource/configHelper.go @@ -46,7 +46,14 @@ func (s *ScanOptions) RewriteUAConfigurationFile(utils Utils, projectName string newConfig := properties.LoadMap(newConfigMap) now := time.Now().Format("20060102150405") - newConfigFilePath := fmt.Sprintf("%v.%v", s.ConfigFilePath, now) + + var newConfigFilePath string + + if s.ScanPath != "." { + newConfigFilePath = fmt.Sprintf("%v/%v.%v", s.ScanPath, s.ConfigFilePath, now) + } else { + newConfigFilePath = fmt.Sprintf("%v.%v", s.ConfigFilePath, now) + } var configContent bytes.Buffer _, err = newConfig.Write(&configContent, properties.UTF8) diff --git a/pkg/whitesource/scanOptions.go b/pkg/whitesource/scanOptions.go index 6cdcd29e5d..0a7a948bad 100644 --- a/pkg/whitesource/scanOptions.go +++ b/pkg/whitesource/scanOptions.go @@ -46,5 +46,7 @@ type ScanOptions struct { InstallCommand string + SkipParentProjectResolution bool + Verbose bool } diff --git a/pkg/whitesource/scanUA.go b/pkg/whitesource/scanUA.go index 0baf0ea37c..e7b591e9d8 100644 --- a/pkg/whitesource/scanUA.go +++ b/pkg/whitesource/scanUA.go @@ -98,8 +98,10 @@ func (s *Scan) ExecuteUAScanInPath(config *ScanOptions, utils Utils, scanPath st // ToDo: Check if Download of Docker/container image should be done here instead of in cmd/whitesourceExecuteScan.go // ToDo: check if this is required - if err := s.AppendScannedProject(s.AggregateProjectName); err != nil { - return err + if !config.SkipParentProjectResolution { + if err := s.AppendScannedProject(s.AggregateProjectName); err != nil { + return err + } } configPath, err := config.RewriteUAConfigurationFile(utils, s.AggregateProjectName) diff --git a/resources/metadata/whitesourceExecuteScan.yaml b/resources/metadata/whitesourceExecuteScan.yaml index 2be99d7ecb..dc1d926ae1 100644 --- a/resources/metadata/whitesourceExecuteScan.yaml +++ b/resources/metadata/whitesourceExecuteScan.yaml @@ -377,6 +377,25 @@ spec: - PARAMETERS - STAGES - STEPS + - name: scanImages + type: "[]string" + description: "For `buildTool: docker`: Allowing to scan multiple docker images. In case parent project will not contain any dependecies, use skipParentProjectResolution parameter" + resourceRef: + - name: commonPipelineEnvironment + param: container/imageNameTags + scope: + - PARAMETERS + - STAGES + - STEPS + - name: skipParentProjectResolution + type: bool + description: "Parameter for multi-module, multi-images projects to skip the parent project resolution for reporing purpose + Could be used if parent project is set as just a placeholder for scan and doesn't contain any dependencies." + scope: + - PARAMETERS + - STAGES + - STEPS + default: false - name: scanImageRegistryUrl type: string description: "For `buildTool: docker`: Defines the registry where the scanImage is located."