From 683ca350010f6eb5617497773974ba76e20e9cfb Mon Sep 17 00:00:00 2001 From: thtri Date: Mon, 3 Jun 2024 10:01:50 +0200 Subject: [PATCH] CxOne: Add param to tag scan and project (#4944) * Initial in progress * compiling but not yet functional * Missed file * updated checkmarxone step * Working up to fetching a project then breaks * Missed file * Breaks when retrieving projects+proxy set * Create project & run scan working, now polling * Fixed polling * added back the zipfile remove command * Fixed polling again * Generates and downloads PDF report * Updated and working, prep for refactor * Added compliance steps * Cleanup, reporting, added groovy connector * fixed groovy file * checkmarxone to checkmarxOne * checkmarxone to checkmarxOne * split credentials (id+secret, apikey), renamed pullrequestname to branch, groovy fix * Fixed filenames & yaml * missed the metadata_generated.go * added json to sarif conversion * fix:type in new checkmarxone package * fix:type in new checkmarxone package * removed test logs, added temp error log for creds * extra debugging to fix crash * improved auth logging, fixed query parse issue * fixed bug with group fetch when using oauth user * CWE can be -1 if not defined, can't be uint * Query also had CweID * Disabled predicates-fetch in sarif generation * Removing leftover info log message * Better error handling * fixed default preset configuration * removing .bat files - sorry * Cleanup per initial review * refactoring per Gist, fixed project find, add apps * small fix - sorry for commit noise while testing * Fixing issues with incremental scans. * removing maxretries * Updated per PR feedback, further changes todo toda * JSON Report changes and reporting cleanup * removing .bat (again?) * adding docs, groovy unit test, linter fixes * Started adding tests maybe 15% covered * fix(checkmarxOne): test cases for pkg and reporting * fix(checkmarxOne):fix formatting * feat(checkmarxone): update interface with missing method * feat(checkmarxone):change runStep signature to be able to inject dependency * feat(checkmarxone): add tests for step (wip) * Adding a bit more coverage * feat(checkmarxOne): fix code review * feat(checkmarxOne): fix code review * feat(checkmarxOne): fix code review * feat(checkmarxOne): fix integration test PR * adding scan-summary bug workaround, reportgen fail * enforceThresholds fix when no results passed in * fixed gap when preset empty in yaml & project conf * fixed another gap in preset selection * fix 0-result panic * fail when no preset is set anywhere * removed comment * initial project-under-app support * fixing sarif reportgen * some cleanup of error messages * post-merge test fixes * revert previous upstream merge * adding "incremental" to "full" triggers * wrong boolean * project-in-application api change prep * Fixing SARIF report without preset access * fix sarif deeplink * removing comments * fix(cxone):formatting * fix(cxone):formatting * small sarif fixes * fixed merge * attempt at pulling git source repo branch * fix(cxone):new endpoint for project creation * fix(cxOne): taxa is an array * fix(cxOne): get Git branch from commonPipelineEnvironment * fix(cxOne): add params to tag a scan and a project * fix(cxOne): unit test - update project * fix(cxOne): unit test - update project tags * fix(cxOne): improve logs * fix(cxOne): improve logs --------- Co-authored-by: michael kubiaczyk Co-authored-by: michaelkubiaczyk <48311127+michaelkubiaczyk@users.noreply.github.com> Co-authored-by: sumeet patil --- cmd/checkmarxOneExecuteScan.go | 48 +++++++++++-- cmd/checkmarxOneExecuteScan_generated.go | 24 ++++++- cmd/checkmarxOneExecuteScan_test.go | 69 ++++++++++++++++++- pkg/checkmarxone/checkmarxone.go | 35 +++++++--- pkg/checkmarxone/checkmarxone_test.go | 46 +++++++++++++ .../metadata/checkmarxOneExecuteScan.yaml | 18 ++++- 6 files changed, 222 insertions(+), 18 deletions(-) diff --git a/cmd/checkmarxOneExecuteScan.go b/cmd/checkmarxOneExecuteScan.go index 24731bc674..d809304290 100644 --- a/cmd/checkmarxOneExecuteScan.go +++ b/cmd/checkmarxOneExecuteScan.go @@ -3,8 +3,10 @@ package cmd import ( "archive/zip" "context" + "encoding/json" "fmt" "io" + "maps" "math" "os" "path/filepath" @@ -78,9 +80,11 @@ func runStep(config checkmarxOneExecuteScanOptions, influx *checkmarxOneExecuteS return fmt.Errorf("failed to get project: %s", err) } - cx1sh.Group, err = cx1sh.GetGroup() // used when creating a project and when generating a SARIF report - if err != nil { - log.Entry().WithError(err).Warnf("failed to get group") + if len(config.GroupName) > 0 { + cx1sh.Group, err = cx1sh.GetGroup() // used when creating a project and when generating a SARIF report + if err != nil { + log.Entry().WithError(err).Warnf("failed to get group") + } } if cx1sh.Project == nil { @@ -112,6 +116,14 @@ func runStep(config checkmarxOneExecuteScanOptions, influx *checkmarxOneExecuteS return fmt.Errorf("failed to set preset: %s", err) } + // update project's tags + if (len(config.ProjectTags)) > 0 { + err = cx1sh.UpdateProjectTags() + if err != nil { + log.Entry().WithError(err).Warnf("failed to tags the project: %s", err) + } + } + scans, err := cx1sh.GetLastScans(10) if err != nil { log.Entry().WithError(err).Warnf("failed to get last 10 scans") @@ -298,6 +310,23 @@ func (c *checkmarxOneExecuteScanHelper) CreateProject() (*checkmarxOne.Project, return &project, nil } +func (c *checkmarxOneExecuteScanHelper) UpdateProjectTags() error { + if len(c.config.ProjectTags) > 0 { + tags := make(map[string]string, 0) + err := json.Unmarshal([]byte(c.config.ProjectTags), &tags) + if err != nil { + log.Entry().Infof("Failed to parse the project tags: %v", c.config.ProjectTags) + return err + } + // merge new tags to the existing ones + maps.Copy(c.Project.Tags, tags) + + return c.sys.UpdateProject(c.Project) + } + + return nil +} + func (c *checkmarxOneExecuteScanHelper) SetProjectPreset() error { projectConf, err := c.sys.GetProjectConfiguration(c.Project.ProjectID) @@ -431,9 +460,18 @@ func (c *checkmarxOneExecuteScanHelper) CreateScanRequest(incremental bool, uplo log.Entry().Infof("Will run a scan with the following configuration: %v", sastConfigString) configs := []checkmarxOne.ScanConfiguration{sastConfig} - // add more engines - scan, err := c.sys.ScanProjectZip(c.Project.ProjectID, uploadLink, branch, configs) + // add scan's tags + tags := make(map[string]string, 0) + if len(c.config.ScanTags) > 0 { + err := json.Unmarshal([]byte(c.config.ScanTags), &tags) + if err != nil { + log.Entry().WithError(err).Warnf("Failed to parse the scan tags: %v", c.config.ScanTags) + } + } + + // add more engines + scan, err := c.sys.ScanProjectZip(c.Project.ProjectID, uploadLink, branch, configs, tags) if err != nil { return nil, fmt.Errorf("Failed to run scan on project %v: %s", c.Project.Name, err) diff --git a/cmd/checkmarxOneExecuteScan_generated.go b/cmd/checkmarxOneExecuteScan_generated.go index 06d8d653f2..6e14070dbb 100644 --- a/cmd/checkmarxOneExecuteScan_generated.go +++ b/cmd/checkmarxOneExecuteScan_generated.go @@ -39,6 +39,8 @@ type checkmarxOneExecuteScanOptions struct { LanguageMode string `json:"languageMode,omitempty"` ProjectCriticality string `json:"projectCriticality,omitempty"` ProjectName string `json:"projectName,omitempty"` + ProjectTags string `json:"projectTags,omitempty"` + ScanTags string `json:"scanTags,omitempty"` Branch string `json:"branch,omitempty"` PullRequestName string `json:"pullRequestName,omitempty"` Repository string `json:"repository,omitempty"` @@ -364,6 +366,8 @@ func addCheckmarxOneExecuteScanFlags(cmd *cobra.Command, stepConfig *checkmarxOn cmd.Flags().StringVar(&stepConfig.LanguageMode, "languageMode", `multi`, "Specifies whether the scan should be run for a 'single' language or 'multi' language, default 'multi'") cmd.Flags().StringVar(&stepConfig.ProjectCriticality, "projectCriticality", `3`, "The criticality of the checkmarxOne project, used during project creation") cmd.Flags().StringVar(&stepConfig.ProjectName, "projectName", os.Getenv("PIPER_projectName"), "The name of the checkmarxOne project to scan into") + cmd.Flags().StringVar(&stepConfig.ProjectTags, "projectTags", os.Getenv("PIPER_projectTags"), "Used to tag a project with a JSON string, e.g., {\"key\":\"value\", \"keywithoutvalue\":\"\"}") + cmd.Flags().StringVar(&stepConfig.ScanTags, "scanTags", os.Getenv("PIPER_scanTags"), "Used to tag a scan with a JSON string, e.g., {\"key\":\"value\", \"keywithoutvalue\":\"\"}") cmd.Flags().StringVar(&stepConfig.Branch, "branch", os.Getenv("PIPER_branch"), "Used to supply the branch scanned in the repository, or a friendly-name set by the user") cmd.Flags().StringVar(&stepConfig.PullRequestName, "pullRequestName", os.Getenv("PIPER_pullRequestName"), "Used to supply the name for the newly created PR project branch when being used in pull request scenarios. This is supplied by the orchestrator.") cmd.Flags().StringVar(&stepConfig.Repository, "repository", os.Getenv("PIPER_repository"), "Set the GitHub repository.") @@ -528,7 +532,7 @@ func checkmarxOneExecuteScanMetadata() config.StepData { ResourceRef: []config.ResourceReference{ { Name: "commonPipelineEnvironment", - Param: "github/branch", + Param: "git/branch", }, }, Scope: []string{"GENERAL", "PARAMETERS", "STAGES", "STEPS"}, @@ -615,6 +619,24 @@ func checkmarxOneExecuteScanMetadata() config.StepData { Aliases: []config.Alias{}, Default: os.Getenv("PIPER_projectName"), }, + { + Name: "projectTags", + ResourceRef: []config.ResourceReference{}, + Scope: []string{"PARAMETERS", "STAGES", "STEPS"}, + Type: "string", + Mandatory: false, + Aliases: []config.Alias{}, + Default: os.Getenv("PIPER_projectTags"), + }, + { + Name: "scanTags", + ResourceRef: []config.ResourceReference{}, + Scope: []string{"PARAMETERS", "STAGES", "STEPS"}, + Type: "string", + Mandatory: false, + Aliases: []config.Alias{}, + Default: os.Getenv("PIPER_scanTags"), + }, { Name: "branch", ResourceRef: []config.ResourceReference{}, diff --git a/cmd/checkmarxOneExecuteScan_test.go b/cmd/checkmarxOneExecuteScan_test.go index 453e8a1780..c9206dbf8d 100644 --- a/cmd/checkmarxOneExecuteScan_test.go +++ b/cmd/checkmarxOneExecuteScan_test.go @@ -4,6 +4,7 @@ import ( "context" "encoding/json" "fmt" + "maps" "testing" "github.com/stretchr/testify/assert" @@ -77,15 +78,15 @@ func (sys *checkmarxOneSystemMock) GetLastScansByStatus(projectID string, limit return []checkmarxOne.Scan{}, nil } -func (sys *checkmarxOneSystemMock) ScanProject(projectID, sourceUrl, branch, scanType string, settings []checkmarxOne.ScanConfiguration) (checkmarxOne.Scan, error) { +func (sys *checkmarxOneSystemMock) ScanProject(projectID, sourceUrl, branch, scanType string, settings []checkmarxOne.ScanConfiguration, tags map[string]string) (checkmarxOne.Scan, error) { return checkmarxOne.Scan{}, nil } -func (sys *checkmarxOneSystemMock) ScanProjectZip(projectID, sourceUrl, branch string, settings []checkmarxOne.ScanConfiguration) (checkmarxOne.Scan, error) { +func (sys *checkmarxOneSystemMock) ScanProjectZip(projectID, sourceUrl, branch string, settings []checkmarxOne.ScanConfiguration, tags map[string]string) (checkmarxOne.Scan, error) { return checkmarxOne.Scan{}, nil } -func (sys *checkmarxOneSystemMock) ScanProjectGit(projectID, repoUrl, branch string, settings []checkmarxOne.ScanConfiguration) (checkmarxOne.Scan, error) { +func (sys *checkmarxOneSystemMock) ScanProjectGit(projectID, repoUrl, branch string, settings []checkmarxOne.ScanConfiguration, tags map[string]string) (checkmarxOne.Scan, error) { return checkmarxOne.Scan{}, nil } @@ -240,6 +241,10 @@ func (sys *checkmarxOneSystemMock) UpdateProjectConfiguration(projectID string, return nil } +func (sys *checkmarxOneSystemMock) UpdateProject(project *checkmarxOne.Project) error { + return nil +} + func (sys *checkmarxOneSystemMock) GetVersion() (checkmarxOne.VersionInfo, error) { return checkmarxOne.VersionInfo{}, nil } @@ -324,3 +329,61 @@ func TestGetGroup(t *testing.T) { assert.Equal(t, group.Name, "Group2") }) } + +func TestUpdateProjectTags(t *testing.T) { + t.Parallel() + + sys := &checkmarxOneSystemMock{} + + t.Run("project tags are not provided", func(t *testing.T) { + t.Parallel() + + options := checkmarxOneExecuteScanOptions{ProjectName: "ssba", VulnerabilityThresholdUnit: "absolute", FullScanCycle: "2", Incremental: true, FullScansScheduled: true, Preset: "CheckmarxDefault" /*GroupName: "NotProvided",*/, VulnerabilityThresholdEnabled: true, GeneratePdfReport: true, APIKey: "testAPIKey", ServerURL: "testURL", IamURL: "testIamURL", Tenant: "testTenant"} + + cx1sh := checkmarxOneExecuteScanHelper{nil, options, sys, nil, nil, nil, nil, nil, nil} + err := cx1sh.UpdateProjectTags() + assert.NoError(t, err, "Error occurred but none expected") + }) + + t.Run("project tags are provided correctly", func(t *testing.T) { + t.Parallel() + + projectJson := `{ "id": "702ba12b-ae61-48c0-9b6a-09b17666be32", + "name": "test-apr24-piper", + "tags": { + "key1": "value1", + "key2": "value2", + "keywithoutvalue1": "" + }, + "groups": [], + "criticality": 3, + "mainBranch": "", + "privatePackage": false + }` + var project checkmarxOne.Project + _ = json.Unmarshal([]byte(projectJson), &project) + + options := checkmarxOneExecuteScanOptions{ProjectName: "ssba", VulnerabilityThresholdUnit: "absolute", FullScanCycle: "2", Incremental: true, FullScansScheduled: true, Preset: "CheckmarxDefault" /*GroupName: "NotProvided",*/, VulnerabilityThresholdEnabled: true, GeneratePdfReport: true, APIKey: "testAPIKey", ServerURL: "testURL", IamURL: "testIamURL", Tenant: "testTenant", ProjectTags: `{"key3":"value3", "key2":"value5", "keywithoutvalue2":""}`} + + cx1sh := checkmarxOneExecuteScanHelper{nil, options, sys, nil, nil, &project, nil, nil, nil} + err := cx1sh.UpdateProjectTags() + assert.NoError(t, err, "Error occurred but none expected") + + oldTagsJson := `{ + "key1": "value1", + "key2": "value2", + "keywithoutvalue1": "" + }` + oldTags := make(map[string]string, 0) + _ = json.Unmarshal([]byte(oldTagsJson), &oldTags) + + newTagsJson := `{"key3":"value3", "key2":"value5", "keywithoutvalue2":""}` + newTags := make(map[string]string, 0) + _ = json.Unmarshal([]byte(newTagsJson), &newTags) + + // merge new tags to the existing ones + maps.Copy(oldTags, newTags) + + assert.Equal(t, project.Tags, oldTags) // project's tags must be merged + }) +} diff --git a/pkg/checkmarxone/checkmarxone.go b/pkg/checkmarxone/checkmarxone.go index 442c14f308..f28a87a3cc 100644 --- a/pkg/checkmarxone/checkmarxone.go +++ b/pkg/checkmarxone/checkmarxone.go @@ -309,9 +309,10 @@ type System interface { GetLastScans(projectID string, limit int) ([]Scan, error) GetLastScansByStatus(projectID string, limit int, status []string) ([]Scan, error) - ScanProject(projectID, sourceUrl, branch, scanType string, settings []ScanConfiguration) (Scan, error) - ScanProjectZip(projectID, sourceUrl, branch string, settings []ScanConfiguration) (Scan, error) - ScanProjectGit(projectID, repoUrl, branch string, settings []ScanConfiguration) (Scan, error) + ScanProject(projectID, sourceUrl, branch, scanType string, settings []ScanConfiguration, tags map[string]string) (Scan, error) + ScanProjectZip(projectID, sourceUrl, branch string, settings []ScanConfiguration, tags map[string]string) (Scan, error) + ScanProjectGit(projectID, repoUrl, branch string, settings []ScanConfiguration, tags map[string]string) (Scan, error) + UpdateProject(project *Project) error UploadProjectSourceCode(projectID string, zipFile string) (string, error) CreateProject(projectName string, groupIDs []string) (Project, error) @@ -651,6 +652,22 @@ func (sys *SystemInstance) UpdateApplication(app *Application) error { return nil } +func (sys *SystemInstance) UpdateProject(project *Project) error { + sys.logger.Debugf("Updating project: %v", project.Name) + jsonBody, err := json.Marshal(*project) + if err != nil { + return err + } + + _, err = sendRequest(sys, http.MethodPut, fmt.Sprintf("/projects/%v", project.ProjectID), bytes.NewReader(jsonBody), nil, []int{}) + if err != nil { + sys.logger.Errorf("Error while updating project: %s", err) + return err + } + + return nil +} + // Updated for Cx1 func (sys *SystemInstance) GetGroups() ([]Group, error) { sys.logger.Debug("Getting Groups...") @@ -936,7 +953,7 @@ func (sys *SystemInstance) scanProject(scanConfig map[string]interface{}) (Scan, return scan, err } -func (sys *SystemInstance) ScanProjectZip(projectID, sourceUrl, branch string, settings []ScanConfiguration) (Scan, error) { +func (sys *SystemInstance) ScanProjectZip(projectID, sourceUrl, branch string, settings []ScanConfiguration, tags map[string]string) (Scan, error) { jsonBody := map[string]interface{}{ "project": map[string]interface{}{"id": projectID}, "type": "upload", @@ -945,6 +962,7 @@ func (sys *SystemInstance) ScanProjectZip(projectID, sourceUrl, branch string, s "branch": branch, }, "config": settings, + "tags": tags, } scan, err := sys.scanProject(jsonBody) @@ -954,7 +972,7 @@ func (sys *SystemInstance) ScanProjectZip(projectID, sourceUrl, branch string, s return scan, err } -func (sys *SystemInstance) ScanProjectGit(projectID, repoUrl, branch string, settings []ScanConfiguration) (Scan, error) { +func (sys *SystemInstance) ScanProjectGit(projectID, repoUrl, branch string, settings []ScanConfiguration, tags map[string]string) (Scan, error) { jsonBody := map[string]interface{}{ "project": map[string]interface{}{"id": projectID}, "type": "git", @@ -963,6 +981,7 @@ func (sys *SystemInstance) ScanProjectGit(projectID, repoUrl, branch string, set "branch": branch, }, "config": settings, + "tags": tags, } scan, err := sys.scanProject(jsonBody) @@ -972,11 +991,11 @@ func (sys *SystemInstance) ScanProjectGit(projectID, repoUrl, branch string, set return scan, err } -func (sys *SystemInstance) ScanProject(projectID, sourceUrl, branch, scanType string, settings []ScanConfiguration) (Scan, error) { +func (sys *SystemInstance) ScanProject(projectID, sourceUrl, branch, scanType string, settings []ScanConfiguration, tags map[string]string) (Scan, error) { if scanType == "upload" { - return sys.ScanProjectZip(projectID, sourceUrl, branch, settings) + return sys.ScanProjectZip(projectID, sourceUrl, branch, settings, tags) } else if scanType == "git" { - return sys.ScanProjectGit(projectID, sourceUrl, branch, settings) + return sys.ScanProjectGit(projectID, sourceUrl, branch, settings, tags) } return Scan{}, errors.New("Invalid scanType provided, must be 'upload' or 'git'") diff --git a/pkg/checkmarxone/checkmarxone_test.go b/pkg/checkmarxone/checkmarxone_test.go index 63b496698e..6b7ccf499c 100644 --- a/pkg/checkmarxone/checkmarxone_test.go +++ b/pkg/checkmarxone/checkmarxone_test.go @@ -2,6 +2,7 @@ package checkmarxOne import ( "bytes" + "encoding/json" "errors" "fmt" "io" @@ -294,3 +295,48 @@ func TestGetApplicationByName(t *testing.T) { assert.Contains(t, fmt.Sprint(err), "Provoked technical error") }) } + +func TestUpdateProject(t *testing.T) { + logger := log.Entry().WithField("package", "SAP/jenkins-library/pkg/checkmarxOne_test") + opts := piperHttp.ClientOptions{} + + requestJson := `{ "id": "702ba12b-ae61-48c0-9b6a-09b17666be32", + "name": "test-apr24-piper", + "tags": { + "\"key1\"": "\"value1\"", + "\"keywithoutvalue\"": "\"\"" + }, + "groups": [], + "criticality": 3, + "mainBranch": "", + "privatePackage": false + }` + var project Project + _ = json.Unmarshal([]byte(requestJson), &project) + + t.Run("test success", func(t *testing.T) { + myTestClient := senderMock{responseBody: ``, httpStatusCode: 204} + serverURL := "https://cx1.server.com" + sys := SystemInstance{serverURL: serverURL, iamURL: "https://cx1iam.server.com", tenant: "tenant", client: &myTestClient, logger: logger} + myTestClient.SetOptions(opts) + + err := sys.UpdateProject(&project) + assert.NoError(t, err, "Error occurred but none expected") + assert.Equal(t, serverURL+"/api/projects/"+project.ProjectID, myTestClient.urlCalled, "Called url incorrect") + assert.Equal(t, "PUT", myTestClient.httpMethod, "HTTP method incorrect") + var body Project + _ = json.Unmarshal([]byte(myTestClient.requestBody), &body) + assert.Equal(t, project, body, "Request body incorrect") + + }) + + t.Run("test technical error", func(t *testing.T) { + myTestClient := senderMock{httpStatusCode: 403} + sys := SystemInstance{serverURL: "https://cx1.server.com", iamURL: "https://cx1iam.server.com", tenant: "tenant", client: &myTestClient, logger: logger} + myTestClient.SetOptions(opts) + myTestClient.errorExp = true + + err := sys.UpdateProject(&project) + assert.Contains(t, fmt.Sprint(err), "Provoked technical error") + }) +} diff --git a/resources/metadata/checkmarxOneExecuteScan.yaml b/resources/metadata/checkmarxOneExecuteScan.yaml index 0807f7ff2e..ff89ea18b3 100644 --- a/resources/metadata/checkmarxOneExecuteScan.yaml +++ b/resources/metadata/checkmarxOneExecuteScan.yaml @@ -131,7 +131,7 @@ spec: description: "Set the GitHub repository branch." resourceRef: - name: commonPipelineEnvironment - param: github/branch + param: git/branch scope: - GENERAL - PARAMETERS @@ -202,6 +202,22 @@ spec: - PARAMETERS - STAGES - STEPS + - name: projectTags + type: string + description: Used to tag a project with a JSON string, e.g., {"key":"value", "keywithoutvalue":""} + mandatory: false + scope: + - PARAMETERS + - STAGES + - STEPS + - name: scanTags + type: string + description: Used to tag a scan with a JSON string, e.g., {"key":"value", "keywithoutvalue":""} + mandatory: false + scope: + - PARAMETERS + - STAGES + - STEPS - name: branch type: string description: Used to supply the branch scanned in the repository, or a friendly-name set by the user