Skip to content

Commit

Permalink
feat(codeqlExecuteScan): adding data to InfluxDB (#4780)
Browse files Browse the repository at this point in the history 
* added influxdb to params, added log for testing

* changed fields for codeql influx db

* added setting codeql findings to influx

* refactored

* fixed typo

* added tests
  • Loading branch information
@daskuznetsova
daskuznetsova authored Jan 17, 2024
1 parent 808b21f commit 6920cad
Show file tree
Hide file tree
Showing 5 changed files with 217 additions and 14 deletions.
33 changes: 29 additions & 4 deletions cmd/codeqlExecuteScan.go
View file
Original file line number Diff line number Diff line change
Expand Up @@ -52,16 +52,19 @@ func newCodeqlExecuteScanUtils() codeqlExecuteScanUtils {
return &utils
}

func codeqlExecuteScan(config codeqlExecuteScanOptions, telemetryData *telemetry.CustomData) {
func codeqlExecuteScan(config codeqlExecuteScanOptions, telemetryData *telemetry.CustomData, influx *codeqlExecuteScanInflux) {

utils := newCodeqlExecuteScanUtils()

reports, err := runCodeqlExecuteScan(&config, telemetryData, utils)
influx.step_data.fields.codeql = false

reports, err := runCodeqlExecuteScan(&config, telemetryData, utils, influx)
piperutils.PersistReportsAndLinks("codeqlExecuteScan", "./", utils, reports, nil)

if err != nil {
log.Entry().WithError(err).Fatal("Codeql scan failed")
}
influx.step_data.fields.codeql = true
}

func codeqlQuery(cmd []string, codeqlQuery string) []string {
Expand Down Expand Up @@ -261,7 +264,7 @@ func waitSarifUploaded(config *codeqlExecuteScanOptions, codeqlSarifUploader cod
}
}

func runCodeqlExecuteScan(config *codeqlExecuteScanOptions, telemetryData *telemetry.CustomData, utils codeqlExecuteScanUtils) ([]piperutils.Path, error) {
func runCodeqlExecuteScan(config *codeqlExecuteScanOptions, telemetryData *telemetry.CustomData, utils codeqlExecuteScanUtils, influx *codeqlExecuteScanInflux) ([]piperutils.Path, error) {
codeqlVersion, err := os.ReadFile("/etc/image-version")
if err != nil {
log.Entry().Infof("CodeQL image version: unknown")
Expand Down Expand Up @@ -361,6 +364,8 @@ func runCodeqlExecuteScan(config *codeqlExecuteScanOptions, telemetryData *telem
repoInfo.CommitId = targetCommitId
}

var scanResults []codeql.CodeqlFindings

if !config.UploadResults {
log.Entry().Warn("The sarif results will not be uploaded to the repository and compliance report will not be generated as uploadResults is set to false.")
} else {
Expand All @@ -380,7 +385,7 @@ func runCodeqlExecuteScan(config *codeqlExecuteScanOptions, telemetryData *telem
}

codeqlScanAuditInstance := codeql.NewCodeqlScanAuditInstance(repoInfo.ServerUrl, repoInfo.Owner, repoInfo.Repo, token, []string{})
scanResults, err := codeqlScanAuditInstance.GetVulnerabilities(repoInfo.Ref)
scanResults, err = codeqlScanAuditInstance.GetVulnerabilities(repoInfo.Ref)
if err != nil {
return reports, errors.Wrap(err, "failed to get scan results")
}
Expand All @@ -403,6 +408,8 @@ func runCodeqlExecuteScan(config *codeqlExecuteScanOptions, telemetryData *telem
}
}

addDataToInfluxDB(repoUrl, repoReference, repoCodeqlScanUrl, config.QuerySuite, scanResults, influx)

toolRecordFileName, err := codeql.CreateAndPersistToolRecord(utils, repoInfo, repoReference, repoUrl, config.ModulePath)
if err != nil {
log.Entry().Warning("TR_CODEQL: Failed to create toolrecord file ...", err)
Expand All @@ -413,6 +420,24 @@ func runCodeqlExecuteScan(config *codeqlExecuteScanOptions, telemetryData *telem
return reports, nil
}

func addDataToInfluxDB(repoUrl, repoRef, repoScanUrl, querySuite string, scanResults []codeql.CodeqlFindings, influx *codeqlExecuteScanInflux) {
influx.codeql_data.fields.repositoryURL = repoUrl
influx.codeql_data.fields.repositoryReferenceURL = repoRef
influx.codeql_data.fields.codeScanningLink = repoScanUrl
influx.codeql_data.fields.querySuite = querySuite

for _, sr := range scanResults {
if sr.ClassificationName == codeql.AuditAll {
influx.codeql_data.fields.auditAllAudited = sr.Audited
influx.codeql_data.fields.auditAllTotal = sr.Total
}
if sr.ClassificationName == codeql.Optional {
influx.codeql_data.fields.optionalAudited = sr.Audited
influx.codeql_data.fields.optionalTotal = sr.Total
}
}
}

func getRamAndThreadsFromConfig(config *codeqlExecuteScanOptions) []string {
params := make([]string, 0, 2)
if len(config.Threads) > 0 {
Expand Down
69 changes: 68 additions & 1 deletion cmd/codeqlExecuteScan_generated.go
View file

Some generated files are not rendered by default. Learn more about how customized files appear on GitHub.

102 changes: 95 additions & 7 deletions cmd/codeqlExecuteScan_test.go
View file
Original file line number Diff line number Diff line change
Expand Up @@ -31,45 +31,47 @@ func newCodeqlExecuteScanTestsUtils() codeqlExecuteScanMockUtils {

func TestRunCodeqlExecuteScan(t *testing.T) {

influx := &codeqlExecuteScanInflux{}

t.Run("Valid CodeqlExecuteScan", func(t *testing.T) {
config := codeqlExecuteScanOptions{BuildTool: "maven", ModulePath: "./"}
_, err := runCodeqlExecuteScan(&config, nil, newCodeqlExecuteScanTestsUtils())
_, err := runCodeqlExecuteScan(&config, nil, newCodeqlExecuteScanTestsUtils(), influx)
assert.NoError(t, err)
})

t.Run("No auth token passed on upload results", func(t *testing.T) {
config := codeqlExecuteScanOptions{BuildTool: "maven", UploadResults: true, ModulePath: "./"}
_, err := runCodeqlExecuteScan(&config, nil, newCodeqlExecuteScanTestsUtils())
_, err := runCodeqlExecuteScan(&config, nil, newCodeqlExecuteScanTestsUtils(), influx)
assert.Error(t, err)
})

t.Run("GitCommitID is NA on upload results", func(t *testing.T) {
config := codeqlExecuteScanOptions{BuildTool: "maven", UploadResults: true, ModulePath: "./", CommitID: "NA"}
_, err := runCodeqlExecuteScan(&config, nil, newCodeqlExecuteScanTestsUtils())
_, err := runCodeqlExecuteScan(&config, nil, newCodeqlExecuteScanTestsUtils(), influx)
assert.Error(t, err)
})

t.Run("Custom buildtool", func(t *testing.T) {
config := codeqlExecuteScanOptions{BuildTool: "custom", Language: "javascript", ModulePath: "./"}
_, err := runCodeqlExecuteScan(&config, nil, newCodeqlExecuteScanTestsUtils())
_, err := runCodeqlExecuteScan(&config, nil, newCodeqlExecuteScanTestsUtils(), influx)
assert.NoError(t, err)
})

t.Run("Custom buildtool but no language specified", func(t *testing.T) {
config := codeqlExecuteScanOptions{BuildTool: "custom", ModulePath: "./", GithubToken: "test"}
_, err := runCodeqlExecuteScan(&config, nil, newCodeqlExecuteScanTestsUtils())
_, err := runCodeqlExecuteScan(&config, nil, newCodeqlExecuteScanTestsUtils(), influx)
assert.Error(t, err)
})

t.Run("Invalid buildtool and no language specified", func(t *testing.T) {
config := codeqlExecuteScanOptions{BuildTool: "test", ModulePath: "./", GithubToken: "test"}
_, err := runCodeqlExecuteScan(&config, nil, newCodeqlExecuteScanTestsUtils())
_, err := runCodeqlExecuteScan(&config, nil, newCodeqlExecuteScanTestsUtils(), influx)
assert.Error(t, err)
})

t.Run("Invalid buildtool but language specified", func(t *testing.T) {
config := codeqlExecuteScanOptions{BuildTool: "test", Language: "javascript", ModulePath: "./", GithubToken: "test"}
_, err := runCodeqlExecuteScan(&config, nil, newCodeqlExecuteScanTestsUtils())
_, err := runCodeqlExecuteScan(&config, nil, newCodeqlExecuteScanTestsUtils(), influx)
assert.NoError(t, err)
})
}
Expand Down Expand Up @@ -401,6 +403,92 @@ func TestGetMavenSettings(t *testing.T) {
})
}

func TestAddDataToInfluxDB(t *testing.T) {
repoUrl := "https://github.htllo.test/Testing/codeql"
repoRef := "https://github.htllo.test/Testing/codeql/tree/branch"
repoScanUrl := "https://github.htllo.test/Testing/codeql/security/code-scanning"
querySuite := "security.ql"

t.Run("No findings", func(t *testing.T) {
scanResults := []codeql.CodeqlFindings{}
influx := &codeqlExecuteScanInflux{}
addDataToInfluxDB(repoUrl, repoRef, repoScanUrl, querySuite, scanResults, influx)
assert.Equal(t, repoUrl, influx.codeql_data.fields.repositoryURL)
assert.Equal(t, repoRef, influx.codeql_data.fields.repositoryReferenceURL)
assert.Equal(t, repoScanUrl, influx.codeql_data.fields.codeScanningLink)
assert.Equal(t, querySuite, influx.codeql_data.fields.querySuite)
assert.Equal(t, 0, influx.codeql_data.fields.auditAllTotal)
assert.Equal(t, 0, influx.codeql_data.fields.auditAllAudited)
assert.Equal(t, 0, influx.codeql_data.fields.optionalTotal)
assert.Equal(t, 0, influx.codeql_data.fields.optionalAudited)
})

t.Run("Audit All findings category only", func(t *testing.T) {
scanResults := []codeql.CodeqlFindings{
{
ClassificationName: codeql.AuditAll,
Total: 100,
Audited: 50,
},
}
influx := &codeqlExecuteScanInflux{}
addDataToInfluxDB(repoUrl, repoRef, repoScanUrl, querySuite, scanResults, influx)
assert.Equal(t, repoUrl, influx.codeql_data.fields.repositoryURL)
assert.Equal(t, repoRef, influx.codeql_data.fields.repositoryReferenceURL)
assert.Equal(t, repoScanUrl, influx.codeql_data.fields.codeScanningLink)
assert.Equal(t, querySuite, influx.codeql_data.fields.querySuite)
assert.Equal(t, scanResults[0].Total, influx.codeql_data.fields.auditAllTotal)
assert.Equal(t, scanResults[0].Audited, influx.codeql_data.fields.auditAllAudited)
assert.Equal(t, 0, influx.codeql_data.fields.optionalTotal)
assert.Equal(t, 0, influx.codeql_data.fields.optionalAudited)
})

t.Run("Optional findings category only", func(t *testing.T) {
scanResults := []codeql.CodeqlFindings{
{
ClassificationName: codeql.Optional,
Total: 100,
Audited: 50,
},
}
influx := &codeqlExecuteScanInflux{}
addDataToInfluxDB(repoUrl, repoRef, repoScanUrl, querySuite, scanResults, influx)
assert.Equal(t, repoUrl, influx.codeql_data.fields.repositoryURL)
assert.Equal(t, repoRef, influx.codeql_data.fields.repositoryReferenceURL)
assert.Equal(t, repoScanUrl, influx.codeql_data.fields.codeScanningLink)
assert.Equal(t, querySuite, influx.codeql_data.fields.querySuite)
assert.Equal(t, 0, influx.codeql_data.fields.auditAllTotal)
assert.Equal(t, 0, influx.codeql_data.fields.auditAllAudited)
assert.Equal(t, scanResults[0].Total, influx.codeql_data.fields.optionalTotal)
assert.Equal(t, scanResults[0].Audited, influx.codeql_data.fields.optionalAudited)
})

t.Run("Both findings category", func(t *testing.T) {
scanResults := []codeql.CodeqlFindings{
{
ClassificationName: codeql.AuditAll,
Total: 100,
Audited: 50,
},
{
ClassificationName: codeql.Optional,
Total: 100,
Audited: 50,
},
}
influx := &codeqlExecuteScanInflux{}
addDataToInfluxDB(repoUrl, repoRef, repoScanUrl, querySuite, scanResults, influx)
assert.Equal(t, repoUrl, influx.codeql_data.fields.repositoryURL)
assert.Equal(t, repoRef, influx.codeql_data.fields.repositoryReferenceURL)
assert.Equal(t, repoScanUrl, influx.codeql_data.fields.codeScanningLink)
assert.Equal(t, querySuite, influx.codeql_data.fields.querySuite)
assert.Equal(t, scanResults[0].Total, influx.codeql_data.fields.auditAllTotal)
assert.Equal(t, scanResults[0].Audited, influx.codeql_data.fields.auditAllAudited)
assert.Equal(t, scanResults[1].Total, influx.codeql_data.fields.optionalTotal)
assert.Equal(t, scanResults[1].Audited, influx.codeql_data.fields.optionalAudited)
})
}

type CodeqlSarifUploaderMock struct {
counter int
}
Expand Down
6 changes: 4 additions & 2 deletions pkg/codeql/codeql.go
View file
Original file line number Diff line number Diff line change
Expand Up @@ -19,6 +19,8 @@ const auditStateOpen string = "open"
const auditStateDismissed string = "dismissed"
const codeqlToolName string = "CodeQL"
const perPageCount int = 100
const AuditAll string = "Audit All"
const Optional string = "Optional"

func NewCodeqlScanAuditInstance(serverUrl, owner, repository, token string, trustedCerts []string) CodeqlScanAuditInstance {
return CodeqlScanAuditInstance{serverUrl: serverUrl, owner: owner, repository: repository, token: token, trustedCerts: trustedCerts}
Expand Down Expand Up @@ -104,12 +106,12 @@ func getVulnerabilitiesFromClient(ctx context.Context, codeScanning githubCodeql
}

auditAll := CodeqlFindings{
ClassificationName: "Audit All",
ClassificationName: AuditAll,
Total: totalAlerts,
Audited: audited,
}
optionalIssues := CodeqlFindings{
ClassificationName: "Optional",
ClassificationName: Optional,
Total: totalOptionalAlerts,
Audited: optionalAudited,
}
Expand Down
Loading

0 comments on commit 6920cad

Please sign in to comment.