diff --git a/cmd/malwareExecuteScan.go b/cmd/malwareExecuteScan.go index 55c90a0e5d..0e99c64da4 100644 --- a/cmd/malwareExecuteScan.go +++ b/cmd/malwareExecuteScan.go @@ -3,6 +3,11 @@ package cmd import ( "encoding/json" "fmt" + "io" + "os" + "strings" + "time" + piperDocker "github.com/SAP/jenkins-library/pkg/docker" piperhttp "github.com/SAP/jenkins-library/pkg/http" "github.com/SAP/jenkins-library/pkg/log" @@ -11,10 +16,6 @@ import ( "github.com/SAP/jenkins-library/pkg/telemetry" "github.com/SAP/jenkins-library/pkg/toolrecord" "github.com/pkg/errors" - "io" - "os" - "strings" - "time" ) type malwareScanUtils interface { @@ -45,10 +46,22 @@ func (utils *malwareScanUtilsBundle) newDockerClient(options piperDocker.ClientO func newMalwareScanUtilsBundle(config malwareExecuteScanOptions) *malwareScanUtilsBundle { timeout, err := time.ParseDuration(fmt.Sprintf("%ss", config.Timeout)) if err != nil { - timeout = 60 + timeout = 60 * time.Second log.Entry().Warnf("Unable to parse timeout for malwareScan: '%v'. Falling back to %ds", err, timeout) } + pollinterval, err := time.ParseDuration(fmt.Sprintf("%ss", config.PollingInterval)) + if err != nil { + pollinterval = 10 * time.Second + log.Entry().Warnf("Unable to parse poll interval for malwareScan: '%v'. Falling back to %ds", err, pollinterval) + } + + pollingTimeout, err := time.ParseDuration(fmt.Sprintf("%ss", config.PollingTimeout)) + if err != nil { + pollingTimeout = 600 * time.Second + log.Entry().Warnf("Unable to parse poll timeout for malwareScan: '%v'. Falling back to %ds", err, pollingTimeout) + } + httpClientOptions := piperhttp.ClientOptions{ Username: config.Username, Password: config.Password, @@ -61,8 +74,10 @@ func newMalwareScanUtilsBundle(config malwareExecuteScanOptions) *malwareScanUti return &malwareScanUtilsBundle{ Client: &malwarescan.ClientImpl{ - HTTPClient: httpClient, - Host: config.Host, + HTTPClient: httpClient, + Host: config.Host, + PollInterval: pollinterval, + PollingTimeout: pollingTimeout, }, Files: &piperutils.Files{}, } @@ -85,12 +100,6 @@ func runMalwareScan(config *malwareExecuteScanOptions, telemetryData *telemetry. log.Entry().Infof("Scanning file \"%s\" for malware using service \"%s\"", file, config.Host) - candidate, err := utils.OpenFile(file, os.O_RDONLY, 0666) - if err != nil { - return err - } - defer candidate.Close() - scannerInfo, err := utils.Info() log.Entry().Infof("***************************************") @@ -102,10 +111,25 @@ func runMalwareScan(config *malwareExecuteScanOptions, telemetryData *telemetry. return err } - scanResponse, err := utils.Scan(candidate) + var scanResponse *malwarescan.ScanResult - if err != nil { - return err + if config.Asynchronous { + scanResponse, err = utils.ScanAsync(file) + if err != nil { + return err + } + + } else { + candidate, err := utils.OpenFile(file, os.O_RDONLY, 0666) + if err != nil { + return err + } + defer candidate.Close() + + scanResponse, err = utils.Scan(candidate) + if err != nil { + return err + } } if err = createMalwareScanReport(config, scanResponse, utils); err != nil { diff --git a/cmd/malwareExecuteScan_generated.go b/cmd/malwareExecuteScan_generated.go index 54de40ee15..1eae0f86f7 100644 --- a/cmd/malwareExecuteScan_generated.go +++ b/cmd/malwareExecuteScan_generated.go @@ -32,6 +32,9 @@ type malwareExecuteScanOptions struct { ScanFile string `json:"scanFile,omitempty"` Timeout string `json:"timeout,omitempty"` ReportFileName string `json:"reportFileName,omitempty"` + Asynchronous bool `json:"asynchronous,omitempty"` + PollingInterval string `json:"pollingInterval,omitempty"` + PollingTimeout string `json:"pollingTimeout,omitempty"` } type malwareExecuteScanReports struct { @@ -183,6 +186,9 @@ func addMalwareExecuteScanFlags(cmd *cobra.Command, stepConfig *malwareExecuteSc cmd.Flags().StringVar(&stepConfig.ScanFile, "scanFile", os.Getenv("PIPER_scanFile"), "The file which is scanned for malware") cmd.Flags().StringVar(&stepConfig.Timeout, "timeout", `600`, "timeout for http layer in seconds") cmd.Flags().StringVar(&stepConfig.ReportFileName, "reportFileName", `malwarescan_report.json`, "The file name of the report to be created") + cmd.Flags().BoolVar(&stepConfig.Asynchronous, "asynchronous", false, "Enable asynchronous scanning API.") + cmd.Flags().StringVar(&stepConfig.PollingInterval, "pollingInterval", `10`, "Polling interval for asynchronous scan in seconds.") + cmd.Flags().StringVar(&stepConfig.PollingTimeout, "pollingTimeout", `600`, "Polling timeout for asynchronous scan API in seconds.") cmd.MarkFlagRequired("buildTool") cmd.MarkFlagRequired("host") @@ -387,6 +393,33 @@ func malwareExecuteScanMetadata() config.StepData { Aliases: []config.Alias{}, Default: `malwarescan_report.json`, }, + { + Name: "asynchronous", + ResourceRef: []config.ResourceReference{}, + Scope: []string{"PARAMETERS", "STAGES", "STEPS"}, + Type: "bool", + Mandatory: false, + Aliases: []config.Alias{}, + Default: false, + }, + { + Name: "pollingInterval", + ResourceRef: []config.ResourceReference{}, + Scope: []string{"PARAMETERS", "STAGES", "STEPS"}, + Type: "string", + Mandatory: false, + Aliases: []config.Alias{}, + Default: `10`, + }, + { + Name: "pollingTimeout", + ResourceRef: []config.ResourceReference{}, + Scope: []string{"PARAMETERS", "STAGES", "STEPS"}, + Type: "string", + Mandatory: false, + Aliases: []config.Alias{}, + Default: `600`, + }, }, }, Outputs: config.StepOutputs{ diff --git a/cmd/malwareExecuteScan_test.go b/cmd/malwareExecuteScan_test.go index a3cfd21832..ea17da6005 100644 --- a/cmd/malwareExecuteScan_test.go +++ b/cmd/malwareExecuteScan_test.go @@ -53,6 +53,10 @@ func (utils *malwareScanUtilsMockBundle) Scan(candidate io.Reader) (*malwarescan return utils.returnScanResult, nil } +func (utils *malwareScanUtilsMockBundle) ScanAsync(file string) (*malwarescan.ScanResult, error) { + return utils.returnScanResult, nil +} + func (utils *malwareScanUtilsMockBundle) newDockerClient(options piperDocker.ClientOptions) piperDocker.Download { return &dockerClientMock{imageName: options.ImageName, registryURL: options.RegistryURL, localPath: options.LocalPath} } diff --git a/pkg/malwarescan/malwarescan.go b/pkg/malwarescan/malwarescan.go index 5bc1fc1392..9440476962 100644 --- a/pkg/malwarescan/malwarescan.go +++ b/pkg/malwarescan/malwarescan.go @@ -1,13 +1,19 @@ package malwarescan import ( + "bytes" "encoding/json" "fmt" - piperhttp "github.com/SAP/jenkins-library/pkg/http" - "github.com/pkg/errors" "io" "io/ioutil" "net/http" + "os" + "strings" + "time" + + piperhttp "github.com/SAP/jenkins-library/pkg/http" + "github.com/SAP/jenkins-library/pkg/log" + "github.com/pkg/errors" ) // ScanResult : Returned by the scan endpoint of the malwarescan api of SAP CP @@ -20,6 +26,41 @@ type ScanResult struct { SHA256 string `json:"SHA256"` } +// AsyncUploadResult : Upload result returned by the malwarescan asynchronous scan api +type AsyncUploadResult struct { + JobID string `json:"jobID"` +} + +// AsyncStatus : Scan status returned by the malwarescan asynchronous scan api +type AsyncStatus string + +const ( + STATE_PROCESSING AsyncStatus = "processing" + STATE_DONE AsyncStatus = "done" + STATE_ERROR AsyncStatus = "error" +) + +// AsyncIncident : Incident detail returned by the malwarescan asynchronous scan result +type AsyncIncident struct { + Message string `json:"message"` + FilePath string `json:"filePath"` +} + +// AsyncScanResult : Scan result returned by the malwarescan asynchronous scan api +type AsyncScanResult struct { + Created time.Time `json:"created"` + Errors []AsyncIncident `json:"errors"` + Findings []AsyncIncident `json:"findings"` + Encrypted []AsyncIncident `json:"encryptedContent"` + Processed int `json:"processed"` + FileCount int `json:"fileCount"` + Sha256 string `json:"sha256"` + SizeByte int `json:"sizeByte"` + Status AsyncStatus `json:"status"` + Updated time.Time `json:"updated"` + CorrelationID string `json:"correlation_id"` +} + // Info : Returned by the info endpoint of the malwarescan api of SAP CP type Info struct { MaxScanSize int @@ -34,14 +75,113 @@ type ScanError struct { // Client : Interface for the malwarescan api provided by SAP CP (see https://api.sap.com/api/MalwareScanAPI/overview) type Client interface { + // ScanAsync(candidate io.Reader) (*ScanResult, error) + ScanAsync(file string) (*ScanResult, error) Scan(candidate io.Reader) (*ScanResult, error) Info() (*Info, error) } // ClientImpl : Client implementation of the malwarescan api provided by SAP CP (see https://api.sap.com/api/MalwareScanAPI/overview) type ClientImpl struct { - HTTPClient piperhttp.Sender - Host string + HTTPClient piperhttp.Sender + Host string + PollInterval time.Duration + PollingTimeout time.Duration +} + +// ScanAsync : Triggers an asynchronous malwarescan for the given content. +func (c *ClientImpl) ScanAsync(file string) (*ScanResult, error) { + var scanResult ScanResult + var asyncUploadResult AsyncUploadResult + + log.Entry().Debugf("Starting asynchronous scan") + + // Without using a bytes.Reader for the request body, the net/http NewRequest + // method does not set Content-Length header and set the transfer to "chunked" + // which is not supported by the malware scan api. + fileHandle, err := os.ReadFile(file) + if err != nil { + return nil, err + } + + err = c.sendAPIRequest("POST", "/v2/tar", bytes.NewReader(fileHandle), nil, &asyncUploadResult) + if err != nil { + return nil, err + } + + if len(asyncUploadResult.JobID) != 0 { + log.Entry().Debugf("File uploaded successfuly, start polling for job status") + scanResult, err = c.pollResult(asyncUploadResult.JobID) + if err != nil { + return nil, err + } + + } else { + return nil, fmt.Errorf("Malware asynchronous scan upload failed, missing job id.") + } + + return &scanResult, nil +} + +// pollResult polls the malwarescan api for the scan result +func (c *ClientImpl) pollResult(jobID string) (ScanResult, error) { + var scanResult ScanResult + var asyncScanResult AsyncScanResult + var err error + + ticker := time.NewTicker(c.PollInterval) + defer ticker.Stop() + + timeout := time.NewTimer(c.PollingTimeout) + defer timeout.Stop() + + start := time.Now() + done := make(chan bool) + + go func() { + for t := time.Now(); true; t = <-ticker.C { + log.Entry().Debugf("Checking job ID %v status [elapsed time: %.0f/%.0fs]", jobID, t.Sub(start).Seconds(), c.PollingTimeout.Seconds()) + + err = c.sendAPIRequest(http.MethodGet, fmt.Sprintf("/v2/status?id=%v", jobID), nil, nil, &asyncScanResult) + if err != nil { + done <- true + } + + if asyncScanResult.Status == STATE_ERROR { + errmsg := []string{} + for _, incident := range asyncScanResult.Errors { + errmsg = append(errmsg, incident.Message) + } + err = fmt.Errorf("Error while scanning:\n%s", strings.Join(errmsg, "\n")) + done <- true + + } else if asyncScanResult.Status == STATE_DONE { + findings := append(asyncScanResult.Findings, asyncScanResult.Encrypted...) + scanResult.MalwareDetected = len(asyncScanResult.Findings) > 0 + scanResult.EncryptedContentDetected = len(asyncScanResult.Encrypted) > 0 + scanResult.ScanSize = asyncScanResult.SizeByte + scanResult.Finding = fmt.Sprintf("%+v", findings) + scanResult.MimeType = "application/x-tar" + scanResult.SHA256 = asyncScanResult.Sha256 + done <- true + } else { + log.Entry().Debugf("Scan still in progress. correlation ID %s", asyncScanResult.CorrelationID) + } + } + }() + +loop: + for { + select { + case <-done: + break loop + case <-timeout.C: + err = fmt.Errorf("Time out getting scan results.") + break loop + } + } + + return scanResult, err } // Scan : Triggers a malwarescan in SAP CP for the given content. @@ -58,7 +198,6 @@ func (c *ClientImpl) Scan(candidate io.Reader) (*ScanResult, error) { } return &scanResult, nil - } // Info : Returns some information about the scanengine used by the malwarescan service. @@ -80,14 +219,12 @@ func (c *ClientImpl) sendAPIRequest(method, endpoint string, body io.Reader, hea // sendRequest results in any combination of nil and non-nil response and error. // a response body could even be already closed. - response, err := c.HTTPClient.SendRequest(method, c.Host+endpoint, body, header, nil) + response, _ := c.HTTPClient.SendRequest(method, c.Host+endpoint, body, header, nil) - if response.StatusCode != 200 { + if response.StatusCode < 200 || response.StatusCode >= 300 { var scanError ScanError - err = c.unmarshalResponse(response, &scanError) - - if err != nil { + if err := c.unmarshalResponse(response, &scanError); err != nil { return fmt.Errorf("MalwareService returned with status code %d, no further information available", response.StatusCode) } diff --git a/pkg/malwarescan/malwarescan_test.go b/pkg/malwarescan/malwarescan_test.go index 733ba05034..ff86c3c01e 100644 --- a/pkg/malwarescan/malwarescan_test.go +++ b/pkg/malwarescan/malwarescan_test.go @@ -1,15 +1,163 @@ package malwarescan import ( + "bytes" + "encoding/json" "fmt" - piperhttp "github.com/SAP/jenkins-library/pkg/http" - "github.com/stretchr/testify/assert" "io" "io/ioutil" "net/http" + "net/http/httptest" + "os" + "strings" "testing" + "time" + + piperhttp "github.com/SAP/jenkins-library/pkg/http" + "github.com/stretchr/testify/assert" ) +func TestMalwareServiceAsyncScan(t *testing.T) { + var jobID = "" + var statusCode int + var asyncScanResult *AsyncScanResult + var passedHeaders = map[string][]string{} + + server := httptest.NewServer(http.HandlerFunc(func(rw http.ResponseWriter, req *http.Request) { + var b bytes.Buffer + var resp interface{} + newStatusCode := statusCode + passedHeaders = map[string][]string{} + + if req.Header != nil { + for name, headers := range req.Header { + passedHeaders[name] = headers + } + } + + rw.Header().Add("Content-Type", "application/json") + + if strings.HasSuffix(req.URL.Path, "/v2/tar") { + switch newStatusCode { + case http.StatusNotAcceptable: + resp = ScanError{"wrong content type"} + case http.StatusUnsupportedMediaType: + resp = ScanError{"unsupported media type"} + default: + if len(jobID) > 0 { + newStatusCode = http.StatusOK + rw.Header().Set("Location", fmt.Sprintf("/v2/status?id=%s", jobID)) + resp = AsyncUploadResult{JobID: jobID} + } else { + newStatusCode = http.StatusInternalServerError + resp = ScanError{"internal error"} + } + } + + } else if strings.Contains(req.URL.Path, "/v2/status") { + id, _ := req.URL.Query()["id"] + + switch newStatusCode { + case http.StatusBadRequest: + resp = ScanError{id[0]} + case http.StatusNotFound: + resp = ScanError{"job uuid not found"} + case http.StatusOK: + resp = asyncScanResult + default: + newStatusCode = http.StatusInternalServerError + resp = ScanError{"internal error"} + } + } + + rw.WriteHeader(newStatusCode) + json.NewEncoder(&b).Encode(&resp) + rw.Write([]byte(b.Bytes())) + })) + defer server.Close() + + httpClient := &piperhttp.Client{} + malwareService := ClientImpl{ + HTTPClient: httpClient, + Host: server.URL, + PollInterval: time.Second * 1, + PollingTimeout: time.Second * 5, + } + + cases := []struct { + name string + jobID string + statusCode int + asyncResult *AsyncScanResult + want interface{} + }{ + {name: "test wrong content type", jobID: "test-contenttype", statusCode: http.StatusNotAcceptable, want: fmt.Errorf("MalwareService returned with status code 406: wrong content type")}, + {name: "test wrong file type", jobID: "test-badfiletype", statusCode: http.StatusUnsupportedMediaType, want: fmt.Errorf("MalwareService returned with status code 415: unsupported media type")}, + {name: "test internal error", jobID: "", statusCode: http.StatusInternalServerError, want: fmt.Errorf("MalwareService returned with status code 500: internal error")}, + {name: "test wrong job id", jobID: "test-badjobid", statusCode: http.StatusBadRequest, want: fmt.Errorf("MalwareService returned with status code 400: test-badjobid")}, + {name: "test bad archive", jobID: "test-badarchive", statusCode: http.StatusBadRequest, want: fmt.Errorf("MalwareService returned with status code 400: test-badarchive")}, + {name: "test job id not found", jobID: "test-jobidnotfound", statusCode: http.StatusNotFound, want: fmt.Errorf("MalwareService returned with status code 404: job uuid not found")}, + { + name: "test status check timeout", + jobID: "test-timeout", + statusCode: http.StatusOK, + asyncResult: &AsyncScanResult{Created: time.Now(), Status: STATE_PROCESSING, Updated: time.Now()}, + want: fmt.Errorf("Time out getting scan results."), + }, + { + name: "test successful scan", + jobID: "test-success", + statusCode: http.StatusOK, + asyncResult: &AsyncScanResult{Created: time.Now(), Sha256: "test-success", SizeByte: 1, Status: STATE_DONE, Updated: time.Now()}, + want: &ScanResult{MalwareDetected: false, EncryptedContentDetected: false, ScanSize: 1, Finding: "[]", MimeType: "application/x-tar", SHA256: "test-success"}, + }, + { + name: "test successful scan with malware", + jobID: "test-walmare", + statusCode: http.StatusOK, + asyncResult: &AsyncScanResult{Created: time.Now(), Sha256: "test-walmare", SizeByte: 1, Status: STATE_DONE, Updated: time.Now(), Findings: []AsyncIncident{{Message: "found one!", FilePath: "/tmp/malware.zip"}}}, + want: &ScanResult{MalwareDetected: true, EncryptedContentDetected: false, ScanSize: 1, Finding: "[{Message:found one! FilePath:/tmp/malware.zip}]", MimeType: "application/x-tar", SHA256: "test-walmare"}, + }, + { + name: "test successful scan with encrypted content", + jobID: "test-encrypted", + statusCode: http.StatusOK, + asyncResult: &AsyncScanResult{Created: time.Now(), Sha256: "test-encrypted", SizeByte: 1, Status: STATE_DONE, Updated: time.Now(), Encrypted: []AsyncIncident{{Message: "found one!", FilePath: "/tmp/encrypted.zip"}}}, + want: &ScanResult{MalwareDetected: false, EncryptedContentDetected: true, ScanSize: 1, Finding: "[{Message:found one! FilePath:/tmp/encrypted.zip}]", MimeType: "application/x-tar", SHA256: "test-encrypted"}, + }, + { + name: "test scan with error", + jobID: "test-scanerror", + statusCode: http.StatusOK, + asyncResult: &AsyncScanResult{Created: time.Now(), Sha256: "test-scanerror", SizeByte: 1, Status: STATE_ERROR, Updated: time.Now(), Errors: []AsyncIncident{{Message: "found one!", FilePath: "/tmp/error.zip"}}}, + want: fmt.Errorf("Error while scanning:\nfound one!"), + }, + } + + testFile, err := ioutil.TempFile("", "testFileUpload.tar") + if err != nil { + t.FailNow() + } + defer os.RemoveAll(testFile.Name()) // clean up + + for _, c := range cases { + t.Run(c.name, func(t *testing.T) { + jobID = c.jobID + statusCode = c.statusCode + asyncScanResult = c.asyncResult + scanResult, err := malwareService.ScanAsync(testFile.Name()) + if err != nil { + assert.Error(t, err) + assert.Equal(t, c.want, err) + } else { + assert.NoError(t, err) + assert.Equal(t, c.want, scanResult) + } + }) + + } +} + func TestMalwareServiceScan(t *testing.T) { t.Run("Scan without finding", func(t *testing.T) { httpClient := &httpMock{StatusCode: 200, ResponseBody: "{\"malwareDetected\":false,\"encryptedContentDetected\":false,\"scanSize\":298782,\"mimeType\":\"application/octet-stream\",\"SHA256\":\"96ca802fbd54d31903f1115a1d95590c685160637d9262bd340ab30d0f817e85\"}"} diff --git a/resources/metadata/malwareExecuteScan.yaml b/resources/metadata/malwareExecuteScan.yaml index b86208cf7e..130f8ba81f 100644 --- a/resources/metadata/malwareExecuteScan.yaml +++ b/resources/metadata/malwareExecuteScan.yaml @@ -151,6 +151,30 @@ spec: - STAGES - STEPS default: malwarescan_report.json + - name: asynchronous + type: bool + description: "Enable asynchronous scanning API." + scope: + - PARAMETERS + - STAGES + - STEPS + default: false + - name: pollingInterval + type: string + description: "Polling interval for asynchronous scan in seconds." + scope: + - PARAMETERS + - STAGES + - STEPS + default: 10 + - name: pollingTimeout + type: string + description: "Polling timeout for asynchronous scan API in seconds." + scope: + - PARAMETERS + - STAGES + - STEPS + default: 600 outputs: resources: - name: reports