Merge remote-tracking branch 'origin/malwarescan-async-scan' into v1.…

…275.0-edp

cmd/malwareExecuteScan.go

@@ -3,6 +3,11 @@ package cmd
 import (
 	"encoding/json"
 	"fmt"
+	"io"
+	"os"
+	"strings"
+	"time"
+
 	piperDocker "github.com/SAP/jenkins-library/pkg/docker"
 	piperhttp "github.com/SAP/jenkins-library/pkg/http"
 	"github.com/SAP/jenkins-library/pkg/log"
@@ -11,10 +16,6 @@ import (
 	"github.com/SAP/jenkins-library/pkg/telemetry"
 	"github.com/SAP/jenkins-library/pkg/toolrecord"
 	"github.com/pkg/errors"
-	"io"
-	"os"
-	"strings"
-	"time"
 )
 
 type malwareScanUtils interface {
@@ -45,10 +46,22 @@ func (utils *malwareScanUtilsBundle) newDockerClient(options piperDocker.ClientO
 func newMalwareScanUtilsBundle(config malwareExecuteScanOptions) *malwareScanUtilsBundle {
 	timeout, err := time.ParseDuration(fmt.Sprintf("%ss", config.Timeout))
 	if err != nil {
-		timeout = 60
+		timeout = 60 * time.Second
 		log.Entry().Warnf("Unable to parse timeout for malwareScan: '%v'. Falling back to %ds", err, timeout)
 	}
 
+	pollinterval, err := time.ParseDuration(fmt.Sprintf("%ss", config.PollingInterval))
+	if err != nil {
+		pollinterval = 10 * time.Second
+		log.Entry().Warnf("Unable to parse poll interval for malwareScan: '%v'. Falling back to %ds", err, pollinterval)
+	}
+
+	pollingTimeout, err := time.ParseDuration(fmt.Sprintf("%ss", config.PollingTimeout))
+	if err != nil {
+		pollingTimeout = 600 * time.Second
+		log.Entry().Warnf("Unable to parse poll timeout for malwareScan: '%v'. Falling back to %ds", err, pollingTimeout)
+	}
+
 	httpClientOptions := piperhttp.ClientOptions{
 		Username:           config.Username,
 		Password:           config.Password,
@@ -61,8 +74,10 @@ func newMalwareScanUtilsBundle(config malwareExecuteScanOptions) *malwareScanUti
 
 	return &malwareScanUtilsBundle{
 		Client: &malwarescan.ClientImpl{
-			HTTPClient: httpClient,
-			Host:       config.Host,
+			HTTPClient:     httpClient,
+			Host:           config.Host,
+			PollInterval:   pollinterval,
+			PollingTimeout: pollingTimeout,
 		},
 		Files: &piperutils.Files{},
 	}
@@ -85,12 +100,6 @@ func runMalwareScan(config *malwareExecuteScanOptions, telemetryData *telemetry.
 
 	log.Entry().Infof("Scanning file \"%s\" for malware using service \"%s\"", file, config.Host)
 
-	candidate, err := utils.OpenFile(file, os.O_RDONLY, 0666)
-	if err != nil {
-		return err
-	}
-	defer candidate.Close()
-
 	scannerInfo, err := utils.Info()
 
 	log.Entry().Infof("***************************************")
@@ -102,10 +111,25 @@ func runMalwareScan(config *malwareExecuteScanOptions, telemetryData *telemetry.
 		return err
 	}
 
-	scanResponse, err := utils.Scan(candidate)
+	var scanResponse *malwarescan.ScanResult
 
-	if err != nil {
-		return err
+	if config.Asynchronous {
+		scanResponse, err = utils.ScanAsync(file)
+		if err != nil {
+			return err
+		}
+
+	} else {
+		candidate, err := utils.OpenFile(file, os.O_RDONLY, 0666)
+		if err != nil {
+			return err
+		}
+		defer candidate.Close()
+
+		scanResponse, err = utils.Scan(candidate)
+		if err != nil {
+			return err
+		}
 	}
 
 	if err = createMalwareScanReport(config, scanResponse, utils); err != nil {

cmd/malwareExecuteScan_test.go

@@ -53,6 +53,10 @@ func (utils *malwareScanUtilsMockBundle) Scan(candidate io.Reader) (*malwarescan
 	return utils.returnScanResult, nil
 }
 
+func (utils *malwareScanUtilsMockBundle) ScanAsync(file string) (*malwarescan.ScanResult, error) {
+	return utils.returnScanResult, nil
+}
+
 func (utils *malwareScanUtilsMockBundle) newDockerClient(options piperDocker.ClientOptions) piperDocker.Download {
 	return &dockerClientMock{imageName: options.ImageName, registryURL: options.RegistryURL, localPath: options.LocalPath}
 }

pkg/malwarescan/malwarescan.go

@@ -1,13 +1,19 @@
 package malwarescan
 
 import (
+	"bytes"
 	"encoding/json"
 	"fmt"
-	piperhttp "github.com/SAP/jenkins-library/pkg/http"
-	"github.com/pkg/errors"
 	"io"
 	"io/ioutil"
 	"net/http"
+	"os"
+	"strings"
+	"time"
+
+	piperhttp "github.com/SAP/jenkins-library/pkg/http"
+	"github.com/SAP/jenkins-library/pkg/log"
+	"github.com/pkg/errors"
 )
 
 // ScanResult : Returned by the scan endpoint of the malwarescan api of SAP CP
@@ -20,6 +26,41 @@ type ScanResult struct {
 	SHA256                   string `json:"SHA256"`
 }
 
+// AsyncUploadResult : Upload result returned by the malwarescan asynchronous scan api
+type AsyncUploadResult struct {
+	JobID string `json:"jobID"`
+}
+
+// AsyncStatus : Scan status returned by the malwarescan asynchronous scan api
+type AsyncStatus string
+
+const (
+	STATE_PROCESSING AsyncStatus = "processing"
+	STATE_DONE       AsyncStatus = "done"
+	STATE_ERROR      AsyncStatus = "error"
+)
+
+// AsyncIncident : Incident detail returned by the malwarescan asynchronous scan result
+type AsyncIncident struct {
+	Message  string `json:"message"`
+	FilePath string `json:"filePath"`
+}
+
+// AsyncScanResult : Scan result returned by the malwarescan asynchronous scan api
+type AsyncScanResult struct {
+	Created       time.Time       `json:"created"`
+	Errors        []AsyncIncident `json:"errors"`
+	Findings      []AsyncIncident `json:"findings"`
+	Encrypted     []AsyncIncident `json:"encryptedContent"`
+	Processed     int             `json:"processed"`
+	FileCount     int             `json:"fileCount"`
+	Sha256        string          `json:"sha256"`
+	SizeByte      int             `json:"sizeByte"`
+	Status        AsyncStatus     `json:"status"`
+	Updated       time.Time       `json:"updated"`
+	CorrelationID string          `json:"correlation_id"`
+}
+
 // Info : Returned by the info endpoint of the malwarescan api of SAP CP
 type Info struct {
 	MaxScanSize        int
@@ -34,14 +75,113 @@ type ScanError struct {
 
 // Client : Interface for the malwarescan api provided by SAP CP (see https://api.sap.com/api/MalwareScanAPI/overview)
 type Client interface {
+	// ScanAsync(candidate io.Reader) (*ScanResult, error)
+	ScanAsync(file string) (*ScanResult, error)
 	Scan(candidate io.Reader) (*ScanResult, error)
 	Info() (*Info, error)
 }
 
 // ClientImpl : Client implementation of the malwarescan api provided by SAP CP (see https://api.sap.com/api/MalwareScanAPI/overview)
 type ClientImpl struct {
-	HTTPClient piperhttp.Sender
-	Host       string
+	HTTPClient     piperhttp.Sender
+	Host           string
+	PollInterval   time.Duration
+	PollingTimeout time.Duration
+}
+
+// ScanAsync : Triggers an asynchronous malwarescan for the given content.
+func (c *ClientImpl) ScanAsync(file string) (*ScanResult, error) {
+	var scanResult ScanResult
+	var asyncUploadResult AsyncUploadResult
+
+	log.Entry().Debugf("Starting asynchronous scan")
+
+	// Without using a bytes.Reader for the request body, the net/http NewRequest
+	// method does not set Content-Length header and set the transfer to "chunked"
+	// which is not supported by the malware scan api.
+	fileHandle, err := os.ReadFile(file)
+	if err != nil {
+		return nil, err
+	}
+
+	err = c.sendAPIRequest("POST", "/v2/tar", bytes.NewReader(fileHandle), nil, &asyncUploadResult)
+	if err != nil {
+		return nil, err
+	}
+
+	if len(asyncUploadResult.JobID) != 0 {
+		log.Entry().Debugf("File uploaded successfuly, start polling for job status")
+		scanResult, err = c.pollResult(asyncUploadResult.JobID)
+		if err != nil {
+			return nil, err
+		}
+
+	} else {
+		return nil, fmt.Errorf("Malware asynchronous scan upload failed, missing job id.")
+	}
+
+	return &scanResult, nil
+}
+
+// pollResult polls the malwarescan api for the scan result
+func (c *ClientImpl) pollResult(jobID string) (ScanResult, error) {
+	var scanResult ScanResult
+	var asyncScanResult AsyncScanResult
+	var err error
+
+	ticker := time.NewTicker(c.PollInterval)
+	defer ticker.Stop()
+
+	timeout := time.NewTimer(c.PollingTimeout)
+	defer timeout.Stop()
+
+	start := time.Now()
+	done := make(chan bool)
+
+	go func() {
+		for t := time.Now(); true; t = <-ticker.C {
+			log.Entry().Debugf("Checking job ID %v status [elapsed time: %.0f/%.0fs]", jobID, t.Sub(start).Seconds(), c.PollingTimeout.Seconds())
+
+			err = c.sendAPIRequest(http.MethodGet, fmt.Sprintf("/v2/status?id=%v", jobID), nil, nil, &asyncScanResult)
+			if err != nil {
+				done <- true
+			}
+
+			if asyncScanResult.Status == STATE_ERROR {
+				errmsg := []string{}
+				for _, incident := range asyncScanResult.Errors {
+					errmsg = append(errmsg, incident.Message)
+				}
+				err = fmt.Errorf("Error while scanning:\n%s", strings.Join(errmsg, "\n"))
+				done <- true
+
+			} else if asyncScanResult.Status == STATE_DONE {
+				findings := append(asyncScanResult.Findings, asyncScanResult.Encrypted...)
+				scanResult.MalwareDetected = len(asyncScanResult.Findings) > 0
+				scanResult.EncryptedContentDetected = len(asyncScanResult.Encrypted) > 0
+				scanResult.ScanSize = asyncScanResult.SizeByte
+				scanResult.Finding = fmt.Sprintf("%+v", findings)
+				scanResult.MimeType = "application/x-tar"
+				scanResult.SHA256 = asyncScanResult.Sha256
+				done <- true
+			} else {
+				log.Entry().Debugf("Scan still in progress. correlation ID %s", asyncScanResult.CorrelationID)
+			}
+		}
+	}()
+
+loop:
+	for {
+		select {
+		case <-done:
+			break loop
+		case <-timeout.C:
+			err = fmt.Errorf("Time out getting scan results.")
+			break loop
+		}
+	}
+
+	return scanResult, err
 }
 
 // Scan : Triggers a malwarescan in SAP CP for the given content.
@@ -58,7 +198,6 @@ func (c *ClientImpl) Scan(candidate io.Reader) (*ScanResult, error) {
 	}
 
 	return &scanResult, nil
-
 }
 
 // Info : Returns some information about the scanengine used by the malwarescan service.
@@ -80,14 +219,12 @@ func (c *ClientImpl) sendAPIRequest(method, endpoint string, body io.Reader, hea
 
 	// sendRequest results in any combination of nil and non-nil response and error.
 	// a response body could even be already closed.
-	response, err := c.HTTPClient.SendRequest(method, c.Host+endpoint, body, header, nil)
+	response, _ := c.HTTPClient.SendRequest(method, c.Host+endpoint, body, header, nil)
 
-	if response.StatusCode != 200 {
+	if response.StatusCode < 200 || response.StatusCode >= 300 {
 		var scanError ScanError
 
-		err = c.unmarshalResponse(response, &scanError)
-
-		if err != nil {
+		if err := c.unmarshalResponse(response, &scanError); err != nil {
 			return fmt.Errorf("MalwareService returned with status code %d, no further information available", response.StatusCode)
 		}