From 464027b6a956e28586efe1a41095cd30e20fdf76 Mon Sep 17 00:00:00 2001 From: yahavi Date: Thu, 7 Sep 2023 19:06:24 +0300 Subject: [PATCH] Support JFrog Apps Config file --- src/main/scanLogic/scanManager.ts | 78 ++++++------ .../scanLogic/scanRunners/analyzerModels.ts | 2 +- .../scanLogic/scanRunners/binaryRunner.ts | 89 +++++++------ src/main/scanLogic/scanRunners/iacScan.ts | 12 +- .../scanRunners/{eosScan.ts => sastScan.ts} | 79 ++++++++---- src/main/scanLogic/scanRunners/secretsScan.ts | 12 +- .../{eosTreeNode.ts => sastTreeNode.ts} | 6 +- .../issuesTree/fileTreeNode.ts | 4 +- .../issuesTree/issuesRootTreeNode.ts | 22 ++-- .../issuesTree/issuesTreeDataProvider.ts | 79 ++++++------ .../treeDataProviders/utils/analyzerUtils.ts | 103 ++++++--------- .../utils/dependencyUtils.ts | 4 +- src/main/types/jfrogAppsConfig.ts | 35 +++++ src/main/types/workspaceIssuesDetails.ts | 32 ++--- src/main/utils/appConfigUtils.ts | 53 ++++++++ src/main/utils/translators.ts | 42 +++--- .../applicableScan/analyzerResponse.json | 12 +- .../npm/expectedScanResponse.json | 4 +- .../.jfrog/jfrog-apps-config.yml | 50 ++++++++ src/test/tests/appsConfig.test.ts | 120 ++++++++++++++++++ src/test/tests/integration/iac.test.ts | 15 +-- src/test/tests/integration/secrets.test.ts | 3 +- src/test/tests/issuesTreeNode.test.ts | 2 +- 23 files changed, 563 insertions(+), 295 deletions(-) rename src/main/scanLogic/scanRunners/{eosScan.ts => sastScan.ts} (81%) rename src/main/treeDataProviders/issuesTree/codeFileTree/{eosTreeNode.ts => sastTreeNode.ts} (95%) create mode 100644 src/main/types/jfrogAppsConfig.ts create mode 100644 src/main/utils/appConfigUtils.ts create mode 100644 src/test/resources/jfrogAppsConfig/.jfrog/jfrog-apps-config.yml create mode 100644 src/test/tests/appsConfig.test.ts diff --git a/src/main/scanLogic/scanManager.ts b/src/main/scanLogic/scanManager.ts index 4a1468d8c..3fe95a71d 100644 --- a/src/main/scanLogic/scanManager.ts +++ b/src/main/scanLogic/scanManager.ts @@ -2,29 +2,31 @@ import * as vscode from 'vscode'; import { ExtensionComponent } from '../extensionComponent'; -import { LogManager } from '../log/logManager'; import { ConnectionManager } from '../connect/connectionManager'; import { ConnectionUtils, EntitlementScanFeature } from '../connect/connectionUtils'; +import { LogManager } from '../log/logManager'; -import { RootNode } from '../treeDataProviders/dependenciesTree/dependenciesRoot/rootTree'; import { IGraphResponse, XrayScanProgress } from 'jfrog-client-js'; -import { GraphScanLogic } from './scanGraphLogic'; -import { ApplicabilityRunner, ApplicabilityScanResponse } from './scanRunners/applicabilityScan'; -import { EosRunner, EosScanRequest, EosScanResponse } from './scanRunners/eosScan'; +import { RootNode } from '../treeDataProviders/dependenciesTree/dependenciesRoot/rootTree'; import { AnalyzerUtils } from '../treeDataProviders/utils/analyzerUtils'; +import { StepProgress } from '../treeDataProviders/utils/stepProgress'; +import { ExcludeScanner, Module } from '../types/jfrogAppsConfig'; +import { AppsConfigUtils } from '../utils/appConfigUtils'; import { Configuration } from '../utils/configuration'; import { Resource } from '../utils/resource'; -import { BinaryRunner } from './scanRunners/binaryRunner'; import { ScanUtils } from '../utils/scanUtils'; -import { StepProgress } from '../treeDataProviders/utils/stepProgress'; import { Utils } from '../utils/utils'; +import { GraphScanLogic } from './scanGraphLogic'; +import { ApplicabilityRunner, ApplicabilityScanResponse } from './scanRunners/applicabilityScan'; +import { BinaryRunner } from './scanRunners/binaryRunner'; import { IacRunner, IacScanResponse } from './scanRunners/iacScan'; +import { EosScanResponse, SastRunner } from './scanRunners/sastScan'; import { SecretsRunner, SecretsScanResponse } from './scanRunners/secretsScan'; export interface SupportedScans { dependencies: boolean; applicability: boolean; - eos: boolean; + sast: boolean; iac: boolean; secrets: boolean; } @@ -167,9 +169,9 @@ export class ScanManager implements ExtensionComponent { } /** - * Check if Eos scan is supported for the user + * Check if SAST scan is supported for the user */ - public async isEosSupported(): Promise { + public async isSastSupported(): Promise { return true; } @@ -200,8 +202,8 @@ export class ScanManager implements ExtensionComponent { .catch(err => ScanUtils.onScanError(err, this._logManager, true)) ); requests.push( - this.isEosSupported() - .then(res => (supportedScans.eos = res)) + this.isSastSupported() + .then(res => (supportedScans.sast = res)) .catch(err => ScanUtils.onScanError(err, this._logManager, true)) ); await Promise.all(requests); @@ -250,61 +252,57 @@ export class ScanManager implements ExtensionComponent { /** * Scan directory for 'Infrastructure As Code' (Iac) issues. - * @param directory - the directory that will be scan + * @param module - the module that will be scanned * @param checkCancel - check if should cancel * @returns the Iac scan response */ - public async scanIac(directory: string, checkCancel: () => void): Promise { + public async scanIac(module: Module, checkCancel: () => void): Promise { let iacRunner: IacRunner = new IacRunner(this._connectionManager, this.logManager); if (!iacRunner.validateSupported()) { this._logManager.logMessage('Iac runner could not find binary to run', 'WARN'); return {} as IacScanResponse; } - let skipFiles: string[] = AnalyzerUtils.getAnalyzerManagerExcludePattern(Configuration.getScanExcludePattern()); - this._logManager.logMessage("Scanning directory '" + directory + "', for Iac issues. Skipping files: " + skipFiles, 'DEBUG'); - return await iacRunner.scan(directory, checkCancel, skipFiles); + if (AppsConfigUtils.ShouldSkipScanner(module, ExcludeScanner.Iac)) { + this._logManager.debug('Skipping IaC scanning'); + return {} as IacScanResponse; + } + return await iacRunner.scan(module, checkCancel); } /** * Scan directory for secrets issues. - * @param directory - the directory that will be scan + * @param module - the module that will be scanned * @param checkCancel - check if should cancel * @returns the Secrets scan response */ - public async scanSecrets(directory: string, checkCancel: () => void): Promise { + public async scanSecrets(module: Module, checkCancel: () => void): Promise { let secretsRunner: SecretsRunner = new SecretsRunner(this._connectionManager, this.logManager); if (!secretsRunner.validateSupported()) { this._logManager.logMessage('Secrets runner could not find binary to run', 'WARN'); return {} as SecretsScanResponse; } - let skipFiles: string[] = AnalyzerUtils.getAnalyzerManagerExcludePattern(Configuration.getScanExcludePattern()); - this._logManager.logMessage("Scanning directory '" + directory + "', for Secrets issues. Skipping files: " + skipFiles, 'DEBUG'); - return await secretsRunner.scan(directory, checkCancel, skipFiles); + if (AppsConfigUtils.ShouldSkipScanner(module, ExcludeScanner.Secrets)) { + this._logManager.debug('Skipping secrets scanning'); + return {} as SecretsScanResponse; + } + return await secretsRunner.scan(module, checkCancel); } /** - * Scan for Eos issues. - * @param checkCancel - check if should cancel - * @param requests - the Eos requests to run + * Scan for SAST issues. + * @param module - the module that will be scanned + * @param requests - the SAST requests to run * @returns the scan response */ - public async scanEos(checkCancel: () => void, runDirectory?: string, ...requests: EosScanRequest[]): Promise { - let eosRunner: EosRunner = new EosRunner(this._connectionManager, this._logManager, undefined, undefined, runDirectory); - if (!eosRunner.validateSupported()) { - this._logManager.logMessage('Eos runner could not find binary to run', 'WARN'); + public async scanSast(module: Module, checkCancel: () => void): Promise { + let sastRunner: SastRunner = new SastRunner(this._connectionManager, this._logManager); + if (!sastRunner.validateSupported()) { + this._logManager.logMessage('Sast runner could not find binary to run', 'WARN'); return {} as EosScanResponse; } - if (requests.length === 0) { - this._logManager.logMessage('Eos runner must receive at least one request to run', 'ERR'); + if (AppsConfigUtils.ShouldSkipScanner(module, ExcludeScanner.Sast)) { + this._logManager.debug('Skipping SAST scanning'); return {} as EosScanResponse; } - let skipFiles: string[] = AnalyzerUtils.getAnalyzerManagerExcludePattern(Configuration.getScanExcludePattern()); - this._logManager.logMessage( - 'Scanning for Eos issues in ' + - requests.map(request => `(Language '${request.language}', roots: [${request.roots.join()}])`) + - '. Skipping files: ' + - skipFiles, - 'DEBUG' - ); - return eosRunner.scan(checkCancel, skipFiles, ...requests); + return sastRunner.scan(module, checkCancel); } } diff --git a/src/main/scanLogic/scanRunners/analyzerModels.ts b/src/main/scanLogic/scanRunners/analyzerModels.ts index 0a451ce4a..4decf6c63 100644 --- a/src/main/scanLogic/scanRunners/analyzerModels.ts +++ b/src/main/scanLogic/scanRunners/analyzerModels.ts @@ -5,7 +5,7 @@ export interface AnalyzerRequest { export enum ScanType { ContextualAnalysis = 'analyze-applicability', Iac = 'iac-scan-modules', - Eos = 'analyze-codebase', + Sast = 'sast', Secrets = 'secrets-scan' } diff --git a/src/main/scanLogic/scanRunners/binaryRunner.ts b/src/main/scanLogic/scanRunners/binaryRunner.ts index 070f0ca83..41944c503 100644 --- a/src/main/scanLogic/scanRunners/binaryRunner.ts +++ b/src/main/scanLogic/scanRunners/binaryRunner.ts @@ -1,19 +1,19 @@ import * as fs from 'fs'; -import yaml from 'js-yaml'; +import yaml, { DumpOptions } from 'js-yaml'; import * as path from 'path'; -import { LogManager } from '../../log/logManager'; -import { Utils } from '../../utils/utils'; -import { NotEntitledError, NotSupportedError, OsNotSupportedError, ScanCancellationError, ScanUtils } from '../../utils/scanUtils'; -import { AnalyzerRequest, AnalyzerScanResponse, ScanType, AnalyzeScanRequest } from './analyzerModels'; +import { IProxyConfig } from 'jfrog-client-js'; import { ConnectionManager } from '../../connect/connectionManager'; import { ConnectionUtils } from '../../connect/connectionUtils'; -import { IProxyConfig } from 'jfrog-client-js'; +import { LogManager } from '../../log/logManager'; +import { LogUtils } from '../../log/logUtils'; import { Configuration } from '../../utils/configuration'; import { Resource } from '../../utils/resource'; import { RunUtils } from '../../utils/runUtils'; +import { NotEntitledError, NotSupportedError, OsNotSupportedError, ScanCancellationError, ScanUtils } from '../../utils/scanUtils'; import { Translators } from '../../utils/translators'; -import { LogUtils } from '../../log/logUtils'; +import { Utils } from '../../utils/utils'; +import { AnalyzeScanRequest, AnalyzerRequest, AnalyzerScanResponse, ScanType } from './analyzerModels'; /** * Arguments for running binary async @@ -47,7 +47,7 @@ export abstract class BinaryRunner { protected _verbose: boolean = false; private static readonly RUNNER_NAME: string = 'analyzerManager'; - public static readonly RUNNER_VERSION: string = '1.3.2.2005632'; + public static readonly RUNNER_VERSION: string = '1.3.2.2019257'; private static readonly DOWNLOAD_URL: string = '/xsc-gen-exe-analyzer-manager-local/v1/'; public static readonly NOT_ENTITLED: number = 31; @@ -106,9 +106,15 @@ export abstract class BinaryRunner { * Run the executeBinary method with the provided request path * @param checkCancel - check if cancel * @param yamlConfigPath - the path to the request - * @param executionLogDirectory - og file will be written to the dir + * @param executionLogDirectory - log file will be written to the dir + * @param responsePath - path to the output file */ - protected abstract runBinary(yamlConfigPath: string, executionLogDirectory: string | undefined, checkCancel: () => void): Promise; + protected abstract runBinary( + yamlConfigPath: string, + executionLogDirectory: string | undefined, + checkCancel: () => void, + responsePath: string | undefined + ): Promise; /** * Validates that the binary exists and can run @@ -137,11 +143,9 @@ export abstract class BinaryRunner { * @param executionLogDirectory - the directory to save the execution log in */ private async executeBinaryTask(args: string[], executionLogDirectory?: string): Promise { - let std: any = await ScanUtils.executeCmdAsync( - '"' + this._binary.fullPath + '" ' + args.join(' '), - this._runDirectory, - this.createEnvForRun(executionLogDirectory) - ); + let command: string = '"' + this._binary.fullPath + '" ' + args.join(' '); + this._logManager.debug('Executing ' + command); + let std: any = await ScanUtils.executeCmdAsync(command, this._runDirectory, this.createEnvForRun(executionLogDirectory)); if (std.stdout && std.stdout.length > 0) { this.logTaskResult(std.stdout, false); } @@ -230,11 +234,11 @@ export abstract class BinaryRunner { return url; } - public async run(checkCancel: () => void, ...requests: AnalyzeScanRequest[]): Promise { + public async run(checkCancel: () => void, request: AnalyzeScanRequest): Promise { if (!this.validateSupported()) { return undefined; } - let args: RunArgs = this.createRunArguments(...requests); + let args: RunArgs = this.createRunArguments(request); try { if (args.requests.length == 0) { return undefined; @@ -250,27 +254,27 @@ export abstract class BinaryRunner { * @param requests - the run requests information * @return run arguments for the given requests */ - private createRunArguments(...requests: AnalyzeScanRequest[]): RunArgs { + private createRunArguments(request: AnalyzeScanRequest): RunArgs { let args: RunArgs = new RunArgs(ScanUtils.createTmpDir()); let processedRoots: Set = new Set(); - for (const request of requests) { - if (request.roots.length > 0 && request.roots.every(root => !processedRoots.has(root))) { - // Prepare request information and insert as an actual request - const requestPath: string = path.join(args.directory, 'request_' + args.requests.length); - const responsePath: string = path.join(args.directory, 'response_' + args.requests.length); + if (request.roots.length > 0 && request.roots.every(root => !processedRoots.has(root))) { + // Prepare request information and insert as an actual request + const requestPath: string = path.join(args.directory, 'request_' + args.requests.length); + const responsePath: string = path.join(args.directory, 'response_' + args.requests.length); + if (request.type !== ScanType.Sast) { request.output = responsePath; - request.type = this._type; - request.roots.forEach(root => processedRoots.add(root)); - // Add request to run - args.requests.push({ - type: request.type, - request: this.requestsToYaml(request), - requestPath: requestPath, - roots: request.roots, - responsePath: responsePath - } as RunRequest); } + request.type = this._type; + request.roots.forEach(root => processedRoots.add(root)); + // Add request to run + args.requests.push({ + type: request.type, + request: this.requestsToYaml(request), + requestPath: requestPath, + roots: request.roots, + responsePath: responsePath + } as RunRequest); } return args; @@ -283,9 +287,12 @@ export abstract class BinaryRunner { */ public requestsToYaml(...requests: AnalyzeScanRequest[]): string { return yaml - .dump({ - scans: requests - } as AnalyzerRequest) + .dump( + { + scans: requests + } as AnalyzerRequest, + { lineWidth: 1000 } as DumpOptions + ) .replace('skipped_folders', 'skipped-folders'); } @@ -297,7 +304,7 @@ export abstract class BinaryRunner { this.runRequest( checkCancel, args.requests[i].request, - args.requests[i].type === ScanType.Eos ? args.requests[i].responsePath : args.requests[i].requestPath, + args.requests[i].requestPath, args.requests[i].type, args.requests[i].responsePath ) @@ -390,11 +397,11 @@ export abstract class BinaryRunner { responsePath: string ): Promise { // 1. Save requests as yaml file in folder - if (type !== ScanType.Eos) { - fs.writeFileSync(requestPath, request); - } + fs.writeFileSync(requestPath, request); + this._logManager.debug('Input YAML:\n' + request); + // 2. Run the binary - await this.runBinary(requestPath, this._verbose ? undefined : path.dirname(requestPath), checkCancel).catch(error => { + await this.runBinary(requestPath, this._verbose ? undefined : path.dirname(requestPath), checkCancel, responsePath).catch(error => { if (error.code) { // Not entitled to run binary if (error.code === BinaryRunner.NOT_ENTITLED) { diff --git a/src/main/scanLogic/scanRunners/iacScan.ts b/src/main/scanLogic/scanRunners/iacScan.ts index 4100d2d21..77e97a7ea 100644 --- a/src/main/scanLogic/scanRunners/iacScan.ts +++ b/src/main/scanLogic/scanRunners/iacScan.ts @@ -1,6 +1,8 @@ import { ConnectionManager } from '../../connect/connectionManager'; import { LogManager } from '../../log/logManager'; import { AnalyzerUtils, FileWithSecurityIssues } from '../../treeDataProviders/utils/analyzerUtils'; +import { Module } from '../../types/jfrogAppsConfig'; +import { AppsConfigUtils } from '../../utils/appConfigUtils'; import { Resource } from '../../utils/resource'; import { ScanUtils } from '../../utils/scanUtils'; import { AnalyzeScanRequest, AnalyzerScanResponse, ScanType } from './analyzerModels'; @@ -28,12 +30,16 @@ export class IacRunner extends BinaryRunner { await this.executeBinary(checkCancel, ['iac', yamlConfigPath], executionLogDirectory); } - public async scan(directory: string, checkCancel: () => void, skipFolders: string[] = []): Promise { + public async scan(module: Module, checkCancel: () => void): Promise { let request: AnalyzeScanRequest = { type: ScanType.Iac, - roots: [directory], - skipped_folders: skipFolders + roots: AppsConfigUtils.GetSourceRoots(module, module.scanners?.iac), + skipped_folders: AppsConfigUtils.GetExcludePatterns(module, module.scanners?.iac) } as AnalyzeScanRequest; + this._logManager.logMessage( + "Scanning directories '" + request.roots + "', for Iac issues. Skipping folders: " + request.skipped_folders, + 'DEBUG' + ); return await this.run(checkCancel, request).then(runResult => this.convertResponse(runResult)); } diff --git a/src/main/scanLogic/scanRunners/eosScan.ts b/src/main/scanLogic/scanRunners/sastScan.ts similarity index 81% rename from src/main/scanLogic/scanRunners/eosScan.ts rename to src/main/scanLogic/scanRunners/sastScan.ts index 3d5d31b3b..d5bfcff70 100644 --- a/src/main/scanLogic/scanRunners/eosScan.ts +++ b/src/main/scanLogic/scanRunners/sastScan.ts @@ -1,28 +1,34 @@ +import { ConnectionManager } from '../../connect/connectionManager'; import { LogManager } from '../../log/logManager'; -import { BinaryRunner } from './binaryRunner'; +import { AnalyzerUtils } from '../../treeDataProviders/utils/analyzerUtils'; +import { Module, SastScanner } from '../../types/jfrogAppsConfig'; +import { Severity } from '../../types/severity'; +import { AppsConfigUtils } from '../../utils/appConfigUtils'; +import { Resource } from '../../utils/resource'; +import { ScanUtils } from '../../utils/scanUtils'; +import { Translators } from '../../utils/translators'; import { AnalyzeIssue, - AnalyzerScanResponse, - AnalyzeScanRequest, AnalyzeLocation, - FileRegion, - FileLocation, + AnalyzeScanRequest, + AnalyzerScanResponse, CodeFlow, + FileLocation, + FileRegion, ScanType } from './analyzerModels'; -import { ConnectionManager } from '../../connect/connectionManager'; -import { AnalyzerUtils } from '../../treeDataProviders/utils/analyzerUtils'; -import { Resource } from '../../utils/resource'; -import { Severity } from '../../types/severity'; -import { Translators } from '../../utils/translators'; -import { ScanUtils } from '../../utils/scanUtils'; +import { BinaryRunner } from './binaryRunner'; -export interface EosScanRequest extends AnalyzeScanRequest { - language: LanguageType; +/** + * The request that is sent to the binary to scan Sast + */ +export interface SastScanRequest extends AnalyzeScanRequest { + language: string; exclude_patterns: string[]; + excluded_rules: string[]; } -export type LanguageType = 'python' | 'javascript' | 'java'; +export type LanguageType = 'python' | 'javascript' | 'typescript' | 'java'; export interface EosScanResponse { filesWithIssues: EosFileIssues[]; @@ -46,34 +52,53 @@ export interface EosIssueLocation { threadFlows: FileLocation[][]; } -export class EosRunner extends BinaryRunner { +export class SastRunner extends BinaryRunner { constructor( connectionManager: ConnectionManager, logManager: LogManager, binary?: Resource, - timeout: number = ScanUtils.ANALYZER_TIMEOUT_MILLISECS, - runDirectory?: string + timeout: number = ScanUtils.ANALYZER_TIMEOUT_MILLISECS ) { - super(connectionManager, timeout, ScanType.Eos, logManager, binary, runDirectory); + super(connectionManager, timeout, ScanType.Sast, logManager, binary); + } + + /** @override */ + protected async runBinary( + yamlConfigPath: string, + executionLogDirectory: string | undefined, + checkCancel: () => void, + responsePath: string + ): Promise { + await this.executeBinary(checkCancel, ['zd', yamlConfigPath, responsePath], executionLogDirectory); } /** @override */ - protected async runBinary(yamlConfigPath: string, executionLogDirectory: string | undefined, checkCancel: () => void): Promise { - await this.executeBinary(checkCancel, ['zd', yamlConfigPath], executionLogDirectory); + public requestsToYaml(...requests: AnalyzeScanRequest[]): string { + let str: string = super.requestsToYaml(...requests); + return str.replace('excluded_rules', 'excluded-rules'); } /** - * Scan for EOS issues + * Scan for SAST issues * @param checkCancel - check if cancel * @param requests - requests to run * @returns the response generated from the scan */ - public async scan(checkCancel: () => void, skipFiles: string[], ...requests: EosScanRequest[]): Promise { - requests.forEach(request => { - request.type = ScanType.Eos; - request.exclude_patterns = skipFiles; - }); - return await this.run(checkCancel, ...requests).then(runResult => this.generateScanResponse(runResult)); + public async scan(module: Module, checkCancel: () => void): Promise { + let sastScanner: SastScanner | undefined = module.scanners?.sast; + let request: SastScanRequest = { + type: ScanType.Sast, + roots: AppsConfigUtils.GetSourceRoots(module, sastScanner), + language: sastScanner?.language, + excluded_rules: sastScanner?.excluded_rules, + exclude_patterns: AppsConfigUtils.GetExcludePatterns(module, sastScanner) + } as SastScanRequest; + this._logManager.logMessage( + "Scanning directories '" + request.roots + "', for SAST issues. Skipping folders: " + request.exclude_patterns, + 'DEBUG' + ); + + return await this.run(checkCancel, request).then(runResult => this.generateScanResponse(runResult)); } /** diff --git a/src/main/scanLogic/scanRunners/secretsScan.ts b/src/main/scanLogic/scanRunners/secretsScan.ts index 03e490834..721e5a75c 100644 --- a/src/main/scanLogic/scanRunners/secretsScan.ts +++ b/src/main/scanLogic/scanRunners/secretsScan.ts @@ -1,6 +1,8 @@ import { ConnectionManager } from '../../connect/connectionManager'; import { LogManager } from '../../log/logManager'; import { AnalyzerUtils, FileWithSecurityIssues } from '../../treeDataProviders/utils/analyzerUtils'; +import { Module } from '../../types/jfrogAppsConfig'; +import { AppsConfigUtils } from '../../utils/appConfigUtils'; import { Resource } from '../../utils/resource'; import { ScanUtils } from '../../utils/scanUtils'; import { AnalyzeScanRequest, AnalyzerScanResponse, ScanType } from './analyzerModels'; @@ -28,12 +30,16 @@ export class SecretsRunner extends BinaryRunner { await this.executeBinary(checkCancel, ['sec', yamlConfigPath], executionLogDirectory); } - public async scan(directory: string, checkCancel: () => void, skipFolders: string[] = []): Promise { + public async scan(module: Module, checkCancel: () => void): Promise { let request: AnalyzeScanRequest = { type: ScanType.Secrets, - roots: [directory], - skipped_folders: skipFolders + roots: AppsConfigUtils.GetSourceRoots(module, module.scanners?.secrets), + skipped_folders: AppsConfigUtils.GetExcludePatterns(module, module.scanners?.secrets) } as AnalyzeScanRequest; + this._logManager.logMessage( + "Scanning directories '" + request.roots + "', for secrets. Skipping folders: " + request.skipped_folders, + 'DEBUG' + ); return await this.run(checkCancel, request).then(runResult => this.convertResponse(runResult)); } diff --git a/src/main/treeDataProviders/issuesTree/codeFileTree/eosTreeNode.ts b/src/main/treeDataProviders/issuesTree/codeFileTree/sastTreeNode.ts similarity index 95% rename from src/main/treeDataProviders/issuesTree/codeFileTree/eosTreeNode.ts rename to src/main/treeDataProviders/issuesTree/codeFileTree/sastTreeNode.ts index 1431469d3..72463bd87 100644 --- a/src/main/treeDataProviders/issuesTree/codeFileTree/eosTreeNode.ts +++ b/src/main/treeDataProviders/issuesTree/codeFileTree/sastTreeNode.ts @@ -1,6 +1,6 @@ import * as vscode from 'vscode'; import { IAnalysisStep, IEosPage, PageType } from 'jfrog-ide-webview'; -import { EosIssue, EosIssueLocation } from '../../../scanLogic/scanRunners/eosScan'; +import { EosIssue, EosIssueLocation } from '../../../scanLogic/scanRunners/sastScan'; import { SeverityUtils } from '../../../types/severity'; import { Translators } from '../../../utils/translators'; import { Utils } from '../../../utils/utils'; @@ -8,9 +8,9 @@ import { CodeFileTreeNode } from './codeFileTreeNode'; import { CodeIssueTreeNode } from './codeIssueTreeNode'; /** - * Describe a Eos issue + * Describe a SAST issue */ -export class EosTreeNode extends CodeIssueTreeNode { +export class SastTreeNode extends CodeIssueTreeNode { private _codeFlows: IAnalysisStep[][]; private _fullDescription?: string; diff --git a/src/main/treeDataProviders/issuesTree/fileTreeNode.ts b/src/main/treeDataProviders/issuesTree/fileTreeNode.ts index 9f526cf06..e4a99b419 100644 --- a/src/main/treeDataProviders/issuesTree/fileTreeNode.ts +++ b/src/main/treeDataProviders/issuesTree/fileTreeNode.ts @@ -44,8 +44,8 @@ export abstract class FileTreeNode extends vscode.TreeItem { public apply() { // If no description is set, show the full path of the file or the relative path base on the path of the parent workspace if exists let description: string | undefined = this._fullPath; - if (this._parent && this._fullPath.startsWith(this._parent.workSpace.uri.fsPath)) { - let localPath: string = this._fullPath.substring(this._parent.workSpace.uri.fsPath.length + 1); + if (this._parent && this._fullPath.startsWith(this._parent.workspace.uri.fsPath)) { + let localPath: string = this._fullPath.substring(this._parent.workspace.uri.fsPath.length + 1); if (localPath !== this.name) { description = '.' + path.sep + localPath; } else { diff --git a/src/main/treeDataProviders/issuesTree/issuesRootTreeNode.ts b/src/main/treeDataProviders/issuesTree/issuesRootTreeNode.ts index f9e74baf4..34b003ff3 100644 --- a/src/main/treeDataProviders/issuesTree/issuesRootTreeNode.ts +++ b/src/main/treeDataProviders/issuesTree/issuesRootTreeNode.ts @@ -9,12 +9,12 @@ import { FileTreeNode } from './fileTreeNode'; export class IssuesRootTreeNode extends vscode.TreeItem { private _children: FileTreeNode[] = []; private _title: string = ''; - private _eosScanTimeStamp?: number | undefined; + private _sastScanTimeStamp?: number | undefined; private _iacScanTimeStamp?: number | undefined; private _secretsScanTimeStamp?: number | undefined; - constructor(private readonly _workSpace: vscode.WorkspaceFolder, title?: string, collapsibleState?: vscode.TreeItemCollapsibleState) { - super(_workSpace.name, collapsibleState ?? vscode.TreeItemCollapsibleState.Expanded); + constructor(private readonly _workspace: vscode.WorkspaceFolder, title?: string, collapsibleState?: vscode.TreeItemCollapsibleState) { + super(_workspace.name, collapsibleState ?? vscode.TreeItemCollapsibleState.Expanded); this.title = title ?? ''; } @@ -34,7 +34,7 @@ export class IssuesRootTreeNode extends vscode.TreeItem { } this.tooltip = 'Issue count: ' + issueCount + '\n'; - this.tooltip += 'Full path: ' + this._workSpace.uri.fsPath + ''; + this.tooltip += 'Full path: ' + this._workspace.uri.fsPath + ''; if (this._title != '') { this.tooltip += '\nStatus: ' + this._title; } else if (this.oldestScanTimestamp) { @@ -77,12 +77,12 @@ export class IssuesRootTreeNode extends vscode.TreeItem { return this._children.find(child => child.projectFilePath === file); } - public get eosScanTimeStamp(): number | undefined { - return this._eosScanTimeStamp; + public get sastScanTimeStamp(): number | undefined { + return this._sastScanTimeStamp; } - public set eosScanTimeStamp(value: number | undefined) { - this._eosScanTimeStamp = value; + public set sastScanTimeStamp(value: number | undefined) { + this._sastScanTimeStamp = value; } public get iacScanTimeStamp(): number | undefined { @@ -109,7 +109,7 @@ export class IssuesRootTreeNode extends vscode.TreeItem { ...this.children.map(file => file.timeStamp), this._iacScanTimeStamp, this._secretsScanTimeStamp, - this._eosScanTimeStamp + this._sastScanTimeStamp ); } @@ -121,8 +121,8 @@ export class IssuesRootTreeNode extends vscode.TreeItem { this.description = val; } - public get workSpace(): vscode.WorkspaceFolder { - return this._workSpace; + public get workspace(): vscode.WorkspaceFolder { + return this._workspace; } public get children(): FileTreeNode[] { diff --git a/src/main/treeDataProviders/issuesTree/issuesTreeDataProvider.ts b/src/main/treeDataProviders/issuesTree/issuesTreeDataProvider.ts index 8fe5c819c..fc1911ea9 100644 --- a/src/main/treeDataProviders/issuesTree/issuesTreeDataProvider.ts +++ b/src/main/treeDataProviders/issuesTree/issuesTreeDataProvider.ts @@ -19,7 +19,7 @@ import { LicenseIssueTreeNode } from './descriptorTree/licenseIssueTreeNode'; import { CodeIssueTreeNode } from './codeFileTree/codeIssueTreeNode'; import { CodeFileTreeNode } from './codeFileTree/codeFileTreeNode'; import { ApplicableTreeNode } from './codeFileTree/applicableTreeNode'; -import { EosTreeNode } from './codeFileTree/eosTreeNode'; +import { SastTreeNode } from './codeFileTree/sastTreeNode'; import { EnvironmentTreeNode } from './descriptorTree/environmentTreeNode'; import { ProjectDependencyTreeNode } from './descriptorTree/projectDependencyTreeNode'; import { DependencyScanResults, EntryIssuesData, ScanResults } from '../../types/workspaceIssuesDetails'; @@ -27,6 +27,8 @@ import { AnalyzerUtils } from '../utils/analyzerUtils'; import { IacTreeNode } from './codeFileTree/iacTreeNode'; import { SecretTreeNode } from './codeFileTree/secretsTreeNode'; import { UsageUtils } from '../../utils/usageUtils'; +import { AppsConfigUtils } from '../../utils/appConfigUtils'; +import { JFrogAppsConfig } from '../../types/jfrogAppsConfig'; /** * Describes Xray issues data provider for the 'Issues' tree view and provides API to get issues data for files. @@ -167,14 +169,14 @@ export class IssuesTreeDataProvider implements vscode.TreeDataProvider): number { return ( - (supportedScans.eos ? 1 : 0) + + (supportedScans.sast ? 1 : 0) + (supportedScans.iac ? 1 : 0) + (supportedScans.secrets ? 1 : 0) + (supportedScans.dependencies ? descriptors.size + Array.from(descriptors.values()).reduce((acc, val) => acc + val.length, 0) : 0) @@ -200,8 +202,9 @@ export class IssuesTreeDataProvider implements vscode.TreeDataProvider this.onChangeFire(), this._logManager); - let workspaceDescriptors: Map = await ScanUtils.locatePackageDescriptors([root.workSpace], this._logManager); + let workspaceDescriptors: Map = await ScanUtils.locatePackageDescriptors([root.workspace], this._logManager); let subStepsCount: number = IssuesTreeDataProvider.getNumberOfTasksInRepopulate(this._scanManager.supportedScans, workspaceDescriptors); + let jfrogAppConfig: JFrogAppsConfig = AppsConfigUtils.LoadConfig(root.workspace.uri.path); checkCanceled(); UsageUtils.sendUsageReport(this._scanManager.supportedScans, workspaceDescriptors, this._treesManager.connectionManager); // Scan workspace @@ -223,33 +226,37 @@ export class IssuesTreeDataProvider implements vscode.TreeDataProvider - ScanUtils.onScanError(err, this._logManager, true) - ) - ); - } - if (this._scanManager.supportedScans.secrets) { - // Scan the workspace for Secrets issues - scansPromises.push( - AnalyzerUtils.runSecrets(scanResults, root, this._scanManager, progressManager).catch(err => - ScanUtils.onScanError(err, this._logManager, true) - ) - ); - } - if (this._scanManager.supportedScans.eos) { - // Scan the workspace for Eos issues - scansPromises.push( - AnalyzerUtils.runEos( - scanResults, - root, - Array.from(workspaceDescriptors.keys() ?? []), - this._scanManager, - progressManager - ).catch(err => ScanUtils.onScanError(err, this._logManager, true)) - ); + + for (let module of jfrogAppConfig.modules) { + if (this._scanManager.supportedScans.iac) { + // Scan the workspace for Infrastructure As Code (Iac) issues + scansPromises.push( + AnalyzerUtils.runIac(scanResults, root, this._scanManager, module, progressManager).catch(err => + ScanUtils.onScanError(err, this._logManager, true) + ) + ); + } + if (this._scanManager.supportedScans.secrets) { + // Scan the workspace for Secrets issues + scansPromises.push( + AnalyzerUtils.runSecrets(scanResults, root, this._scanManager, module, progressManager).catch(err => + ScanUtils.onScanError(err, this._logManager, true) + ) + ); + } + if (this._scanManager.supportedScans.sast) { + // Scan the workspace for Eos issues + scansPromises.push( + AnalyzerUtils.runSast( + scanResults, + root, + Array.from(workspaceDescriptors.keys() ?? []), + this._scanManager, + module, + progressManager + ).catch(err => ScanUtils.onScanError(err, this._logManager, true)) + ); + } } await Promise.all(scansPromises); @@ -280,7 +287,7 @@ export class IssuesTreeDataProvider implements vscode.TreeDataProvider { let startIacTime: number = Date.now(); - scanResults.iacScan = await scanManager.scanIac(root.workSpace.uri.fsPath, progressManager.checkCancel); + scanResults.iacScan = await scanManager.scanIac(module, progressManager.checkCancel); if (scanResults.iacScan) { scanResults.iacScanTimestamp = Date.now(); let issuesCount: number = AnalyzerUtils.populateIacIssues(root, scanResults); @@ -539,10 +541,11 @@ export class AnalyzerUtils { scanResults: ScanResults, root: IssuesRootTreeNode, scanManager: ScanManager, + module: Module, progressManager: StepProgress ): Promise { let startSecretsTime: number = Date.now(); - scanResults.secretsScan = await scanManager.scanSecrets(root.workSpace.uri.fsPath, progressManager.checkCancel); + scanResults.secretsScan = await scanManager.scanSecrets(module, progressManager.checkCancel); if (scanResults.secretsScan) { scanResults.secretsScanTimestamp = Date.now(); let issuesCount: number = AnalyzerUtils.populateSecretsIssues(root, scanResults); @@ -587,59 +590,33 @@ export class AnalyzerUtils { } /** - * Create Eos scan request for each given package type supported by the scan. - * Default will create request for all supported languages. - * @param root - the root node of the workspace - * @param languages - each supported language will generate a request - * @returns list of Eos requests - */ - private static createEosRequests(root: IssuesRootTreeNode, logManager: LogManager, types?: PackageType[]): EosScanRequest[] { - let requests: EosScanRequest[] = [{ roots: [root.workSpace.uri.fsPath] } as EosScanRequest]; - - if (types && types.length > 0) { - // Check if the package type is supported for scan. - if (types.find(type => !!Translators.toLanguageType(type))) { - return requests; - } - } else { - // In case there are no descriptors to extract language from, add all. - return requests; - } - logManager.logMessage('Skip Eos scan for not supported package type:' + types.map(type => fromPackageType(type)).join(' '), 'INFO'); - return []; - } - - /** - * Run Eos scan async task + * Run SAST scan async task * @param workspaceData - the issues data for the workspace * @param root - the root node of the workspace * @param types - the packages types that were detected in the workspace * @param scanManager - the scan manager to use for scan * @param progressManager - the progress manager of the process for abort control */ - public static async runEos( + public static async runSast( workspaceData: ScanResults, root: IssuesRootTreeNode, types: PackageType[], scanManager: ScanManager, + module: Module, progressManager: StepProgress ): Promise { let startTime: number = Date.now(); - workspaceData.eosScan = await scanManager.scanEos( - progressManager.checkCancel, - root.workSpace.uri.fsPath, - ...this.createEosRequests(root, scanManager.logManager, types) - ); - if (workspaceData.eosScan) { - workspaceData.eosScanTimestamp = Date.now(); - let applicableIssuesCount: number = AnalyzerUtils.populateEosIssues(root, workspaceData); + workspaceData.sastScan = await scanManager.scanSast(module, progressManager.checkCancel); + if (workspaceData.sastScan) { + workspaceData.sastScanTimestamp = Date.now(); + let sastIssuesCount: number = AnalyzerUtils.populateSastIssues(root, workspaceData); scanManager.logManager.logMessage( 'Found ' + - applicableIssuesCount + - " Eos issues in workspace = '" + + sastIssuesCount + + " SAST issues in workspace = '" + workspaceData.path + "' (elapsed " + - (workspaceData.eosScanTimestamp - startTime) / 1000 + + (workspaceData.sastScanTimestamp - startTime) / 1000 + ' seconds)', 'INFO' ); @@ -650,20 +627,20 @@ export class AnalyzerUtils { } /** - * Populate eos information in the view + * Populate SAST information in the view * @param root - root node to populate data inside * @param workspaceData - data to populate on node - * @returns number of eos issues populated + * @returns number of SAST issues populated */ - public static populateEosIssues(root: IssuesRootTreeNode, workspaceData: ScanResults): number { - root.eosScanTimeStamp = workspaceData.eosScanTimestamp; + public static populateSastIssues(root: IssuesRootTreeNode, workspaceData: ScanResults): number { + root.sastScanTimeStamp = workspaceData.sastScanTimestamp; let issuesCount: number = 0; - if (workspaceData.eosScan && workspaceData.eosScan.filesWithIssues) { - workspaceData.eosScan.filesWithIssues.forEach(fileWithIssues => { + if (workspaceData.sastScan && workspaceData.sastScan.filesWithIssues) { + workspaceData.sastScan.filesWithIssues.forEach(fileWithIssues => { let fileNode: CodeFileTreeNode = this.getOrCreateCodeFileNode(root, fileWithIssues.full_path); fileWithIssues.issues.forEach((issue: EosIssue) => { issue.locations.forEach((location: EosIssueLocation) => { - fileNode.issues.push(new EosTreeNode(issue, location, fileNode)); + fileNode.issues.push(new SastTreeNode(issue, location, fileNode)); issuesCount++; }); }); diff --git a/src/main/treeDataProviders/utils/dependencyUtils.ts b/src/main/treeDataProviders/utils/dependencyUtils.ts index 6b9364672..ffd8c9c9c 100644 --- a/src/main/treeDataProviders/utils/dependencyUtils.ts +++ b/src/main/treeDataProviders/utils/dependencyUtils.ts @@ -57,7 +57,7 @@ export class DependencyUtils { let descriptorsParsed: Set = new Set(); // Build dependency tree for all the package descriptors let packageDependenciesTree: DependenciesTreeNode = await DependencyUtils.createDependenciesTree( - root.workSpace, + root.workspace, type, descriptorsPaths, () => progressManager.onProgress, @@ -295,7 +295,7 @@ export class DependencyUtils { if (fileScanBundle) { logger.logMessage( "Workspace '" + - fileScanBundle.rootNode.workSpace.name + + fileScanBundle.rootNode.workspace.name + "' scan on file '" + fileScanBundle.data.fullPath + "' ended with error:\n" + diff --git a/src/main/types/jfrogAppsConfig.ts b/src/main/types/jfrogAppsConfig.ts new file mode 100644 index 000000000..039efdad6 --- /dev/null +++ b/src/main/types/jfrogAppsConfig.ts @@ -0,0 +1,35 @@ +export interface JFrogAppsConfig { + version: string; + modules: Module[]; +} + +export enum ExcludeScanner { + ContextualAnalysis = 'applicability', + Iac = 'iac', + Sast = 'sast', + Secrets = 'secrets' +} + +export interface Module { + name: string; + source_root: string; + exclude_patterns: string[]; + exclude_scanners: ExcludeScanner[]; + scanners: Scanners; +} + +export interface Scanners { + secrets: Scanner; + iac: Scanner; + sast: SastScanner; +} + +export interface Scanner { + working_dirs: string[]; + exclude_patterns: string[]; +} + +export interface SastScanner extends Scanner { + language: string; + excluded_rules: string[]; +} diff --git a/src/main/types/workspaceIssuesDetails.ts b/src/main/types/workspaceIssuesDetails.ts index 8f445a801..be68972bd 100644 --- a/src/main/types/workspaceIssuesDetails.ts +++ b/src/main/types/workspaceIssuesDetails.ts @@ -1,7 +1,7 @@ import { IGraphResponse } from 'jfrog-client-js'; import { IImpactGraph } from 'jfrog-ide-webview'; import { ApplicabilityScanResponse } from '../scanLogic/scanRunners/applicabilityScan'; -import { EosScanResponse } from '../scanLogic/scanRunners/eosScan'; +import { EosScanResponse } from '../scanLogic/scanRunners/sastScan'; import { Utils } from '../utils/utils'; import { PackageType } from './projectType'; import { IacScanResponse } from '../scanLogic/scanRunners/iacScan'; @@ -12,8 +12,8 @@ import { SecretsScanResponse } from '../scanLogic/scanRunners/secretsScan'; */ export class ScanResults { private _descriptorsIssues: DependencyScanResults[] = []; - private _eosScan: EosScanResponse = {} as EosScanResponse; - private _eosScanTimestamp?: number; + private _sastScan: EosScanResponse = {} as EosScanResponse; + private _sastScanTimestamp?: number; private _iacScan: IacScanResponse = {} as IacScanResponse; private _iacScanTimestamp?: number; private _secretsScan: SecretsScanResponse = {} as SecretsScanResponse; @@ -30,9 +30,9 @@ export class ScanResults { if (jsonScanResults._descriptorsIssues) { workspaceIssuesDetails.descriptorsIssues.push(...jsonScanResults._descriptorsIssues); } - // Eos - workspaceIssuesDetails.eosScan = jsonScanResults._eosScan; - workspaceIssuesDetails.eosScanTimestamp = jsonScanResults._eosScanTimestamp; + // SAST + workspaceIssuesDetails.sastScan = jsonScanResults._sastScan; + workspaceIssuesDetails.sastScanTimestamp = jsonScanResults._sastScanTimestamp; // Iac workspaceIssuesDetails.iacScan = jsonScanResults._iacScan; workspaceIssuesDetails.iacScanTimestamp = jsonScanResults._iacScanTimestamp; @@ -54,7 +54,7 @@ export class ScanResults { ...this._descriptorsIssues.map(descriptorIssues => descriptorIssues.applicableScanTimestamp), this.iacScanTimestamp, this.secretsScanTimestamp, - this.eosScanTimestamp + this.sastScanTimestamp ); } @@ -65,7 +65,7 @@ export class ScanResults { public hasIssues(): boolean { return ( this.descriptorsIssues.length > 0 || - this.eosScan?.filesWithIssues?.length > 0 || + this.sastScan?.filesWithIssues?.length > 0 || this.iacScan?.filesWithIssues?.length > 0 || this.secretsScan?.filesWithIssues?.length > 0 ); @@ -83,20 +83,20 @@ export class ScanResults { this._descriptorsIssues = value; } - get eosScan(): EosScanResponse { - return this._eosScan; + get sastScan(): EosScanResponse { + return this._sastScan; } - set eosScan(value: EosScanResponse) { - this._eosScan = value; + set sastScan(value: EosScanResponse) { + this._sastScan = value; } - get eosScanTimestamp(): number | undefined { - return this._eosScanTimestamp; + get sastScanTimestamp(): number | undefined { + return this._sastScanTimestamp; } - set eosScanTimestamp(value: number | undefined) { - this._eosScanTimestamp = value; + set sastScanTimestamp(value: number | undefined) { + this._sastScanTimestamp = value; } get iacScan(): IacScanResponse { diff --git a/src/main/utils/appConfigUtils.ts b/src/main/utils/appConfigUtils.ts new file mode 100644 index 000000000..7873c83ad --- /dev/null +++ b/src/main/utils/appConfigUtils.ts @@ -0,0 +1,53 @@ +import * as fs from 'fs'; +import yaml from 'js-yaml'; +import * as path from 'path'; +import { AnalyzerUtils } from '../treeDataProviders/utils/analyzerUtils'; +import { ExcludeScanner, JFrogAppsConfig, Module, Scanner } from '../types/jfrogAppsConfig'; +import { Configuration } from './configuration'; + +export class AppsConfigUtils { + public static JFROG_APP_CONFIG_VERSION: string = '1.0'; + + public static LoadConfig(workspace: string): JFrogAppsConfig { + let appConfigPath: string = path.join(workspace, '.jfrog', 'jfrog-apps-config.yml'); + if (!fs.existsSync(appConfigPath)) { + return { + version: AppsConfigUtils.JFROG_APP_CONFIG_VERSION, + modules: [{ source_root: workspace } as Module] + } as JFrogAppsConfig; + } + return yaml.load(fs.readFileSync(appConfigPath, 'utf8')) as JFrogAppsConfig; + } + + public static ShouldSkipScanner(module: Module, scanType: ExcludeScanner): boolean { + return !!module.exclude_scanners?.includes(scanType); + } + + public static GetSourceRoots(module: Module, scanner: Scanner | undefined): string[] { + let root: string; + if (path.isAbsolute(module.source_root)) { + root = module.source_root; + } else { + root = path.join(__dirname, module.source_root); + } + if (!scanner || !scanner.working_dirs) { + return [root]; + } + let roots: string[] = []; + for (let workingDir of scanner.working_dirs) { + roots.push(path.join(root, workingDir)); + } + return roots; + } + + public static GetExcludePatterns(module: Module, scanner: Scanner | undefined): string[] { + let excludePatterns: string[] = module.exclude_patterns || []; + if (scanner && scanner.exclude_patterns) { + excludePatterns = excludePatterns.concat(scanner.exclude_patterns); + } + if (excludePatterns.length === 0) { + return AnalyzerUtils.getAnalyzerManagerExcludePattern(Configuration.getScanExcludePattern()); + } + return excludePatterns; + } +} diff --git a/src/main/utils/translators.ts b/src/main/utils/translators.ts index 8cf9532bc..806c1af7c 100644 --- a/src/main/utils/translators.ts +++ b/src/main/utils/translators.ts @@ -1,27 +1,31 @@ import { + Severity as ClientSeverity, ICve, + IExtendedInformation, IGeneral, + IGraphCve, IIssue, ILicense, - IVulnerableComponent, - Severity as ClientSeverity, IReference, - IExtendedInformation, - IGraphCve + IVulnerableComponent } from 'jfrog-client-js'; -import { IExtendedInformation as WebExtendedInformation, ISeverityReasons, ICve as WebICve, IAnalysisStep } from 'jfrog-ide-webview'; +import { + IAnalysisStep, + IApplicableDetails, + ISeverityReasons, + IExtendedInformation as WebExtendedInformation, + ICve as WebICve +} from 'jfrog-ide-webview'; import Set from 'typescript-collections/dist/lib/Set'; -import { IApplicableDetails } from 'jfrog-ide-webview'; +import { LogLevel } from '../log/logManager'; +import { AnalyzerManagerSeverityLevel, FileLocation, ScanType } from '../scanLogic/scanRunners/analyzerModels'; import { GavGeneralInfo } from '../types/gavGeneralinfo'; import { GeneralInfo } from '../types/generalInfo'; import { IIssueCacheObject } from '../types/issueCacheObject'; import { ILicenseCacheObject } from '../types/licenseCacheObject'; +import { toPackageType } from '../types/projectType'; import { Severity } from '../types/severity'; -import { FileLocation, AnalyzerManagerSeverityLevel, ScanType } from '../scanLogic/scanRunners/analyzerModels'; -import { PackageType, toPackageType } from '../types/projectType'; import { Utils } from './utils'; -import { LogLevel } from '../log/logManager'; -import { LanguageType } from '../scanLogic/scanRunners/eosScan'; export class Translators { public static toAnalyzerTypeString(type: ScanType): string { @@ -32,8 +36,8 @@ export class Translators { return 'iac-scan'; case ScanType.Secrets: return 'secrets-detection'; - case ScanType.Eos: - return 'Eos'; + case ScanType.Sast: + return 'sast'; default: return type; } @@ -46,20 +50,6 @@ export class Translators { return logLevel.toUpperCase(); } - public static toLanguageType(type: PackageType): LanguageType | undefined { - switch (type) { - case PackageType.Python: - return 'python'; - case PackageType.Npm: - case PackageType.Yarn: - return 'javascript'; - case PackageType.Maven: - return 'java'; - default: - return undefined; - } - } - public static levelToSeverity(level?: AnalyzerManagerSeverityLevel): Severity { switch (level) { case 'none': diff --git a/src/test/resources/applicableScan/analyzerResponse.json b/src/test/resources/applicableScan/analyzerResponse.json index 0bc5ad4c6..8acf7653f 100644 --- a/src/test/resources/applicableScan/analyzerResponse.json +++ b/src/test/resources/applicableScan/analyzerResponse.json @@ -19,8 +19,8 @@ { "id": "applic_CVE-2021-3918", "fullDescription": { - "text": "The scanner checks whether any of the following vulnerable functions are called:\n\n* `json-schema.validate` with external input to its 1st (`instance`) argument.\n* `json-schema.checkPropertyChange` with external input to its 2nd (`schema`) argument.", - "markdown": "The scanner checks whether any of the following vulnerable functions are called:\n\n* `json-schema.validate` with external input to its 1st (`instance`) argument.\n* `json-schema.checkPropertyChange` with external input to its 2nd (`schema`) argument." + "text": "The scanner checks whether any of the following vulnerable functions are called:\n\n* `util.setProperty` with external input to its 2nd (`path`) or 3rd (`value`) arguments.\n* `ReflectionObject.setParsedOption` with external input to its 2nd (`name`) or 3rd (`value`) arguments.\n* `parse` with external input to its 1st (`source`) argument.\n* `load`\n* `loadSync`\n\nThe scanner also checks whether the `Object.freeze()` remediation is not present.", + "markdown": "The scanner checks whether any of the following vulnerable functions are called:\n\n* `util.setProperty` with external input to its 2nd (`path`) or 3rd (`value`) arguments.\n* `ReflectionObject.setParsedOption` with external input to its 2nd (`name`) or 3rd (`value`) arguments.\n* `parse` with external input to its 1st (`source`) argument.\n* `load`\n* `loadSync`\n\nThe scanner also checks whether the `Object.freeze()` remediation is not present." }, "name": "CVE-2021-3918", "shortDescription": { @@ -30,8 +30,8 @@ { "id": "applic_CVE-2022-25878", "fullDescription": { - "text": "The scanner checks whether any of the following vulnerable functions are called:\n\n* `util.setProperty` with external input to its 2nd (`path`) or 3rd (`value`) arguments.\n* `ReflectionObject.setParsedOption` with external input to its 2nd (`name`) or 3rd (`value`) arguments.\n* `parse` with external input to its 1st (`source`) argument.\n* `load`", - "markdown": "The scanner checks whether any of the following vulnerable functions are called:\n\n* `util.setProperty` with external input to its 2nd (`path`) or 3rd (`value`) arguments.\n* `ReflectionObject.setParsedOption` with external input to its 2nd (`name`) or 3rd (`value`) arguments.\n* `parse` with external input to its 1st (`source`) argument.\n* `load`" + "text": "The scanner checks whether any of the following vulnerable functions are called:\n\n* `util.setProperty` with external input to its 2nd (`path`) or 3rd (`value`) arguments.\n* `ReflectionObject.setParsedOption` with external input to its 2nd (`name`) or 3rd (`value`) arguments.\n* `parse` with external input to its 1st (`source`) argument.\n* `load`\n* `loadSync`\n\nThe scanner also checks whether the `Object.freeze()` remediation is not present.", + "markdown": "The scanner checks whether any of the following vulnerable functions are called:\n\n* `util.setProperty` with external input to its 2nd (`path`) or 3rd (`value`) arguments.\n* `ReflectionObject.setParsedOption` with external input to its 2nd (`name`) or 3rd (`value`) arguments.\n* `parse` with external input to its 1st (`source`) argument.\n* `load`\n* `loadSync`\n\nThe scanner also checks whether the `Object.freeze()` remediation is not present." }, "name": "CVE-2022-25878", "shortDescription": { @@ -58,7 +58,7 @@ "results": [ { "message": { - "text": "Prototype pollution `Object.freeze` remediation was not detected, The vulnerable function protobufjs.parse is called with external input, The vulnerable function protobufjs.load is called" + "text": "Prototype pollution `Object.freeze` remediation was not detected, The vulnerable function protobufjs.parse is called with external input, The vulnerable function protobufjs.load(Sync) is called" }, "locations": [ { @@ -82,7 +82,7 @@ }, { "message": { - "text": "Prototype pollution `Object.freeze` remediation was not detected, The vulnerable function protobufjs.parse is called with external input, The vulnerable function protobufjs.load is called" + "text": "Prototype pollution `Object.freeze` remediation was not detected, The vulnerable function protobufjs.parse is called with external input, The vulnerable function protobufjs.load(Sync) is called" }, "locations": [ { diff --git a/src/test/resources/applicableScan/npm/expectedScanResponse.json b/src/test/resources/applicableScan/npm/expectedScanResponse.json index 13f37e60d..2690d80ba 100644 --- a/src/test/resources/applicableScan/npm/expectedScanResponse.json +++ b/src/test/resources/applicableScan/npm/expectedScanResponse.json @@ -6,7 +6,7 @@ ], "applicableCve": { "CVE-2022-25878": { - "fixReason": "Prototype pollution `Object.freeze` remediation was not detected, The vulnerable function protobufjs.parse is called with external input, The vulnerable function protobufjs.load is called", + "fixReason": "Prototype pollution `Object.freeze` remediation was not detected, The vulnerable function protobufjs.parse is called with external input, The vulnerable function protobufjs.load(Sync) is called", "fileEvidences": [ { "full_path": "index.js", @@ -32,7 +32,7 @@ ] } ], - "fullDescription": "The scanner checks whether any of the following vulnerable functions are called:\n\n* `util.setProperty` with external input to its 2nd (`path`) or 3rd (`value`) arguments.\n* `ReflectionObject.setParsedOption` with external input to its 2nd (`name`) or 3rd (`value`) arguments.\n* `parse` with external input to its 1st (`source`) argument.\n* `load`" + "fullDescription": "The scanner checks whether any of the following vulnerable functions are called:\n\n* `util.setProperty` with external input to its 2nd (`path`) or 3rd (`value`) arguments.\n* `ReflectionObject.setParsedOption` with external input to its 2nd (`name`) or 3rd (`value`) arguments.\n* `parse` with external input to its 1st (`source`) argument.\n* `load`\n* `loadSync`\n\nThe scanner also checks whether the `Object.freeze()` remediation is not present." } } } \ No newline at end of file diff --git a/src/test/resources/jfrogAppsConfig/.jfrog/jfrog-apps-config.yml b/src/test/resources/jfrogAppsConfig/.jfrog/jfrog-apps-config.yml new file mode 100644 index 000000000..cf4de853d --- /dev/null +++ b/src/test/resources/jfrogAppsConfig/.jfrog/jfrog-apps-config.yml @@ -0,0 +1,50 @@ +# [Required] JFrog Applications Config version +version: "1.0" + +modules: + # [Required] Module name + - name: FrogLeapApp + # [Optional, default: "."] Application's root directory + source_root: "src" + # [Optional] Directories to exclude from scanning across all scanners + exclude_patterns: + - "docs/" + # [Optional] Scanners to exclude from JFrog Advanced Security (Options: "secrets", "sast", "iac") + exclude_scanners: + - secrets + # [Optional] Customize scanner configurations + scanners: + # [Optional] Configuration for Static Application Security Testing (SAST) + sast: + # [Optional] Specify the programming language for SAST + language: java + # [Optional] Working directories specific to SAST (Relative to source_root) + working_dirs: + - "dir1" + - "dir2" + # [Optional] Additional exclude patterns for this scanner + exclude_patterns: + - "dir1/test/**" + # [Optional] List of specific scan rules to exclude from the scan + excluded_rules: + - xss-injection + + # [Optional] Configuration for secrets scan + secrets: + # [Optional] Working directories specific to the secret scanner (Relative to source_root) + working_dirs: + - "dir1" + - "dir2" + # [Optional] Additional exclude patterns for this scanner + exclude_patterns: + - "dir1/test/**" + + # [Optional] Configuration for Infrastructure as Code scan (IaC) + iac: + # [Optional] Working directories specific to IaC (Relative to source_root) + working_dirs: + - "dir1" + - "dir2" + # [Optional] Additional exclude patterns for this Scanner + exclude_patterns: + - "dir1/test/**" \ No newline at end of file diff --git a/src/test/tests/appsConfig.test.ts b/src/test/tests/appsConfig.test.ts new file mode 100644 index 000000000..9450fcb4a --- /dev/null +++ b/src/test/tests/appsConfig.test.ts @@ -0,0 +1,120 @@ +import { assert } from 'chai'; +import * as path from 'path'; +import { ExcludeScanner, JFrogAppsConfig, Module, SastScanner, Scanner } from '../../main/types/jfrogAppsConfig'; +import { AppsConfigUtils } from '../../main/utils/appConfigUtils'; + +describe('JFrog Apps Config Tests', () => { + const jfrogAppsConfigDir: string = path.join(__dirname, '..', 'resources', 'jfrogAppsConfig'); + + it('Load full config', () => { + // Load config + let appsConfig: JFrogAppsConfig | undefined = AppsConfigUtils.LoadConfig(jfrogAppsConfigDir); + assert.isDefined(appsConfig); + assert.equal(appsConfig?.version, '1.0'); + assert.lengthOf(appsConfig!.modules, 1); + + // Check module + let module: Module = appsConfig!.modules[0]; + assert.equal(module.name, 'FrogLeapApp'); + assert.equal(module.source_root, 'src'); + assert.deepEqual(module.exclude_patterns, ['docs/']); + assert.deepEqual(module.exclude_scanners, ['secrets']); + + // Check SAST scanner + let sast: SastScanner = module.scanners.sast; + assert.equal(sast.language, 'java'); + assert.deepEqual(sast.working_dirs, ['dir1', 'dir2']); + assert.deepEqual(sast.exclude_patterns, ['dir1/test/**']); + assert.deepEqual(sast.excluded_rules, ['xss-injection']); + + // Check SAST scanner + let secrets: Scanner = module.scanners.secrets; + assert.deepEqual(secrets.working_dirs, ['dir1', 'dir2']); + assert.deepEqual(secrets.exclude_patterns, ['dir1/test/**']); + + // Check IaC scanner + let iac: Scanner = module.scanners.iac; + assert.deepEqual(iac.working_dirs, ['dir1', 'dir2']); + assert.deepEqual(iac.exclude_patterns, ['dir1/test/**']); + }); + + it('Should skip scanner', () => { + [ + { excludeScanners: undefined, shouldSkip: false }, + { excludeScanners: [] as ExcludeScanner[], shouldSkip: false }, + { excludeScanners: [ExcludeScanner.Iac] as ExcludeScanner[], shouldSkip: false }, + { excludeScanners: [ExcludeScanner.Iac, ExcludeScanner.Sast] as ExcludeScanner[], shouldSkip: false }, + { excludeScanners: [ExcludeScanner.ContextualAnalysis] as ExcludeScanner[], shouldSkip: true }, + { excludeScanners: [ExcludeScanner.Secrets, ExcludeScanner.ContextualAnalysis] as ExcludeScanner[], shouldSkip: true } + ].forEach(testCase => { + assert.equal( + AppsConfigUtils.ShouldSkipScanner({ exclude_scanners: testCase.excludeScanners } as Module, ExcludeScanner.ContextualAnalysis), + testCase.shouldSkip + ); + }); + }); + + let getSourceRootCases: AppConfigTest[] = [ + { scanner: undefined }, + { scanner: { working_dirs: ['working-dir'] } as Scanner }, + { scanner: { working_dirs: ['working-dir-1', 'working-dir-2'] } as Scanner } + ]; + + it('Get source roots - With module source', () => { + let sourceRoot: string = path.join(__dirname, 'source-root'); + let module: Module = { source_root: sourceRoot } as Module; + getSourceRootCases.forEach(testCase => { + let actualSourceRoots: string[] = AppsConfigUtils.GetSourceRoots(module, testCase.scanner); + if (!testCase.scanner) { + assert.sameMembers(actualSourceRoots, [module.source_root]); + return; + } + let expectedWorkingDirs: string[] = []; + for (let workingDir of testCase.scanner.working_dirs) { + expectedWorkingDirs.push(path.join(module.source_root, workingDir)); + } + assert.sameMembers(actualSourceRoots, expectedWorkingDirs); + }); + }); + + it('Get source roots - Without module source', () => { + let sourceRoot: string = path.join(__dirname); + let module: Module = { source_root: sourceRoot } as Module; + getSourceRootCases.forEach(testCase => { + let actualSourceRoots: string[] = AppsConfigUtils.GetSourceRoots(module, testCase.scanner); + if (!testCase.scanner) { + assert.sameMembers(actualSourceRoots, [module.source_root]); + return; + } + let expectedWorkingDirs: string[] = []; + for (let workingDir of testCase.scanner.working_dirs) { + expectedWorkingDirs.push(path.join(module.source_root, workingDir)); + } + assert.sameMembers(actualSourceRoots, expectedWorkingDirs); + }); + }); + + let getExcludePatternsCases: AppConfigTest[] = [ + { scanner: undefined }, + { scanner: { exclude_patterns: ['exclude-dir'] } as Scanner }, + { scanner: { exclude_patterns: ['exclude-dir-1', 'exclude-dir-2'] } as Scanner } + ]; + + it('Get exclude patterns', () => { + let module: Module = { exclude_patterns: ['exclude-root'] } as Module; + getExcludePatternsCases.forEach(testCase => { + let actualExcludePatterns: string[] = AppsConfigUtils.GetExcludePatterns(module, testCase.scanner); + if (!testCase.scanner) { + assert.sameMembers(actualExcludePatterns, module.exclude_patterns); + return; + } + let expectedExcludePatterns: string[] = module.exclude_patterns; + expectedExcludePatterns = expectedExcludePatterns.concat(testCase.scanner.exclude_patterns); + assert.sameMembers(actualExcludePatterns, expectedExcludePatterns); + }); + }); + + interface AppConfigTest { + scanner: Scanner | undefined; + } +}); diff --git a/src/test/tests/integration/iac.test.ts b/src/test/tests/integration/iac.test.ts index ca3860e1a..f476b48dd 100644 --- a/src/test/tests/integration/iac.test.ts +++ b/src/test/tests/integration/iac.test.ts @@ -1,8 +1,9 @@ -import * as path from 'path'; -import * as fs from 'fs'; import { assert } from 'chai'; +import * as fs from 'fs'; +import * as path from 'path'; import { IacRunner, IacScanResponse } from '../../../main/scanLogic/scanRunners/iacScan'; +import { Module } from '../../../main/types/jfrogAppsConfig'; import { AnalyzerManagerIntegrationEnv, assertFileIssuesExist, @@ -13,7 +14,6 @@ import { assertIssuesRuleNameExist, assertIssuesSeverityExist } from '../utils/testIntegration.test'; -import { NotSupportedError } from '../../../main/utils/scanUtils'; describe('Iac Integration Tests', async () => { const integrationManager: AnalyzerManagerIntegrationEnv = new AnalyzerManagerIntegrationEnv(); @@ -35,14 +35,7 @@ describe('Iac Integration Tests', async () => { assert.isDefined(expectedContent, 'Failed to read expected IacScanResponse content from ' + dataPath); // Run scan // Try/Catch (with skip) should be removed after Iac is released - try { - response = await runner.scan(testDataRoot, () => undefined); - } catch (err) { - if (err instanceof NotSupportedError) { - this.skip(); - } - throw err; - } + response = await runner.scan({ source_root: testDataRoot } as Module, () => undefined); }); it('Check response defined', () => { diff --git a/src/test/tests/integration/secrets.test.ts b/src/test/tests/integration/secrets.test.ts index be76c46cf..538d2289b 100644 --- a/src/test/tests/integration/secrets.test.ts +++ b/src/test/tests/integration/secrets.test.ts @@ -14,6 +14,7 @@ import { } from '../utils/testIntegration.test'; import { NotSupportedError } from '../../../main/utils/scanUtils'; import { SecretsRunner, SecretsScanResponse } from '../../../main/scanLogic/scanRunners/secretsScan'; +import { Module } from '../../../main/types/jfrogAppsConfig'; describe('Secrets Scan Integration Tests', async () => { const integrationManager: AnalyzerManagerIntegrationEnv = new AnalyzerManagerIntegrationEnv(); @@ -36,7 +37,7 @@ describe('Secrets Scan Integration Tests', async () => { // Run scan // Try/Catch (with skip) should be removed after Secrets scan is released try { - response = await runner.scan(testDataRoot, () => undefined); + response = await runner.scan({ source_root: testDataRoot } as Module, () => undefined); } catch (err) { if (err instanceof NotSupportedError) { this.skip(); diff --git a/src/test/tests/issuesTreeNode.test.ts b/src/test/tests/issuesTreeNode.test.ts index 66a0b5c6c..1db4046be 100644 --- a/src/test/tests/issuesTreeNode.test.ts +++ b/src/test/tests/issuesTreeNode.test.ts @@ -108,7 +108,7 @@ describe('Issues Root Node Tests', () => { it('Tooltip test - ' + testCase.test, () => { let testNode: IssuesRootTreeNode = createAndPopulateRootTestNode(testCase.path, testCase.data); // Path - assert.equal(testNode.workSpace.uri.fsPath, testCase.path); + assert.equal(testNode.workspace.uri.fsPath, testCase.path); assert.include(testNode.tooltip, 'Full path: ' + testCase.path); // Issue count assert.include(testNode.tooltip, 'Issue count: ' + testCase.expectedIssueCount);