Skip to content

Commit

Permalink
Support JFrog Apps Config file
Browse files Browse the repository at this point in the history
  • Loading branch information
@yahavi
yahavi committed Sep 26, 2023
1 parent f3d1c78 commit 464027b
Show file tree
Hide file tree
Showing 23 changed files with 563 additions and 295 deletions.
78 changes: 38 additions & 40 deletions src/main/scanLogic/scanManager.ts
View file
Original file line number Diff line number Diff line change
Expand Up @@ -2,29 +2,31 @@ import * as vscode from 'vscode';

import { ExtensionComponent } from '../extensionComponent';

import { LogManager } from '../log/logManager';
import { ConnectionManager } from '../connect/connectionManager';
import { ConnectionUtils, EntitlementScanFeature } from '../connect/connectionUtils';
import { LogManager } from '../log/logManager';

import { RootNode } from '../treeDataProviders/dependenciesTree/dependenciesRoot/rootTree';
import { IGraphResponse, XrayScanProgress } from 'jfrog-client-js';
import { GraphScanLogic } from './scanGraphLogic';
import { ApplicabilityRunner, ApplicabilityScanResponse } from './scanRunners/applicabilityScan';
import { EosRunner, EosScanRequest, EosScanResponse } from './scanRunners/eosScan';
import { RootNode } from '../treeDataProviders/dependenciesTree/dependenciesRoot/rootTree';
import { AnalyzerUtils } from '../treeDataProviders/utils/analyzerUtils';
import { StepProgress } from '../treeDataProviders/utils/stepProgress';
import { ExcludeScanner, Module } from '../types/jfrogAppsConfig';
import { AppsConfigUtils } from '../utils/appConfigUtils';
import { Configuration } from '../utils/configuration';
import { Resource } from '../utils/resource';
import { BinaryRunner } from './scanRunners/binaryRunner';
import { ScanUtils } from '../utils/scanUtils';
import { StepProgress } from '../treeDataProviders/utils/stepProgress';
import { Utils } from '../utils/utils';
import { GraphScanLogic } from './scanGraphLogic';
import { ApplicabilityRunner, ApplicabilityScanResponse } from './scanRunners/applicabilityScan';
import { BinaryRunner } from './scanRunners/binaryRunner';
import { IacRunner, IacScanResponse } from './scanRunners/iacScan';
import { EosScanResponse, SastRunner } from './scanRunners/sastScan';
import { SecretsRunner, SecretsScanResponse } from './scanRunners/secretsScan';

export interface SupportedScans {
dependencies: boolean;
applicability: boolean;
eos: boolean;
sast: boolean;
iac: boolean;
secrets: boolean;
}
Expand Down Expand Up @@ -167,9 +169,9 @@ export class ScanManager implements ExtensionComponent {
}

/**
* Check if Eos scan is supported for the user
* Check if SAST scan is supported for the user
*/
public async isEosSupported(): Promise<boolean> {
public async isSastSupported(): Promise<boolean> {
return true;
}

Expand Down Expand Up @@ -200,8 +202,8 @@ export class ScanManager implements ExtensionComponent {
.catch(err => ScanUtils.onScanError(err, this._logManager, true))
);
requests.push(
this.isEosSupported()
.then(res => (supportedScans.eos = res))
this.isSastSupported()
.then(res => (supportedScans.sast = res))
.catch(err => ScanUtils.onScanError(err, this._logManager, true))
);
await Promise.all(requests);
Expand Down Expand Up @@ -250,61 +252,57 @@ export class ScanManager implements ExtensionComponent {

/**
* Scan directory for 'Infrastructure As Code' (Iac) issues.
* @param directory - the directory that will be scan
* @param module - the module that will be scanned
* @param checkCancel - check if should cancel
* @returns the Iac scan response
*/
public async scanIac(directory: string, checkCancel: () => void): Promise<IacScanResponse> {
public async scanIac(module: Module, checkCancel: () => void): Promise<IacScanResponse> {
let iacRunner: IacRunner = new IacRunner(this._connectionManager, this.logManager);
if (!iacRunner.validateSupported()) {
this._logManager.logMessage('Iac runner could not find binary to run', 'WARN');
return {} as IacScanResponse;
}
let skipFiles: string[] = AnalyzerUtils.getAnalyzerManagerExcludePattern(Configuration.getScanExcludePattern());
this._logManager.logMessage("Scanning directory '" + directory + "', for Iac issues. Skipping files: " + skipFiles, 'DEBUG');
return await iacRunner.scan(directory, checkCancel, skipFiles);
if (AppsConfigUtils.ShouldSkipScanner(module, ExcludeScanner.Iac)) {
this._logManager.debug('Skipping IaC scanning');
return {} as IacScanResponse;
}
return await iacRunner.scan(module, checkCancel);
}
/**
* Scan directory for secrets issues.
* @param directory - the directory that will be scan
* @param module - the module that will be scanned
* @param checkCancel - check if should cancel
* @returns the Secrets scan response
*/
public async scanSecrets(directory: string, checkCancel: () => void): Promise<SecretsScanResponse> {
public async scanSecrets(module: Module, checkCancel: () => void): Promise<SecretsScanResponse> {
let secretsRunner: SecretsRunner = new SecretsRunner(this._connectionManager, this.logManager);
if (!secretsRunner.validateSupported()) {
this._logManager.logMessage('Secrets runner could not find binary to run', 'WARN');
return {} as SecretsScanResponse;
}
let skipFiles: string[] = AnalyzerUtils.getAnalyzerManagerExcludePattern(Configuration.getScanExcludePattern());
this._logManager.logMessage("Scanning directory '" + directory + "', for Secrets issues. Skipping files: " + skipFiles, 'DEBUG');
return await secretsRunner.scan(directory, checkCancel, skipFiles);
if (AppsConfigUtils.ShouldSkipScanner(module, ExcludeScanner.Secrets)) {
this._logManager.debug('Skipping secrets scanning');
return {} as SecretsScanResponse;
}
return await secretsRunner.scan(module, checkCancel);
}

/**
* Scan for Eos issues.
* @param checkCancel - check if should cancel
* @param requests - the Eos requests to run
* Scan for SAST issues.
* @param module - the module that will be scanned
* @param requests - the SAST requests to run
* @returns the scan response
*/
public async scanEos(checkCancel: () => void, runDirectory?: string, ...requests: EosScanRequest[]): Promise<EosScanResponse> {
let eosRunner: EosRunner = new EosRunner(this._connectionManager, this._logManager, undefined, undefined, runDirectory);
if (!eosRunner.validateSupported()) {
this._logManager.logMessage('Eos runner could not find binary to run', 'WARN');
public async scanSast(module: Module, checkCancel: () => void): Promise<EosScanResponse> {
let sastRunner: SastRunner = new SastRunner(this._connectionManager, this._logManager);
if (!sastRunner.validateSupported()) {
this._logManager.logMessage('Sast runner could not find binary to run', 'WARN');
return {} as EosScanResponse;
}
if (requests.length === 0) {
this._logManager.logMessage('Eos runner must receive at least one request to run', 'ERR');
if (AppsConfigUtils.ShouldSkipScanner(module, ExcludeScanner.Sast)) {
this._logManager.debug('Skipping SAST scanning');
return {} as EosScanResponse;
}
let skipFiles: string[] = AnalyzerUtils.getAnalyzerManagerExcludePattern(Configuration.getScanExcludePattern());
this._logManager.logMessage(
'Scanning for Eos issues in ' +
requests.map(request => `(Language '${request.language}', roots: [${request.roots.join()}])`) +
'. Skipping files: ' +
skipFiles,
'DEBUG'
);
return eosRunner.scan(checkCancel, skipFiles, ...requests);
return sastRunner.scan(module, checkCancel);
}
}
2 changes: 1 addition & 1 deletion src/main/scanLogic/scanRunners/analyzerModels.ts
View file
Original file line number Diff line number Diff line change
Expand Up @@ -5,7 +5,7 @@ export interface AnalyzerRequest {
export enum ScanType {
ContextualAnalysis = 'analyze-applicability',
Iac = 'iac-scan-modules',
Eos = 'analyze-codebase',
Sast = 'sast',
Secrets = 'secrets-scan'
}

Expand Down
89 changes: 48 additions & 41 deletions src/main/scanLogic/scanRunners/binaryRunner.ts
View file
Original file line number Diff line number Diff line change
@@ -1,19 +1,19 @@
import * as fs from 'fs';
import yaml from 'js-yaml';
import yaml, { DumpOptions } from 'js-yaml';
import * as path from 'path';

import { LogManager } from '../../log/logManager';
import { Utils } from '../../utils/utils';
import { NotEntitledError, NotSupportedError, OsNotSupportedError, ScanCancellationError, ScanUtils } from '../../utils/scanUtils';
import { AnalyzerRequest, AnalyzerScanResponse, ScanType, AnalyzeScanRequest } from './analyzerModels';
import { IProxyConfig } from 'jfrog-client-js';
import { ConnectionManager } from '../../connect/connectionManager';
import { ConnectionUtils } from '../../connect/connectionUtils';
import { IProxyConfig } from 'jfrog-client-js';
import { LogManager } from '../../log/logManager';
import { LogUtils } from '../../log/logUtils';
import { Configuration } from '../../utils/configuration';
import { Resource } from '../../utils/resource';
import { RunUtils } from '../../utils/runUtils';
import { NotEntitledError, NotSupportedError, OsNotSupportedError, ScanCancellationError, ScanUtils } from '../../utils/scanUtils';
import { Translators } from '../../utils/translators';
import { LogUtils } from '../../log/logUtils';
import { Utils } from '../../utils/utils';
import { AnalyzeScanRequest, AnalyzerRequest, AnalyzerScanResponse, ScanType } from './analyzerModels';

/**
* Arguments for running binary async
Expand Down Expand Up @@ -47,7 +47,7 @@ export abstract class BinaryRunner {
protected _verbose: boolean = false;

private static readonly RUNNER_NAME: string = 'analyzerManager';
public static readonly RUNNER_VERSION: string = '1.3.2.2005632';
public static readonly RUNNER_VERSION: string = '1.3.2.2019257';
private static readonly DOWNLOAD_URL: string = '/xsc-gen-exe-analyzer-manager-local/v1/';

public static readonly NOT_ENTITLED: number = 31;
Expand Down Expand Up @@ -106,9 +106,15 @@ export abstract class BinaryRunner {
* Run the executeBinary method with the provided request path
* @param checkCancel - check if cancel
* @param yamlConfigPath - the path to the request
* @param executionLogDirectory - og file will be written to the dir
* @param executionLogDirectory - log file will be written to the dir
* @param responsePath - path to the output file
*/
protected abstract runBinary(yamlConfigPath: string, executionLogDirectory: string | undefined, checkCancel: () => void): Promise<void>;
protected abstract runBinary(
yamlConfigPath: string,
executionLogDirectory: string | undefined,
checkCancel: () => void,
responsePath: string | undefined
): Promise<void>;

/**
* Validates that the binary exists and can run
Expand Down Expand Up @@ -137,11 +143,9 @@ export abstract class BinaryRunner {
* @param executionLogDirectory - the directory to save the execution log in
*/
private async executeBinaryTask(args: string[], executionLogDirectory?: string): Promise<any> {
let std: any = await ScanUtils.executeCmdAsync(
'"' + this._binary.fullPath + '" ' + args.join(' '),
this._runDirectory,
this.createEnvForRun(executionLogDirectory)
);
let command: string = '"' + this._binary.fullPath + '" ' + args.join(' ');
this._logManager.debug('Executing ' + command);
let std: any = await ScanUtils.executeCmdAsync(command, this._runDirectory, this.createEnvForRun(executionLogDirectory));
if (std.stdout && std.stdout.length > 0) {
this.logTaskResult(std.stdout, false);
}
Expand Down Expand Up @@ -230,11 +234,11 @@ export abstract class BinaryRunner {
return url;
}

public async run(checkCancel: () => void, ...requests: AnalyzeScanRequest[]): Promise<AnalyzerScanResponse | undefined> {
public async run(checkCancel: () => void, request: AnalyzeScanRequest): Promise<AnalyzerScanResponse | undefined> {
if (!this.validateSupported()) {
return undefined;
}
let args: RunArgs = this.createRunArguments(...requests);
let args: RunArgs = this.createRunArguments(request);
try {
if (args.requests.length == 0) {
return undefined;
Expand All @@ -250,27 +254,27 @@ export abstract class BinaryRunner {
* @param requests - the run requests information
* @return run arguments for the given requests
*/
private createRunArguments(...requests: AnalyzeScanRequest[]): RunArgs {
private createRunArguments(request: AnalyzeScanRequest): RunArgs {
let args: RunArgs = new RunArgs(ScanUtils.createTmpDir());
let processedRoots: Set<string> = new Set<string>();

for (const request of requests) {
if (request.roots.length > 0 && request.roots.every(root => !processedRoots.has(root))) {
// Prepare request information and insert as an actual request
const requestPath: string = path.join(args.directory, 'request_' + args.requests.length);
const responsePath: string = path.join(args.directory, 'response_' + args.requests.length);
if (request.roots.length > 0 && request.roots.every(root => !processedRoots.has(root))) {
// Prepare request information and insert as an actual request
const requestPath: string = path.join(args.directory, 'request_' + args.requests.length);
const responsePath: string = path.join(args.directory, 'response_' + args.requests.length);
if (request.type !== ScanType.Sast) {
request.output = responsePath;
request.type = this._type;
request.roots.forEach(root => processedRoots.add(root));
// Add request to run
args.requests.push({
type: request.type,
request: this.requestsToYaml(request),
requestPath: requestPath,
roots: request.roots,
responsePath: responsePath
} as RunRequest);
}
request.type = this._type;
request.roots.forEach(root => processedRoots.add(root));
// Add request to run
args.requests.push({
type: request.type,
request: this.requestsToYaml(request),
requestPath: requestPath,
roots: request.roots,
responsePath: responsePath
} as RunRequest);
}

return args;
Expand All @@ -283,9 +287,12 @@ export abstract class BinaryRunner {
*/
public requestsToYaml(...requests: AnalyzeScanRequest[]): string {
return yaml
.dump({
scans: requests
} as AnalyzerRequest)
.dump(
{
scans: requests
} as AnalyzerRequest,
{ lineWidth: 1000 } as DumpOptions
)
.replace('skipped_folders', 'skipped-folders');
}

Expand All @@ -297,7 +304,7 @@ export abstract class BinaryRunner {
this.runRequest(
checkCancel,
args.requests[i].request,
args.requests[i].type === ScanType.Eos ? args.requests[i].responsePath : args.requests[i].requestPath,
args.requests[i].requestPath,
args.requests[i].type,
args.requests[i].responsePath
)
Expand Down Expand Up @@ -390,11 +397,11 @@ export abstract class BinaryRunner {
responsePath: string
): Promise<AnalyzerScanResponse> {
// 1. Save requests as yaml file in folder
if (type !== ScanType.Eos) {
fs.writeFileSync(requestPath, request);
}
fs.writeFileSync(requestPath, request);
this._logManager.debug('Input YAML:\n' + request);

// 2. Run the binary
await this.runBinary(requestPath, this._verbose ? undefined : path.dirname(requestPath), checkCancel).catch(error => {
await this.runBinary(requestPath, this._verbose ? undefined : path.dirname(requestPath), checkCancel, responsePath).catch(error => {
if (error.code) {
// Not entitled to run binary
if (error.code === BinaryRunner.NOT_ENTITLED) {
Expand Down
12 changes: 9 additions & 3 deletions src/main/scanLogic/scanRunners/iacScan.ts
View file
Original file line number Diff line number Diff line change
@@ -1,6 +1,8 @@
import { ConnectionManager } from '../../connect/connectionManager';
import { LogManager } from '../../log/logManager';
import { AnalyzerUtils, FileWithSecurityIssues } from '../../treeDataProviders/utils/analyzerUtils';
import { Module } from '../../types/jfrogAppsConfig';
import { AppsConfigUtils } from '../../utils/appConfigUtils';
import { Resource } from '../../utils/resource';
import { ScanUtils } from '../../utils/scanUtils';
import { AnalyzeScanRequest, AnalyzerScanResponse, ScanType } from './analyzerModels';
Expand Down Expand Up @@ -28,12 +30,16 @@ export class IacRunner extends BinaryRunner {
await this.executeBinary(checkCancel, ['iac', yamlConfigPath], executionLogDirectory);
}

public async scan(directory: string, checkCancel: () => void, skipFolders: string[] = []): Promise<IacScanResponse> {
public async scan(module: Module, checkCancel: () => void): Promise<IacScanResponse> {
let request: AnalyzeScanRequest = {
type: ScanType.Iac,
roots: [directory],
skipped_folders: skipFolders
roots: AppsConfigUtils.GetSourceRoots(module, module.scanners?.iac),
skipped_folders: AppsConfigUtils.GetExcludePatterns(module, module.scanners?.iac)
} as AnalyzeScanRequest;
this._logManager.logMessage(
"Scanning directories '" + request.roots + "', for Iac issues. Skipping folders: " + request.skipped_folders,
'DEBUG'
);
return await this.run(checkCancel, request).then(runResult => this.convertResponse(runResult));
}

Expand Down
Loading

0 comments on commit 464027b

Please sign in to comment.