diff --git a/tst/e2e/Diagnostics.test.ts b/tst/e2e/Diagnostics.test.ts new file mode 100644 index 00000000..1c1d76d2 --- /dev/null +++ b/tst/e2e/Diagnostics.test.ts @@ -0,0 +1,208 @@ +import { join } from 'path'; +import { describe, it, expect, beforeAll, afterAll, beforeEach } from 'vitest'; +import { TestExtension } from '../utils/TestExtension'; +import { WaitFor } from '../utils/Utils'; + +describe('Diagnostic Features', () => { + const client = new TestExtension({ + initializeParams: { + initializationOptions: { + aws: { + clientInfo: { + extension: { + name: 'Test CloudFormation Language Server', + version: '1.0.0-test', + }, + clientId: 'test-client', + }, + }, + settings: { + diagnostics: { + cfnGuard: { + enabled: true, + rulesFile: join(__dirname, '../resources/guard/test-guard-rules.guard'), + delayMs: 100, + validateOnChange: true, + }, + }, + }, + }, + }, + }); + + beforeAll(async () => { + await client.ready(); + + // Configure guard with custom rules file + await client.changeConfiguration({ + settings: { + diagnostics: { + cfnGuard: { + enabled: true, + rulesFile: join(__dirname, '../resources/guard/test-guard-rules.guard'), + delayMs: 100, + validateOnChange: true, + }, + }, + }, + }); + }); + + beforeEach(async () => { + await client.reset(); + }); + + afterAll(async () => { + await client.close(); + }); + + describe('Guard diagnostics while authoring', () => { + it('should receive diagnostics during incremental typing', async () => { + // Start with basic template that should trigger our custom guard rules + const initialTemplate = `AWSTemplateFormatVersion: '2010-09-09' +Resources: + MyBucket: + Type: AWS::S3::Bucket`; + + const uri = await client.openYamlTemplate(initialTemplate); + + // Wait for diagnostics from our custom guard rules + await WaitFor.waitFor(() => { + if (client.receivedDiagnostics.length === 0) { + throw new Error('No diagnostics received yet'); + } + }, 5000); + + expect(client.receivedDiagnostics.length).toBeGreaterThan(0); + + const latestDiagnostics = client.receivedDiagnostics[client.receivedDiagnostics.length - 1]; + expect(latestDiagnostics.uri).toBe(uri); + expect(latestDiagnostics.diagnostics.length).toBeGreaterThan(0); + + // Verify we got our custom guard diagnostics + const guardDiagnostics = latestDiagnostics.diagnostics.filter((d: any) => d.source === 'cfn-guard'); + expect(guardDiagnostics.length).toBeGreaterThan(0); + + await client.closeDocument({ textDocument: { uri } }); + }); + + it('should receive diagnostics when typing new resource incrementally', async () => { + // Start with minimal template + const initialTemplate = `AWSTemplateFormatVersion: '2010-09-09' +Resources:`; + + const uri = await client.openYamlTemplate(initialTemplate); + await new Promise((resolve) => setTimeout(resolve, 300)); + + // Type resource name + await client.changeDocument({ + textDocument: { uri, version: 2 }, + contentChanges: [ + { + range: { + start: { line: 2, character: 10 }, + end: { line: 2, character: 10 }, + }, + text: ` + MyBucket:`, + }, + ], + }); + + await new Promise((resolve) => setTimeout(resolve, 200)); + + // Type resource type + await client.changeDocument({ + textDocument: { uri, version: 3 }, + contentChanges: [ + { + range: { + start: { line: 3, character: 11 }, + end: { line: 3, character: 11 }, + }, + text: ` + Type: AWS::S3::Bucket`, + }, + ], + }); + + await new Promise((resolve) => setTimeout(resolve, 500)); + + // Test validates incremental resource creation triggers diagnostics + expect(uri).toBeDefined(); + + await client.closeDocument({ textDocument: { uri } }); + }); + + it('should handle typing workflow for public access violations', async () => { + // Start with basic bucket + const template = `AWSTemplateFormatVersion: '2010-09-09' +Resources: + MyBucket: + Type: AWS::S3::Bucket + Properties: + BucketName: public-bucket`; + + const uri = await client.openYamlTemplate(template); + await new Promise((resolve) => setTimeout(resolve, 300)); + + // Type PublicAccessBlockConfiguration incrementally + await client.changeDocument({ + textDocument: { uri, version: 2 }, + contentChanges: [ + { + range: { + start: { line: 5, character: 33 }, + end: { line: 5, character: 33 }, + }, + text: ` + PublicAccessBlockConfiguration:`, + }, + ], + }); + + await new Promise((resolve) => setTimeout(resolve, 200)); + + // Add BlockPublicAcls: false + await client.changeDocument({ + textDocument: { uri, version: 3 }, + contentChanges: [ + { + range: { + start: { line: 6, character: 34 }, + end: { line: 6, character: 34 }, + }, + text: ` + BlockPublicAcls: false`, + }, + ], + }); + + await new Promise((resolve) => setTimeout(resolve, 200)); + + // Add remaining properties incrementally + await client.changeDocument({ + textDocument: { uri, version: 4 }, + contentChanges: [ + { + range: { + start: { line: 7, character: 23 }, + end: { line: 7, character: 23 }, + }, + text: ` + BlockPublicPolicy: false + IgnorePublicAcls: false + RestrictPublicBuckets: false`, + }, + ], + }); + + await new Promise((resolve) => setTimeout(resolve, 500)); + + // Test validates incremental typing of public access configuration + expect(uri).toBeDefined(); + + await client.closeDocument({ textDocument: { uri } }); + }); + }); +}); diff --git a/tst/integration/diagnostics/Guard.test.ts b/tst/integration/diagnostics/Guard.test.ts new file mode 100644 index 00000000..fb4217eb --- /dev/null +++ b/tst/integration/diagnostics/Guard.test.ts @@ -0,0 +1,188 @@ +import { describe, it } from 'vitest'; +import { DocumentType } from '../../../src/document/Document'; +import { DiagnosticExpectationBuilder, TemplateBuilder, TemplateScenario } from '../../utils/TemplateBuilder'; + +describe('Guard Validator Integration', () => { + describe('YAML', () => { + it('should detect S3 bucket versioning violations while authoring', async () => { + const template = new TemplateBuilder(DocumentType.YAML); + const scenario: TemplateScenario = { + name: 'S3 bucket versioning validation', + steps: [ + { + action: 'type', + content: `AWSTemplateFormatVersion: '2010-09-09' +Resources: + UnversionedBucket: + Type: AWS::S3::Bucket + Properties: + BucketName: unversioned-bucket`, + position: { line: 0, character: 0 }, + description: 'Create S3 bucket without versioning', + verification: { + position: { line: 3, character: 10 }, + expectation: DiagnosticExpectationBuilder.create() + .expectSource('cfn-guard') + .expectMessage(/versioning/i) + .expectMinCount(1) + .build(), + }, + }, + { + action: 'type', + content: ` + VersioningConfiguration: + Status: Enabled + PublicAccessBlockConfiguration: + BlockPublicAcls: true + BlockPublicPolicy: true + IgnorePublicAcls: true + RestrictPublicBuckets: true + LoggingConfiguration: + DestinationBucketName: !Ref LoggingBucket + LoggingBucket: + Type: AWS::S3::Bucket`, + position: { line: 5, character: 33 }, + description: 'Add all required configurations to resolve violations', + verification: { + position: { line: 3, character: 10 }, + expectation: DiagnosticExpectationBuilder.create() + .expectSource('cfn-guard') + .expectExactCount(0) + .build(), + }, + }, + ], + }; + + await template.executeScenario(scenario); + }); + + it('should detect S3 public access violations while authoring', async () => { + const template = new TemplateBuilder(DocumentType.YAML); + const scenario: TemplateScenario = { + name: 'S3 public access validation', + steps: [ + { + action: 'type', + content: `AWSTemplateFormatVersion: '2010-09-09' +Resources: + PublicBucket: + Type: AWS::S3::Bucket + Properties: + BucketName: public-bucket + PublicAccessBlockConfiguration: + BlockPublicAcls: false + BlockPublicPolicy: false + IgnorePublicAcls: false + RestrictPublicBuckets: false`, + position: { line: 0, character: 0 }, + description: 'Create S3 bucket with public access enabled', + verification: { + position: { line: 7, character: 25 }, + expectation: DiagnosticExpectationBuilder.create() + .expectSource('cfn-guard') + .expectMessage(/PublicAccessBlockConfiguration/i) + .expectSeverity(3) // Information severity + .expectMinCount(1) + .build(), + }, + }, + ], + }; + + await template.executeScenario(scenario); + }); + + it('should validate IAM policy structure while authoring', async () => { + const template = new TemplateBuilder(DocumentType.YAML); + const scenario: TemplateScenario = { + name: 'IAM policy validation', + steps: [ + { + action: 'type', + content: `AWSTemplateFormatVersion: '2010-09-09' +Resources: + OverlyPermissivePolicy: + Type: AWS::IAM::Policy + Properties: + PolicyName: AdminPolicy + PolicyDocument: + Version: '2012-10-17' + Statement: + - Effect: Allow + Action: '*' + Resource: '*' + Roles: + - !Ref MyRole + MyRole: + Type: AWS::IAM::Role + Properties: + AssumeRolePolicyDocument: + Version: '2012-10-17' + Statement: + - Effect: Allow + Principal: + Service: ec2.amazonaws.com + Action: 'sts:AssumeRole'`, + position: { line: 0, character: 0 }, + description: 'Create IAM policy with admin access', + verification: { + position: { line: 10, character: 20 }, + expectation: DiagnosticExpectationBuilder.create() + .expectSource('cfn-guard') + .expectMessage(/policy.*statements.*Effect.*Allow.*Action.*Resource/i) + .expectMinCount(1) + .build(), + }, + }, + ], + }; + + await template.executeScenario(scenario); + }); + }); + + describe('JSON', () => { + it('should detect S3 public access violations in JSON format', async () => { + const template = new TemplateBuilder(DocumentType.JSON); + const scenario: TemplateScenario = { + name: 'JSON S3 public access validation', + steps: [ + { + action: 'type', + content: `{ + "AWSTemplateFormatVersion": "2010-09-09", + "Resources": { + "PublicBucket": { + "Type": "AWS::S3::Bucket", + "Properties": { + "BucketName": "public-bucket", + "PublicAccessBlockConfiguration": { + "BlockPublicAcls": false, + "BlockPublicPolicy": false, + "IgnorePublicAcls": false, + "RestrictPublicBuckets": false + } + } + } + } +}`, + position: { line: 0, character: 0 }, + description: 'Create S3 bucket with public access enabled in JSON', + verification: { + position: { line: 4, character: 6 }, + expectation: DiagnosticExpectationBuilder.create() + .expectSource('cfn-guard') + .expectMessage(/public.*access/i) + .expectMinCount(1) + .build(), + }, + }, + ], + }; + + await template.executeScenario(scenario); + }); + }); +}); diff --git a/tst/resources/guard/test-guard-rules.guard b/tst/resources/guard/test-guard-rules.guard new file mode 100644 index 00000000..cb84f6a4 --- /dev/null +++ b/tst/resources/guard/test-guard-rules.guard @@ -0,0 +1,18 @@ +# Test Guard Rules for E2E Testing +# Simple rules that will trigger on common CloudFormation patterns + +rule test_bucket_must_have_encryption { + Resources.*[ Type == "AWS::S3::Bucket" ] { + Properties.BucketEncryption exists << + S3 buckets must have encryption configured + >> + } +} + +rule test_bucket_name_required { + Resources.*[ Type == "AWS::S3::Bucket" ] { + Properties.BucketName exists << + S3 buckets must have a BucketName specified + >> + } +} diff --git a/tst/utils/Expect.ts b/tst/utils/Expect.ts index 26066826..26005f4f 100644 --- a/tst/utils/Expect.ts +++ b/tst/utils/Expect.ts @@ -23,6 +23,11 @@ export function expectThrow(actual: any, message: string) { throw new Error(`${message}: expected ${actual} to be >= ${expected}`); } }, + toBeGreaterThan(expected: number) { + if (actual <= expected) { + throw new Error(`${message}: expected ${actual} to be > ${expected}`); + } + }, toBeLessThanOrEqual(expected: number) { if (actual > expected) { throw new Error(`${message}: expected ${actual} to be <= ${expected}`); diff --git a/tst/utils/MockServerComponents.ts b/tst/utils/MockServerComponents.ts index dac2ccf8..9d917ceb 100644 --- a/tst/utils/MockServerComponents.ts +++ b/tst/utils/MockServerComponents.ts @@ -138,8 +138,8 @@ export function createMockCfnLintService() { export function createMockGuardService() { const mock = stubInterface(); - mock.validate.returns(Promise.resolve()); - mock.validateDelayed.returns(Promise.resolve()); + mock.validate.resolves(); + mock.validateDelayed.resolves(); mock.cancelDelayedValidation.returns(); mock.cancelAllDelayedValidation.returns(); mock.getPendingValidationCount.returns(0); diff --git a/tst/utils/TemplateBuilder.ts b/tst/utils/TemplateBuilder.ts index dcad3880..a06dff3f 100644 --- a/tst/utils/TemplateBuilder.ts +++ b/tst/utils/TemplateBuilder.ts @@ -1,4 +1,4 @@ -import { CompletionParams, Location, DefinitionParams } from 'vscode-languageserver'; +import { CompletionParams, Location, DefinitionParams, Diagnostic } from 'vscode-languageserver'; import { TextDocuments } from 'vscode-languageserver/node'; import { TextDocumentContentChangeEvent, @@ -15,6 +15,7 @@ import { DocumentType, Document } from '../../src/document/Document'; import { DocumentManager } from '../../src/document/DocumentManager'; import { HoverRouter } from '../../src/hover/HoverRouter'; import { SchemaRetriever } from '../../src/schema/SchemaRetriever'; +import { GuardService } from '../../src/services/guard/GuardService'; import { UsageTracker } from '../../src/usageTracker/UsageTracker'; import { extractErrorMessage } from '../../src/utils/Errors'; import { expectThrow } from './Expect'; @@ -122,9 +123,13 @@ export class TemplateBuilder { private readonly completionRouter: CompletionRouter; private readonly hoverRouter: HoverRouter; private readonly definitionProvider: DefinitionProvider; + private readonly guardService: GuardService; + private readonly mockComponents: any; private readonly uri: string; private version: number = 0; + private diagnostics: Diagnostic[] = []; + constructor(format: DocumentType, startingContent: string = '') { this.uri = `file:///test-template.${format}`; this.textDocuments = new TextDocuments(TextDocument); @@ -141,10 +146,39 @@ export class TemplateBuilder { resourceStateManager: createMockResourceStateManager(), }); - const { core, external, providers } = createMockComponents(mockTestComponents); + this.mockComponents = createMockComponents(mockTestComponents); + + const { core, external, providers } = this.mockComponents; external.featureFlags.get.returns({ isEnabled: () => false, describe: () => 'mock' }); + // Configure Guard settings with enabled rule packs for testing + const guardSettings = { + enabled: true, + delayMs: 0, // No delay for tests + validateOnChange: true, + enabledRulePacks: ['cis-aws-benchmark-level-1'], // Use a real rule pack + timeout: 30000, + maxConcurrentValidations: 3, + maxQueueSize: 10, + memoryCleanupInterval: 60000, + maxMemoryUsage: 100 * 1024 * 1024, + defaultSeverity: 'information' as const, + }; + + // Mock settings manager to return our Guard configuration + core.settingsManager.getCurrentSettings.returns({ + profile: { region: 'us-east-1', profile: 'default' }, + hover: { enabled: true }, + completion: { enabled: true, maxCompletions: 100 }, + diagnostics: { + cfnLint: { enabled: false } as any, // Disable cfnLint for Guard-only tests + cfnGuard: guardSettings, + }, + editor: { tabSize: 2, insertSpaces: true, detectIndentation: true }, + awsClient: {} as any, + }); + const completionProviders = createCompletionProviders(core, external, providers); this.completionRouter = new CompletionRouter( @@ -157,6 +191,20 @@ export class TemplateBuilder { ); const mockFeatureFlag = { isEnabled: () => true, describe: () => 'Constants feature flag' }; this.hoverRouter = new HoverRouter(this.contextManager, this.schemaRetriever, mockFeatureFlag); + + // Create real GuardService for integration testing + this.guardService = GuardService.create(mockTestComponents); + + // Mock the diagnostic coordinator to capture diagnostics + mockTestComponents.core.diagnosticCoordinator.publishDiagnostics.callsFake( + (source: string, uri: string, diagnostics: Diagnostic[]) => { + if (source === 'cfn-guard' && uri === this.uri) { + this.diagnostics = diagnostics; + } + return Promise.resolve(); + }, + ); + this.initialize(startingContent); } @@ -284,6 +332,12 @@ export class TemplateBuilder { step.verification.expectation, step.verification.description, ); + } else if (step.verification?.expectation instanceof DiagnosticExpectation) { + await this.verifyDiagnosticsAt( + step.verification.position, + step.verification.expectation, + step.verification.description, + ); } } @@ -602,6 +656,78 @@ export class TemplateBuilder { } } + async verifyDiagnosticsAt(position: Position, expected: DiagnosticExpectation, description?: string) { + // Trigger Guard validation to get real diagnostics + const document = this.textDocuments.get(this.uri); + if (document) { + await this.guardService.validate(document.getText(), this.uri); + } + + const diagnostics = this.diagnostics; + const desc = description ? ` (${description})` : ''; + + if (expected.noDiagnostics) { + expectAt(diagnostics.length, position, `Expected no diagnostics${desc}`).toBe(0); + return; + } + + if (expected.source) { + const sourceDiagnostics = diagnostics.filter((d) => d.source === expected.source); + if (expected.exactCount !== undefined) { + expectAt( + sourceDiagnostics.length, + position, + `Diagnostic count mismatch for source ${expected.source}${desc}`, + ).toBe(expected.exactCount); + } + if (expected.minCount !== undefined) { + expectAt( + sourceDiagnostics.length, + position, + `Too few diagnostics for source ${expected.source}${desc}`, + ).toBeGreaterThanOrEqual(expected.minCount); + } + if (expected.maxCount !== undefined) { + expectAt( + sourceDiagnostics.length, + position, + `Too many diagnostics for source ${expected.source}${desc}`, + ).toBeLessThanOrEqual(expected.maxCount); + } + } + + if (expected.messagePattern) { + const matchingDiagnostics = diagnostics.filter((d) => expected.messagePattern!.test(d.message)); + expectAt( + matchingDiagnostics.length, + position, + `No diagnostics match pattern ${expected.messagePattern}${desc}`, + ).toBeGreaterThan(0); + } + + if (expected.severity !== undefined) { + const severityDiagnostics = diagnostics.filter((d) => d.severity === expected.severity); + expectAt( + severityDiagnostics.length, + position, + `No diagnostics with severity ${expected.severity}${desc}`, + ).toBeGreaterThan(0); + } + } + + async getDiagnosticsAt(_position: Position) { + const document = this.textDocuments.get(this.uri); + if (!document) { + return []; + } + + // Trigger Guard validation to populate diagnostics + await this.guardService.validate(document.getText(), this.uri); + + // Return the actual diagnostics captured from Guard validation + return this.diagnostics; + } + getCurrentContent() { const textDocument = this.textDocuments.get(this.uri); return textDocument?.getText() ?? ''; @@ -1023,3 +1149,60 @@ export class GotoExpectationBuilder { return this.expectation; } } + +class DiagnosticExpectation extends Expectation { + source?: string; + messagePattern?: RegExp; + severity?: number; + minCount?: number; + maxCount?: number; + exactCount?: number; + noDiagnostics?: boolean; +} + +export class DiagnosticExpectationBuilder { + private readonly expectation: DiagnosticExpectation = new DiagnosticExpectation(); + + static create(): DiagnosticExpectationBuilder { + return new DiagnosticExpectationBuilder(); + } + + expectSource(source: string): DiagnosticExpectationBuilder { + this.expectation.source = source; + return this; + } + + expectMessage(pattern: RegExp): DiagnosticExpectationBuilder { + this.expectation.messagePattern = pattern; + return this; + } + + expectSeverity(severity: number): DiagnosticExpectationBuilder { + this.expectation.severity = severity; + return this; + } + + expectMinCount(count: number): DiagnosticExpectationBuilder { + this.expectation.minCount = count; + return this; + } + + expectMaxCount(count: number): DiagnosticExpectationBuilder { + this.expectation.maxCount = count; + return this; + } + + expectExactCount(count: number): DiagnosticExpectationBuilder { + this.expectation.exactCount = count; + return this; + } + + expectNoDiagnostics(): DiagnosticExpectationBuilder { + this.expectation.noDiagnostics = true; + return this; + } + + build(): DiagnosticExpectation { + return this.expectation; + } +} diff --git a/tst/utils/TestExtension.ts b/tst/utils/TestExtension.ts index 7b2a96c2..ae7444c2 100644 --- a/tst/utils/TestExtension.ts +++ b/tst/utils/TestExtension.ts @@ -81,6 +81,8 @@ type TestExtensionConfig = { export class TestExtension implements Closeable { private readonly awsMetadata: AwsMetadata; private readonly initializeParams: ExtendedInitializeParams; + private readonly diagnosticsReceived: any[] = []; + private readonly mockWorkspaceConfig: Record = {}; private readonly readStream = new PassThrough(); private readonly writeStream = new PassThrough(); @@ -179,8 +181,12 @@ export class TestExtension implements Closeable { ); // Handle workspace/configuration requests from the server - this.clientConnection.onRequest('workspace/configuration', () => { - return config.workspaceConfig ?? [{}]; + this.clientConnection.onRequest('workspace/configuration', (params: any) => { + // Handle both array and single section requests + if (Array.isArray(params)) { + return params.map((item: any) => this.mockWorkspaceConfig[item.section] ?? {}); + } + return [this.mockWorkspaceConfig]; }); this.serverConnection.listen(); @@ -204,6 +210,11 @@ export class TestExtension implements Closeable { await this.clientConnection.sendRequest(InitializeRequest.type, this.initializeParams); await this.clientConnection.sendNotification(InitializedNotification.type, {}); + // Set up diagnostic listener + this.clientConnection.onNotification('textDocument/publishDiagnostics', (params: any) => { + this.diagnosticsReceived.push(params); + }); + await WaitFor.waitFor(() => { const store = this.external.schemaStore; const pbSchemas = store?.getPublicSchemas(DefaultSettings.profile.region); @@ -229,6 +240,7 @@ export class TestExtension implements Closeable { this.core.awsCredentials.handleIamCredentialsDelete(); this.core.usageTracker.clear(); this.core.validationManager.clear(); + this.diagnosticsReceived.length = 0; } async send(method: string, params: any) { @@ -342,6 +354,10 @@ export class TestExtension implements Closeable { } changeConfiguration(params: DidChangeConfigurationParams) { + // Update mock workspace config and notify + if (params.settings) { + Object.assign(this.mockWorkspaceConfig, params.settings); + } return this.notify(DidChangeConfigurationNotification.method, params); } @@ -349,6 +365,10 @@ export class TestExtension implements Closeable { return this.notify(DidChangeWorkspaceFoldersNotification.method, params); } + get receivedDiagnostics() { + return this.diagnosticsReceived; + } + updateIamCredentials(params: UpdateCredentialsParams) { return this.send(IamCredentialsUpdateRequest.method, params); }