telemetry(amazonq): add doc generation V2 telemetry (#6427)

## Problem
Add doc generation V2 telemetry

---

- Treat all work as PUBLIC. Private `feature/x` branches will not be
squash-merged at release time.
- Your code changes must meet the guidelines in
[CONTRIBUTING.md](https://github.com/aws/aws-toolkit-vscode/blob/master/CONTRIBUTING.md#guidelines).
- License: I confirm that my contribution is made under the terms of the
Apache 2.0 license.

packages/core/src/amazonq/commons/diff.ts

@@ -33,12 +33,18 @@ export function createAmazonQUri(path: string, tabId: string, scheme: string) {
     return vscode.Uri.from({ scheme: scheme, path, query: `tabID=${tabId}` })
 }
 
-export async function computeDiff(leftPath: string, rightPath: string, tabId: string, scheme: string) {
+export async function computeDiff(
+    leftPath: string,
+    rightPath: string,
+    tabId: string,
+    scheme: string,
+    reportedChanges?: string
+) {
     const { left, right } = await getFileDiffUris(leftPath, rightPath, tabId, scheme)
     const leftFile = await vscode.workspace.openTextDocument(left)
     const rightFile = await vscode.workspace.openTextDocument(right)
 
-    const changes = diffLines(leftFile.getText(), rightFile.getText(), {
+    const changes = diffLines(reportedChanges ?? leftFile.getText(), rightFile.getText(), {
         ignoreWhitespace: true,
     })
 

packages/core/src/amazonqDoc/controllers/chat/controller.ts

@@ -44,6 +44,7 @@ import {
 import { getPathsFromZipFilePath } from '../../../amazonqFeatureDev/util/files'
 import { FollowUpTypes } from '../../../amazonq/commons/types'
 import { DocGenerationTask } from '../docGenerationTask'
+import { DevPhase } from '../../types'
 
 export interface ChatControllerEventEmitters {
     readonly processHumanChatMessage: EventEmitter<any>
@@ -227,8 +228,6 @@ export class DocController {
                 return
             }
 
-            this.docGenerationTask.userIdentity = AuthUtil.instance.conn?.id
-
             const sendFolderConfirmationMessage = (message: string) => {
                 this.messenger.sendFolderConfirmationMessage(
                     data.tabID,
@@ -288,12 +287,12 @@ export class DocController {
                     break
                 case FollowUpTypes.AcceptChanges:
                     this.docGenerationTask.userDecision = 'ACCEPT'
-                    await this.sendDocGenerationEvent(data)
+                    await this.sendDocAcceptanceEvent(data)
                     await this.insertCode(data)
                     return
                 case FollowUpTypes.RejectChanges:
                     this.docGenerationTask.userDecision = 'REJECT'
-                    await this.sendDocGenerationEvent(data)
+                    await this.sendDocAcceptanceEvent(data)
                     this.messenger.sendAnswer({
                         type: 'answer',
                         tabID: data?.tabID,
@@ -323,7 +322,7 @@ export class DocController {
                     }
                     break
                 case FollowUpTypes.CancelFolderSelection:
-                    this.docGenerationTask.reset()
+                    this.docGenerationTask.folderLevel = 'ENTIRE_WORKSPACE'
                     return this.tabOpened(data)
             }
         })
@@ -488,7 +487,7 @@ export class DocController {
                 session.isAuthenticating = true
                 return
             }
-            this.docGenerationTask.numberOfNavigation += 1
+            this.docGenerationTask.numberOfNavigations += 1
             this.messenger.sendAnswer({
                 type: 'answer',
                 tabID: message.tabID,
@@ -595,6 +594,17 @@ export class DocController {
                     tabID: tabID,
                 })
             }
+            if (session?.state.phase === DevPhase.CODEGEN) {
+                const { totalGeneratedChars, totalGeneratedLines, totalGeneratedFiles } =
+                    await session.countGeneratedContent(this.docGenerationTask.interactionType)
+                this.docGenerationTask.conversationId = session.conversationId
+                this.docGenerationTask.numberOfGeneratedChars = totalGeneratedChars
+                this.docGenerationTask.numberOfGeneratedLines = totalGeneratedLines
+                this.docGenerationTask.numberOfGeneratedFiles = totalGeneratedFiles
+                const docGenerationEvent = this.docGenerationTask.docGenerationEventBase()
+
+                await session.sendDocTelemetryEvent(docGenerationEvent, 'generation')
+            }
         } finally {
             if (session?.state?.tokenSource?.token.isCancellationRequested) {
                 await this.newTask({ tabID })
@@ -652,18 +662,18 @@ export class DocController {
             )
         }
     }
-    private async sendDocGenerationEvent(message: any) {
+    private async sendDocAcceptanceEvent(message: any) {
         const session = await this.sessionStorage.getSession(message.tabID)
         this.docGenerationTask.conversationId = session.conversationId
         const { totalAddedChars, totalAddedLines, totalAddedFiles } = await session.countAddedContent(
             this.docGenerationTask.interactionType
         )
-        this.docGenerationTask.numberOfAddChars = totalAddedChars
-        this.docGenerationTask.numberOfAddLines = totalAddedLines
-        this.docGenerationTask.numberOfAddFiles = totalAddedFiles
-        const docGenerationEvent = this.docGenerationTask.docGenerationEventBase()
+        this.docGenerationTask.numberOfAddedChars = totalAddedChars
+        this.docGenerationTask.numberOfAddedLines = totalAddedLines
+        this.docGenerationTask.numberOfAddedFiles = totalAddedFiles
+        const docAcceptanceEvent = this.docGenerationTask.docAcceptanceEventBase()
 
-        await session.sendDocGenerationTelemetryEvent(docGenerationEvent)
+        await session.sendDocTelemetryEvent(docAcceptanceEvent, 'acceptance')
     }
     private processLink(message: any) {
         void openUrl(vscode.Uri.parse(message.link))

packages/core/src/amazonqDoc/controllers/docGenerationTask.ts

@@ -3,24 +3,27 @@
  * SPDX-License-Identifier: Apache-2.0
  */
 import {
-    DocGenerationEvent,
-    DocGenerationFolderLevel,
-    DocGenerationInteractionType,
-    DocGenerationUserDecision,
+    DocFolderLevel,
+    DocInteractionType,
+    DocUserDecision,
+    DocV2AcceptanceEvent,
+    DocV2GenerationEvent,
 } from '../../codewhisperer/client/codewhispereruserclient'
 import { getLogger } from '../../shared'
 
 export class DocGenerationTask {
     // Telemetry fields
     public conversationId?: string
-    public numberOfAddChars?: number
-    public numberOfAddLines?: number
-    public numberOfAddFiles?: number
-    public userDecision?: DocGenerationUserDecision
-    public interactionType?: DocGenerationInteractionType
-    public userIdentity?: string
-    public numberOfNavigation = 0
-    public folderLevel: DocGenerationFolderLevel = 'ENTIRE_WORKSPACE'
+    public numberOfAddedChars?: number
+    public numberOfAddedLines?: number
+    public numberOfAddedFiles?: number
+    public numberOfGeneratedChars?: number
+    public numberOfGeneratedLines?: number
+    public numberOfGeneratedFiles?: number
+    public userDecision?: DocUserDecision
+    public interactionType?: DocInteractionType
+    public numberOfNavigations = 0
+    public folderLevel: DocFolderLevel = 'ENTIRE_WORKSPACE'
 
     constructor(conversationId?: string) {
         this.conversationId = conversationId
@@ -32,31 +35,52 @@ export class DocGenerationTask {
             .map(([key]) => key)
 
         if (undefinedProps.length > 0) {
-            getLogger().debug(`DocGenerationEvent has undefined properties: ${undefinedProps.join(', ')}`)
+            getLogger().debug(`DocV2GenerationEvent has undefined properties: ${undefinedProps.join(', ')}`)
         }
-        const event: DocGenerationEvent = {
+        const event: DocV2GenerationEvent = {
             conversationId: this.conversationId ?? '',
-            numberOfAddChars: this.numberOfAddChars,
-            numberOfAddLines: this.numberOfAddLines,
-            numberOfAddFiles: this.numberOfAddFiles,
-            userDecision: this.userDecision,
+            numberOfGeneratedChars: this.numberOfGeneratedChars ?? 0,
+            numberOfGeneratedLines: this.numberOfGeneratedLines ?? 0,
+            numberOfGeneratedFiles: this.numberOfGeneratedFiles ?? 0,
             interactionType: this.interactionType,
-            userIdentity: this.userIdentity,
-            numberOfNavigation: this.numberOfNavigation,
+            numberOfNavigations: this.numberOfNavigations,
+            folderLevel: this.folderLevel,
+        }
+        return event
+    }
+
+    public docAcceptanceEventBase() {
+        const undefinedProps = Object.entries(this)
+            .filter(([key, value]) => value === undefined)
+            .map(([key]) => key)
+
+        if (undefinedProps.length > 0) {
+            getLogger().debug(`DocV2AcceptanceEvent has undefined properties: ${undefinedProps.join(', ')}`)
+        }
+        const event: DocV2AcceptanceEvent = {
+            conversationId: this.conversationId ?? '',
+            numberOfAddedChars: this.numberOfAddedChars ?? 0,
+            numberOfAddedLines: this.numberOfAddedLines ?? 0,
+            numberOfAddedFiles: this.numberOfAddedFiles ?? 0,
+            userDecision: this.userDecision ?? 'ACCEPTED',
+            interactionType: this.interactionType ?? 'GENERATE_README',
+            numberOfNavigations: this.numberOfNavigations ?? 0,
             folderLevel: this.folderLevel,
         }
         return event
     }
 
     public reset() {
         this.conversationId = undefined
-        this.numberOfAddChars = undefined
-        this.numberOfAddLines = undefined
-        this.numberOfAddFiles = undefined
+        this.numberOfAddedChars = undefined
+        this.numberOfAddedLines = undefined
+        this.numberOfAddedFiles = undefined
+        this.numberOfGeneratedChars = undefined
+        this.numberOfGeneratedLines = undefined
+        this.numberOfGeneratedFiles = undefined
         this.userDecision = undefined
         this.interactionType = undefined
-        this.userIdentity = undefined
-        this.numberOfNavigation = 0
+        this.numberOfNavigations = 0
         this.folderLevel = 'ENTIRE_WORKSPACE'
     }
 }

packages/core/src/amazonqDoc/session/session.ts

@@ -3,7 +3,7 @@
  * SPDX-License-Identifier: Apache-2.0
  */
 
-import { featureName, Mode } from '../constants'
+import { docScheme, featureName, Mode } from '../constants'
 import { DeletedFileInfo, Interaction, NewFileInfo, SessionState, SessionStateConfig } from '../types'
 import { PrepareCodeGenState } from './sessionState'
 import { telemetry } from '../../shared/telemetry/telemetry'
@@ -19,13 +19,14 @@ import { logWithConversationId } from '../../amazonqFeatureDev/userFacingText'
 import { ConversationIdNotFoundError } from '../../amazonqFeatureDev/errors'
 import { referenceLogText } from '../../amazonqFeatureDev/constants'
 import {
-    DocGenerationEvent,
-    DocGenerationInteractionType,
+    DocInteractionType,
+    DocV2AcceptanceEvent,
+    DocV2GenerationEvent,
     SendTelemetryEventRequest,
 } from '../../codewhisperer/client/codewhispereruserclient'
-import { getDiffCharsAndLines } from '../../shared/utilities/diffUtils'
 import { getClientId, getOperatingSystem, getOptOutPreference } from '../../shared/telemetry/util'
 import { DocMessenger } from '../messenger'
+import { computeDiff } from '../../amazonq/commons/diff'
 
 export class Session {
     private _state?: SessionState | Omit<SessionState, 'uploadId'>
@@ -38,6 +39,7 @@ export class Session {
 
     // Used to keep track of whether or not the current session is currently authenticating/needs authenticating
     public isAuthenticating: boolean
+    private _reportedDocChanges: { [key: string]: string } = {}
 
     constructor(
         public readonly config: SessionConfig,
@@ -177,41 +179,92 @@ export class Session {
         }
     }
 
-    public async countAddedContent(interactionType?: DocGenerationInteractionType) {
-        let totalAddedChars = 0
-        let totalAddedLines = 0
-        let totalAddedFiles = 0
+    private getFromReportedChanges(filepath: NewFileInfo) {
+        const key = `${filepath.workspaceFolder.uri.fsPath}/${filepath.relativePath}`
+        return this._reportedDocChanges[key]
+    }
 
-        for (const filePath of this.state.filePaths?.filter((i) => !i.rejected) ?? []) {
-            const absolutePath = path.join(filePath.workspaceFolder.uri.fsPath, filePath.relativePath)
-            const uri = filePath.virtualMemoryUri
-            const content = await this.config.fs.readFile(uri)
-            const decodedContent = new TextDecoder().decode(content)
-            totalAddedFiles += 1
+    private addToReportedChanges(filepath: NewFileInfo) {
+        const key = `${filepath.workspaceFolder.uri.fsPath}/${filepath.relativePath}`
+        this._reportedDocChanges[key] = filepath.fileContent
+    }
 
-            if ((await fs.exists(absolutePath)) && interactionType === 'UPDATE_README') {
-                const existingContent = await fs.readFileText(absolutePath)
-                const { addedChars, addedLines } = getDiffCharsAndLines(existingContent, decodedContent)
-                totalAddedChars += addedChars
-                totalAddedLines += addedLines
+    public async countGeneratedContent(interactionType?: DocInteractionType) {
+        let totalGeneratedChars = 0
+        let totalGeneratedLines = 0
+        let totalGeneratedFiles = 0
+        const filePaths = this.state.filePaths ?? []
+
+        for (const filePath of filePaths) {
+            const reportedDocChange = this.getFromReportedChanges(filePath)
+            if (interactionType === 'GENERATE_README') {
+                if (reportedDocChange) {
+                    const { charsAdded, linesAdded } = await this.computeFilePathDiff(filePath, reportedDocChange)
+                    totalGeneratedChars += charsAdded
+                    totalGeneratedLines += linesAdded
+                } else {
+                    // If no changes are reported, this is the initial README generation and no comparison with existing files is needed
+                    const fileContent = filePath.fileContent
+                    totalGeneratedChars += fileContent.length
+                    totalGeneratedLines += fileContent.split('\n').length
+                }
             } else {
-                totalAddedChars += decodedContent.length
-                totalAddedLines += decodedContent.split('\n').length
+                const { charsAdded, linesAdded } = await this.computeFilePathDiff(filePath, reportedDocChange)
+                totalGeneratedChars += charsAdded
+                totalGeneratedLines += linesAdded
             }
+            this.addToReportedChanges(filePath)
+            totalGeneratedFiles += 1
+        }
+        return {
+            totalGeneratedChars,
+            totalGeneratedLines,
+            totalGeneratedFiles,
         }
+    }
 
+    public async countAddedContent(interactionType?: DocInteractionType) {
+        let totalAddedChars = 0
+        let totalAddedLines = 0
+        let totalAddedFiles = 0
+        const newFilePaths =
+            this.state.filePaths?.filter((filePath) => !filePath.rejected && !filePath.changeApplied) ?? []
+
+        for (const filePath of newFilePaths) {
+            if (interactionType === 'GENERATE_README') {
+                const fileContent = filePath.fileContent
+                totalAddedChars += fileContent.length
+                totalAddedLines += fileContent.split('\n').length
+            } else {
+                const { charsAdded, linesAdded } = await this.computeFilePathDiff(filePath)
+                totalAddedChars += charsAdded
+                totalAddedLines += linesAdded
+            }
+            totalAddedFiles += 1
+        }
         return {
             totalAddedChars,
             totalAddedLines,
             totalAddedFiles,
         }
     }
-    public async sendDocGenerationTelemetryEvent(docGenerationEvent: DocGenerationEvent) {
+
+    public async computeFilePathDiff(filePath: NewFileInfo, reportedChanges?: string) {
+        const leftPath = `${filePath.workspaceFolder.uri.fsPath}/${filePath.relativePath}`
+        const rightPath = filePath.virtualMemoryUri.path
+        const diff = await computeDiff(leftPath, rightPath, this.tabID, docScheme, reportedChanges)
+        return { leftPath, rightPath, ...diff }
+    }
+
+    public async sendDocTelemetryEvent(
+        telemetryEvent: DocV2GenerationEvent | DocV2AcceptanceEvent,
+        eventType: 'generation' | 'acceptance'
+    ) {
         const client = await this.proxyClient.getClient()
         try {
             const params: SendTelemetryEventRequest = {
                 telemetryEvent: {
-                    docGenerationEvent,
+                    [eventType === 'generation' ? 'docV2GenerationEvent' : 'docV2AcceptanceEvent']: telemetryEvent,
                 },
                 optOutPreference: getOptOutPreference(),
                 userContext: {
@@ -222,13 +275,14 @@ export class Session {
                     ideVersion: extensionVersion,
                 },
             }
+
             const response = await client.sendTelemetryEvent(params).promise()
             getLogger().debug(
-                `${featureName}: successfully sent docGenerationEvent: ConversationId: ${docGenerationEvent.conversationId} RequestId: ${response.$response.requestId}`
+                `${featureName}: successfully sent docV2${eventType === 'generation' ? 'GenerationEvent' : 'AcceptanceEvent'}: ConversationId: ${telemetryEvent.conversationId} RequestId: ${response.$response.requestId}`
             )
         } catch (e) {
             getLogger().error(
-                `${featureName}: failed to send doc generation telemetry: ${(e as Error).name}: ${
+                `${featureName}: failed to send doc ${eventType} telemetry: ${(e as Error).name}: ${
                     (e as Error).message
                 } RequestId: ${(e as any).requestId}`
             )