Skip to content
New issue

Have a question about this project? Sign up for a free GitHub account to open an issue and contact its maintainers and the community.

By clicking “Sign up for GitHub”, you agree to our terms of service and privacy statement. We’ll occasionally send you account related emails.

Already on GitHub? Sign in to your account

telemetry(amazonq): refactor doc telemetry to v2 version #5316

Open
wants to merge 5 commits into
base: main
Choose a base branch
from
Open
Show file tree
Hide file tree
Changes from all commits
Commits
File filter

Filter by extension

Filter by extension

Conversations
Failed to load comments.
Loading
Jump to
Jump to file
Failed to load files.
Loading
Diff view
Diff view
Original file line number Diff line number Diff line change
Expand Up @@ -13,7 +13,8 @@ import software.amazon.awssdk.services.codewhispererruntime.model.ContentChecksu
import software.amazon.awssdk.services.codewhispererruntime.model.CreateTaskAssistConversationRequest
import software.amazon.awssdk.services.codewhispererruntime.model.CreateTaskAssistConversationResponse
import software.amazon.awssdk.services.codewhispererruntime.model.CreateUploadUrlResponse
import software.amazon.awssdk.services.codewhispererruntime.model.DocGenerationEvent
import software.amazon.awssdk.services.codewhispererruntime.model.DocV2AcceptanceEvent
import software.amazon.awssdk.services.codewhispererruntime.model.DocV2GenerationEvent
import software.amazon.awssdk.services.codewhispererruntime.model.GetTaskAssistCodeGenerationResponse
import software.amazon.awssdk.services.codewhispererruntime.model.IdeCategory
import software.amazon.awssdk.services.codewhispererruntime.model.OperatingSystem
Expand Down Expand Up @@ -49,7 +50,7 @@ class AmazonQCodeGenerateClient(private val project: Project) {
OptOutPreference.OPTOUT
}

private val docGenerationUserContext = ClientMetadata.getDefault().let {
private val docUserContext = ClientMetadata.getDefault().let {
val osForFeatureDev: OperatingSystem =
when {
SystemInfo.isWindows -> OperatingSystem.WINDOWS
Expand All @@ -75,15 +76,17 @@ class AmazonQCodeGenerateClient(private val project: Project) {
private val amazonQStreamingClient
get() = AmazonQStreamingClient.getInstance(project)

fun sendDocGenerationTelemetryEvent(
docGenerationEvent: DocGenerationEvent,
fun sendDocTelemetryEvent(
generationEvent: DocV2GenerationEvent? = null,
acceptanceEvent: DocV2AcceptanceEvent? = null,
): SendTelemetryEventResponse =
bearerClient().sendTelemetryEvent { requestBuilder ->
requestBuilder.telemetryEvent { telemetryEventBuilder ->
telemetryEventBuilder.docGenerationEvent(docGenerationEvent)
generationEvent?.let { telemetryEventBuilder.docV2GenerationEvent(it) }
acceptanceEvent?.let { telemetryEventBuilder.docV2AcceptanceEvent(it) }
}
requestBuilder.optOutPreference(getTelemetryOptOutPreference())
requestBuilder.userContext(docGenerationUserContext)
requestBuilder.userContext(docUserContext)
}

fun createTaskAssistConversation(): CreateTaskAssistConversationResponse = bearerClient().createTaskAssistConversation(
Expand Down
Original file line number Diff line number Diff line change
Expand Up @@ -18,9 +18,9 @@ import com.intellij.openapi.vfs.LocalFileSystem
import com.intellij.openapi.vfs.VfsUtil
import com.intellij.openapi.wm.ToolWindowManager
import kotlinx.coroutines.withContext
import software.amazon.awssdk.services.codewhispererruntime.model.DocGenerationFolderLevel
import software.amazon.awssdk.services.codewhispererruntime.model.DocGenerationInteractionType
import software.amazon.awssdk.services.codewhispererruntime.model.DocGenerationUserDecision
import software.amazon.awssdk.services.codewhispererruntime.model.DocFolderLevel
import software.amazon.awssdk.services.codewhispererruntime.model.DocInteractionType
import software.amazon.awssdk.services.codewhispererruntime.model.DocUserDecision
import software.aws.toolkits.core.utils.debug
import software.aws.toolkits.core.utils.error
import software.aws.toolkits.core.utils.getLogger
Expand Down Expand Up @@ -168,25 +168,25 @@ class DocController(
FollowUpTypes.NEW_TASK -> newTask(message.tabId)
FollowUpTypes.CLOSE_SESSION -> closeSession(message.tabId)
FollowUpTypes.CREATE_DOCUMENTATION -> {
docGenerationTask.interactionType = DocGenerationInteractionType.GENERATE_README
docGenerationTask.interactionType = DocInteractionType.GENERATE_README
mode = Mode.CREATE
promptForDocTarget(message.tabId)
}

FollowUpTypes.UPDATE_DOCUMENTATION -> {
docGenerationTask.interactionType = DocGenerationInteractionType.UPDATE_README
docGenerationTask.interactionType = DocInteractionType.UPDATE_README
updateDocumentation(message.tabId)
}

FollowUpTypes.CANCEL_FOLDER_SELECTION -> {
docGenerationTask.reset()
docGenerationTask.folderLevel = DocFolderLevel.ENTIRE_WORKSPACE
newTask(message.tabId)
}

FollowUpTypes.PROCEED_FOLDER_SELECTION -> if (mode == Mode.EDIT) makeChanges(message.tabId) else onDocsGeneration(message)
FollowUpTypes.ACCEPT_CHANGES -> {
docGenerationTask.userDecision = DocGenerationUserDecision.ACCEPT
sendDocGenerationTelemetry(message.tabId)
docGenerationTask.userDecision = DocUserDecision.ACCEPT
sendDocAcceptanceTelemetry(message.tabId)
acceptChanges(message)
}

Expand All @@ -196,8 +196,8 @@ class DocController(
}

FollowUpTypes.REJECT_CHANGES -> {
docGenerationTask.userDecision = DocGenerationUserDecision.REJECT
sendDocGenerationTelemetry(message.tabId)
docGenerationTask.userDecision = DocUserDecision.REJECT
sendDocAcceptanceTelemetry(message.tabId)
rejectChanges(message)
}

Expand All @@ -208,7 +208,7 @@ class DocController(

FollowUpTypes.EDIT_DOCUMENTATION -> {
mode = Mode.EDIT
docGenerationTask.interactionType = DocGenerationInteractionType.EDIT_README
docGenerationTask.interactionType = DocInteractionType.EDIT_README
promptForDocTarget(message.tabId)
}
}
Expand Down Expand Up @@ -454,8 +454,7 @@ class DocController(
session.isAuthenticating = true
return
}
docGenerationTask.userIdentity = session.getUserIdentity()
docGenerationTask.numberOfNavigation += 1
docGenerationTask.numberOfNavigations += 1
messenger.sendUpdatePlaceholder(tabId, message("amazonqDoc.prompt.placeholder"))
} catch (err: Exception) {
val message = createUserFacingErrorMessage(err.message)
Expand Down Expand Up @@ -733,7 +732,6 @@ class DocController(
session.isAuthenticating = true
return
}
docGenerationTask.userIdentity = session.getUserIdentity()
session.preloader(message, messenger)

when (session.sessionState.phase) {
Expand All @@ -747,6 +745,7 @@ class DocController(
is PrepareDocGenerationState -> state.filePaths
else -> emptyList()
}
sendDocGenerationTelemetry(filePaths, session)
broadcastQEvent(QFeatureEvent.INVOCATION)

if (filePaths.isNotEmpty()) {
Expand Down Expand Up @@ -811,6 +810,8 @@ class DocController(
return
}

sendDocGenerationTelemetry(filePaths, session)

messenger.sendAnswer(
message = docGenerationProgressMessage(DocGenerationStep.COMPLETE, mode),
messageType = DocMessageType.AnswerPart,
Expand Down Expand Up @@ -988,9 +989,9 @@ class DocController(
}

if (selectedFolder.path == projectRoot.path) {
docGenerationTask.folderLevel = DocGenerationFolderLevel.ENTIRE_WORKSPACE
docGenerationTask.folderLevel = DocFolderLevel.ENTIRE_WORKSPACE
} else {
docGenerationTask.folderLevel = DocGenerationFolderLevel.SUB_FOLDER
docGenerationTask.folderLevel = DocFolderLevel.SUB_FOLDER
}

logger.info { "Selected correct folder inside workspace: ${selectedFolder.path}" }
Expand All @@ -1005,7 +1006,18 @@ class DocController(
}
}

private fun sendDocGenerationTelemetry(tabId: String) {
private fun sendDocGenerationTelemetry(filePaths: List<NewFileZipInfo>, session: DocSession) {
docGenerationTask.conversationId = session.conversationId
val (totalGeneratedChars, totalGeneratedLines, totalGeneratedFiles) = session.countedGeneratedContent(filePaths, docGenerationTask.interactionType)
docGenerationTask.numberOfGeneratedChars = totalGeneratedChars
docGenerationTask.numberOfGeneratedLines = totalGeneratedLines
docGenerationTask.numberOfGeneratedFiles = totalGeneratedFiles

val docGenerationEvent = docGenerationTask.docGenerationEventBase()
session.sendDocTelemetryEvent(docGenerationEvent)
}

private fun sendDocAcceptanceTelemetry(tabId: String) {
val session = getSessionInfo(tabId)
var filePaths: List<NewFileZipInfo> = emptyList()

Expand All @@ -1016,12 +1028,12 @@ class DocController(
}
docGenerationTask.conversationId = session.conversationId
val (totalAddedChars, totalAddedLines, totalAddedFiles) = session.countAddedContent(filePaths, docGenerationTask.interactionType)
docGenerationTask.numberOfAddChars = totalAddedChars
docGenerationTask.numberOfAddLines = totalAddedLines
docGenerationTask.numberOfAddFiles = totalAddedFiles
docGenerationTask.numberOfAddedChars = totalAddedChars
docGenerationTask.numberOfAddedLines = totalAddedLines
docGenerationTask.numberOfAddedFiles = totalAddedFiles

val docGenerationEvent = docGenerationTask.docGenerationEventBase()
session.sendDocGenerationEvent(docGenerationEvent)
val docAcceptanceEvent = docGenerationTask.docAcceptanceEventBase()
session.sendDocTelemetryEvent(null, docAcceptanceEvent)
}

fun getProject() = context.project
Expand Down
Original file line number Diff line number Diff line change
Expand Up @@ -3,57 +3,82 @@

package software.aws.toolkits.jetbrains.services.amazonqDoc.controller

import software.amazon.awssdk.services.codewhispererruntime.model.DocGenerationEvent
import software.amazon.awssdk.services.codewhispererruntime.model.DocGenerationFolderLevel
import software.amazon.awssdk.services.codewhispererruntime.model.DocGenerationInteractionType
import software.amazon.awssdk.services.codewhispererruntime.model.DocGenerationUserDecision
import software.amazon.awssdk.services.codewhispererruntime.model.DocFolderLevel
import software.amazon.awssdk.services.codewhispererruntime.model.DocInteractionType
import software.amazon.awssdk.services.codewhispererruntime.model.DocUserDecision
import software.amazon.awssdk.services.codewhispererruntime.model.DocV2AcceptanceEvent
import software.amazon.awssdk.services.codewhispererruntime.model.DocV2GenerationEvent
import software.aws.toolkits.core.utils.debug
import software.aws.toolkits.core.utils.getLogger

class DocGenerationTask {
// Telemetry fields
var conversationId: String? = null
var numberOfAddChars: Int? = null
var numberOfAddLines: Int? = null
var numberOfAddFiles: Int? = null
var userDecision: DocGenerationUserDecision? = null
var interactionType: DocGenerationInteractionType? = null
var userIdentity: String? = null
var numberOfNavigation = 0
var folderLevel: DocGenerationFolderLevel? = DocGenerationFolderLevel.ENTIRE_WORKSPACE
fun docGenerationEventBase(): DocGenerationEvent {
var numberOfAddedChars: Int? = null
var numberOfAddedLines: Int? = null
var numberOfAddedFiles: Int? = null
var numberOfGeneratedChars: Int? = null
var numberOfGeneratedLines: Int? = null
var numberOfGeneratedFiles: Int? = null
var userDecision: DocUserDecision? = null
var interactionType: DocInteractionType? = null
var numberOfNavigations = 0
var folderLevel: DocFolderLevel? = DocFolderLevel.ENTIRE_WORKSPACE
fun docGenerationEventBase(): DocV2GenerationEvent {
val undefinedProps = this::class.java.declaredFields
.filter { it.get(this) == null }
.map { it.name }

if (undefinedProps.isNotEmpty()) {
val undefinedValue = undefinedProps.joinToString(", ")
logger.debug { "DocGenerationEvent has undefined properties: $undefinedValue" }
logger.debug { "DocV2GenerationEvent has undefined properties: $undefinedValue" }
}

return DocGenerationEvent.builder()
return DocV2GenerationEvent.builder()
.conversationId(conversationId)
.numberOfAddChars(numberOfAddChars)
.numberOfAddLines(numberOfAddLines)
.numberOfAddFiles(numberOfAddFiles)
.numberOfGeneratedChars(numberOfGeneratedChars)
.numberOfGeneratedLines(numberOfGeneratedLines)
.numberOfGeneratedFiles(numberOfGeneratedFiles)
.interactionType(interactionType)
.numberOfNavigations(numberOfNavigations)
.folderLevel(folderLevel)
.build()
}

fun docAcceptanceEventBase(): DocV2AcceptanceEvent {
val undefinedProps = this::class.java.declaredFields
.filter { it.get(this) == null }
.map { it.name }

if (undefinedProps.isNotEmpty()) {
val undefinedValue = undefinedProps.joinToString(", ")
logger.debug { "DocV2AcceptanceEvent has undefined properties: $undefinedValue" }
}

return DocV2AcceptanceEvent.builder()
.conversationId(conversationId)
.numberOfAddedChars(numberOfAddedChars)
.numberOfAddedLines(numberOfAddedLines)
.numberOfAddedFiles(numberOfAddedFiles)
.userDecision(userDecision)
.interactionType(interactionType)
.userIdentity(userIdentity)
.numberOfNavigation(numberOfNavigation)
.numberOfNavigations(numberOfNavigations)
.folderLevel(folderLevel)
.build()
}

fun reset() {
conversationId = null
numberOfAddChars = null
numberOfAddLines = null
numberOfAddFiles = null
numberOfAddedChars = null
numberOfAddedLines = null
numberOfAddedFiles = null
numberOfGeneratedChars = null
numberOfGeneratedLines = null
numberOfGeneratedFiles = null
userDecision = null
interactionType = null
userIdentity = null
numberOfNavigation = 0
folderLevel = null
numberOfNavigations = 0
folderLevel = DocFolderLevel.ENTIRE_WORKSPACE
}

companion object {
Expand Down
Loading
Loading