telemetry(amazonq): status field tracking user actions #6750

## Problem - The current unit test generation telemetry event lacks a field to distinguish between user actions: acceptance, rejection, failure, or cancellation. ## Solution - Enhancing the amazonq_utgGenerateTests telemetry event by incorporating a status field to capture user actions in toolkit metrics. - Bumping telemetry version to `1.0.307`
aws · Mar 10, 2025 · 600ee8f · 600ee8f
1 parent 514e78d
commit 600ee8f
Show file tree

Hide file tree

Showing 6 changed files with 44 additions and 13 deletions.
diff --git a/package-lock.json b/package-lock.json
diff --git a/package.json b/package.json
@@ -41,7 +41,7 @@
         "skippedTestReport": "ts-node ./scripts/skippedTestReport.ts ./packages/amazonq/test/e2e/"
     },
     "devDependencies": {
-        "@aws-toolkits/telemetry": "^1.0.305",
+        "@aws-toolkits/telemetry": "^1.0.307",
         "@playwright/browser-chromium": "^1.43.1",
         "@stylistic/eslint-plugin": "^2.11.0",
         "@types/he": "^1.2.3",

diff --git a/packages/core/src/amazonqTest/chat/controller/controller.ts b/packages/core/src/amazonqTest/chat/controller/controller.ts
@@ -286,7 +286,7 @@ export class TestController {
         }
         TelemetryHelper.instance.sendTestGenerationToolkitEvent(
             session,
-            true,
+            session.isSupportedLanguage,
             true,
             isCancel ? 'Cancelled' : 'Failed',
             session.startTestGenerationRequestId,
@@ -295,7 +295,15 @@ export class TestController {
             session.isCodeBlockSelected,
             session.artifactsUploadDuration,
             session.srcPayloadSize,
-            session.srcZipFileSize
+            session.srcZipFileSize,
+            session.charsOfCodeAccepted,
+            session.numberOfTestsGenerated,
+            session.linesOfCodeGenerated,
+            session.charsOfCodeGenerated,
+            session.numberOfTestsGenerated,
+            session.linesOfCodeGenerated,
+            undefined,
+            isCancel ? 'CANCELLED' : 'FAILED'
         )
         if (session.stopIteration) {
             // Error from Science
@@ -502,6 +510,7 @@ export class TestController {
                     unsupportedMessage = `<span style="color: #EE9D28;">&#9888;<b>I'm sorry, but /test only supports Python and Java</b><br></span> I will still generate a suggestion below.`
                 }
                 this.messenger.sendMessage(unsupportedMessage, tabID, 'answer')
+                session.isSupportedLanguage = false
                 await this.onCodeGeneration(
                     session,
                     userPrompt,
@@ -529,6 +538,7 @@ export class TestController {
                     )
                 }
                 session.isCodeBlockSelected = selectionRange !== undefined
+                session.isSupportedLanguage = true
 
                 /**
                  * Zip the project
@@ -792,7 +802,9 @@ export class TestController {
             session.linesOfCodeAccepted,
             session.charsOfCodeGenerated,
             session.numberOfTestsGenerated,
-            session.linesOfCodeGenerated
+            session.linesOfCodeGenerated,
+            undefined,
+            'ACCEPTED'
         )
 
         await this.endSession(message, FollowUpTypes.SkipBuildAndFinish)
@@ -918,7 +930,9 @@ export class TestController {
                 0,
                 session.charsOfCodeGenerated,
                 session.numberOfTestsGenerated,
-                session.linesOfCodeGenerated
+                session.linesOfCodeGenerated,
+                undefined,
+                'REJECTED'
             )
             telemetry.ui_click.emit({ elementId: 'unitTestGeneration_rejectDiff' })
         }

diff --git a/packages/core/src/amazonqTest/chat/controller/messenger/messenger.ts b/packages/core/src/amazonqTest/chat/controller/messenger/messenger.ts
@@ -296,7 +296,8 @@ export class Messenger {
                         undefined,
                         undefined,
                         undefined,
-                        'TestGenCancelled'
+                        'TestGenCancelled',
+                        'CANCELLED'
                     )
                     this.dispatcher.sendUpdatePromptProgress(
                         new UpdatePromptProgressMessage(tabID, cancellingProgressField)
@@ -310,7 +311,19 @@ export class Messenger {
                         'Succeeded',
                         messageId,
                         performance.now() - session.testGenerationStartTime,
-                        undefined
+                        undefined,
+                        undefined,
+                        undefined,
+                        undefined,
+                        undefined,
+                        undefined,
+                        undefined,
+                        undefined,
+                        undefined,
+                        undefined,
+                        undefined,
+                        undefined,
+                        'ACCEPTED'
                     )
                     this.dispatcher.sendUpdatePromptProgress(
                         new UpdatePromptProgressMessage(tabID, testGenCompletedField)

diff --git a/packages/core/src/amazonqTest/chat/session/session.ts b/packages/core/src/amazonqTest/chat/session/session.ts
@@ -35,6 +35,7 @@ export class Session {
     public testGenerationJob: TestGenerationJob | undefined
 
     // Start Test generation
+    public isSupportedLanguage: boolean = false
     public conversationState: ConversationState = ConversationState.IDLE
     public shortAnswer: ShortAnswer | undefined
     public sourceFilePath: string = ''

diff --git a/packages/core/src/codewhisperer/util/telemetryHelper.ts b/packages/core/src/codewhisperer/util/telemetryHelper.ts
@@ -13,6 +13,7 @@ import {
     CodewhispererPreviousSuggestionState,
     CodewhispererUserDecision,
     CodewhispererUserTriggerDecision,
+    Status,
     telemetry,
 } from '../../shared/telemetry/telemetry'
 import { CodewhispererCompletionType, CodewhispererSuggestionState } from '../../shared/telemetry/telemetry'
@@ -85,12 +86,13 @@ export class TelemetryHelper {
         generatedCharactersCount?: number,
         generatedCount?: number,
         generatedLinesCount?: number,
-        reason?: string
+        reason?: string,
+        status?: Status
     ) {
         telemetry.amazonq_utgGenerateTests.emit({
             cwsprChatProgrammingLanguage: session.fileLanguage ?? 'plaintext',
             hasUserPromptSupplied: session.hasUserPromptSupplied,
-            isSupportedLanguage: isSupportedLanguage,
+            isSupportedLanguage: session.isSupportedLanguage,
             isFileInWorkspace: isFileInWorkspace,
             result: result,
             artifactsUploadDuration: artifactsUploadDuration,
@@ -110,6 +112,7 @@ export class TelemetryHelper {
             requestId: requestId,
             reasonDesc: reasonDesc,
             reason: reason,
+            status: status,
         })
     }