From 4c190f4d9678a3bae38ab56000f5ff6123eea6e2 Mon Sep 17 00:00:00 2001 From: Laxman Reddy <141967714+laileni-aws@users.noreply.github.com> Date: Mon, 10 Mar 2025 13:48:13 -0700 Subject: [PATCH] feat(amazonq): feedback system for internal users for /test #6748 ## Problem - The unit test generation team lacks sufficient feedback to enhance the quality of their produced tests. ## Solution - Implementing the feedback system for internal users to evaluate and enhance unit test generation quality. --- .../webview/ui/apps/testChatConnector.ts | 1 + .../amazonqTest/chat/controller/controller.ts | 65 +++++++++++++++++-- .../chat/controller/messenger/messenger.ts | 25 ++++--- .../controller/messenger/messengerUtils.ts | 1 + .../chat/views/connector/connector.ts | 2 + 5 files changed, 75 insertions(+), 19 deletions(-) diff --git a/packages/core/src/amazonq/webview/ui/apps/testChatConnector.ts b/packages/core/src/amazonq/webview/ui/apps/testChatConnector.ts index ae179fd6c41..21e09db7cf5 100644 --- a/packages/core/src/amazonq/webview/ui/apps/testChatConnector.ts +++ b/packages/core/src/amazonq/webview/ui/apps/testChatConnector.ts @@ -162,6 +162,7 @@ export class Connector extends BaseConnector { body: messageData.message, canBeVoted: false, informationCard: messageData.informationCard, + buttons: messageData.buttons ?? [], } this.onChatAnswerReceived(messageData.tabID, answer, messageData) } diff --git a/packages/core/src/amazonqTest/chat/controller/controller.ts b/packages/core/src/amazonqTest/chat/controller/controller.ts index bd7b10ae4fb..984fb36e00e 100644 --- a/packages/core/src/amazonqTest/chat/controller/controller.ts +++ b/packages/core/src/amazonqTest/chat/controller/controller.ts @@ -26,7 +26,7 @@ import { import MessengerUtils, { ButtonActions } from './messenger/messengerUtils' import { getTelemetryReasonDesc, isAwsError } from '../../../shared/errors' import { ChatItemType } from '../../../amazonq/commons/model' -import { ProgressField } from '@aws/mynah-ui' +import { ChatItemButton, MynahIcons, ProgressField } from '@aws/mynah-ui' import { FollowUpTypes } from '../../../amazonq/commons/types' import { cancelBuild, @@ -63,6 +63,9 @@ import { } from '../../../codewhisperer/models/constants' import { UserWrittenCodeTracker } from '../../../codewhisperer/tracker/userWrittenCodeTracker' import { ReferenceLogViewProvider } from '../../../codewhisperer/service/referenceLogViewProvider' +import { submitFeedback } from '../../../feedback/vue/submitFeedback' +import { placeholder } from '../../../shared/vscode/commands2' +import { Auth } from '../../../auth/auth' export interface TestChatControllerEventEmitters { readonly tabOpened: vscode.EventEmitter @@ -307,10 +310,22 @@ export class TestController { ) if (session.stopIteration) { // Error from Science - this.messenger.sendMessage(data.error.uiMessage.replaceAll('```', ''), data.tabID, 'answer') + this.messenger.sendMessage( + data.error.uiMessage.replaceAll('```', ''), + data.tabID, + 'answer', + 'testGenErrorMessage', + this.getFeedbackButtons() + ) } else { isCancel - ? this.messenger.sendMessage(data.error.uiMessage, data.tabID, 'answer') + ? this.messenger.sendMessage( + data.error.uiMessage, + data.tabID, + 'answer', + 'testGenErrorMessage', + this.getFeedbackButtons() + ) : this.sendErrorMessage(data) } await this.sessionCleanUp() @@ -330,7 +345,9 @@ export class TestController { return this.messenger.sendMessage( i18n('AWS.amazonq.featureDev.error.monthlyLimitReached'), tabID, - 'answer' + 'answer', + 'testGenErrorMessage', + this.getFeedbackButtons() ) } if (error.message.includes('Too many requests')) { @@ -362,6 +379,7 @@ export class TestController { // This function handles actions if user clicked on any Button one of these cases will be executed private async handleFormActionClicked(data: any) { const typedAction = MessengerUtils.stringToEnumValue(ButtonActions, data.action as any) + let getFeedbackCommentData = '' switch (typedAction) { case ButtonActions.STOP_TEST_GEN: testGenState.setToCancelling() @@ -374,6 +392,16 @@ export class TestController { this.messenger.sendChatInputEnabled(data.tabID, true) await this.sessionCleanUp() break + case ButtonActions.PROVIDE_FEEDBACK: + getFeedbackCommentData = `Q Test Generation: RequestId: ${this.sessionStorage.getSession().startTestGenerationRequestId}, TestGenerationJobId: ${this.sessionStorage.getSession().testGenerationJob?.testGenerationJobId}` + void submitFeedback(placeholder, 'Amazon Q', getFeedbackCommentData) + telemetry.ui_click.emit({ elementId: 'unitTestGeneration_provideFeedback' }) + this.messenger.sendMessage( + 'Unit test generation completed. Thanks for providing feedback.', + data.tabID, + 'answer' + ) + break } } // This function handles actions if user gives any input from the chatInput box @@ -403,12 +431,31 @@ export class TestController { } } + private getFeedbackButtons(): ChatItemButton[] { + const buttons: ChatItemButton[] = [] + if (Auth.instance.isInternalAmazonUser()) { + buttons.push({ + keepCardAfterClick: false, + text: 'How can we make /test better?', + id: ButtonActions.PROVIDE_FEEDBACK, + disabled: false, // allow button to be re-clicked + position: 'outside', + icon: 'comment' as MynahIcons, + }) + } + return buttons + } + /** * Start Test Generation and show the code results */ private async startTestGen(message: any, regenerateTests: boolean) { const session: Session = this.sessionStorage.getSession() + // Perform session cleanup before start of unit test generation workflow unless there is an existing job in progress. + if (!ChatSessionManager.Instance.getIsInProgress()) { + await this.sessionCleanUp() + } const tabID = this.sessionStorage.setActiveTab(message.tabID) getLogger().debug('startTestGen message: %O', message) getLogger().debug('startTestGen tabId: %O', message.tabID) @@ -909,7 +956,13 @@ export class TestController { // TODO: Check if there are more cases to endSession if yes create a enum or type for step private async endSession(data: any, step: FollowUpTypes) { - this.messenger.sendMessage('Unit test generation completed.', data.tabID, 'answer') + this.messenger.sendMessage( + 'Unit test generation completed.', + data.tabID, + 'answer', + 'testGenEndSessionMessage', + this.getFeedbackButtons() + ) const session = this.sessionStorage.getSession() if (step === FollowUpTypes.RejectCode) { @@ -1342,7 +1395,7 @@ export class TestController { } session.listOfTestGenerationJobId = [] session.testGenerationJobGroupName = undefined - session.testGenerationJob = undefined + // session.testGenerationJob = undefined session.updatedBuildCommands = undefined session.shortAnswer = undefined session.testCoveragePercentage = 0 diff --git a/packages/core/src/amazonqTest/chat/controller/messenger/messenger.ts b/packages/core/src/amazonqTest/chat/controller/messenger/messenger.ts index ac317864c5b..5541ef389c5 100644 --- a/packages/core/src/amazonqTest/chat/controller/messenger/messenger.ts +++ b/packages/core/src/amazonqTest/chat/controller/messenger/messenger.ts @@ -24,7 +24,7 @@ import { UpdatePromptProgressMessage, } from '../../views/connector/connector' import { ChatItemType } from '../../../../amazonq/commons/model' -import { ChatItemAction, ProgressField } from '@aws/mynah-ui' +import { ChatItemAction, ChatItemButton, ProgressField } from '@aws/mynah-ui' import * as CodeWhispererConstants from '../../../../codewhisperer/models/constants' import { TriggerPayload } from '../../../../codewhispererChat/controllers/chat/model' import { @@ -76,8 +76,16 @@ export class Messenger { this.dispatcher.sendChatMessage(new CapabilityCardMessage(params.tabID)) } - public sendMessage(message: string, tabID: string, messageType: ChatItemType) { - this.dispatcher.sendChatMessage(new ChatMessage({ message, messageType }, tabID)) + public sendMessage( + message: string, + tabID: string, + messageType: ChatItemType, + messageId?: string, + buttons?: ChatItemButton[] + ) { + this.dispatcher.sendChatMessage( + new ChatMessage({ message, messageType, messageId: messageId, buttons: buttons }, tabID) + ) } public sendShortSummary(params: { @@ -159,16 +167,7 @@ export class Messenger { message = CodeWhispererConstants.invalidFileTypeChatMessage break } - - this.dispatcher.sendChatMessage( - new ChatMessage( - { - message, - messageType: 'answer-stream', - }, - tabID - ) - ) + this.sendMessage(message, tabID, 'answer-stream') } public sendErrorMessage(errorMessage: string, tabID: string) { diff --git a/packages/core/src/amazonqTest/chat/controller/messenger/messengerUtils.ts b/packages/core/src/amazonqTest/chat/controller/messenger/messengerUtils.ts index 9e8fec4594e..1eecc0aa4cd 100644 --- a/packages/core/src/amazonqTest/chat/controller/messenger/messengerUtils.ts +++ b/packages/core/src/amazonqTest/chat/controller/messenger/messengerUtils.ts @@ -12,6 +12,7 @@ export enum ButtonActions { VIEW_DIFF = 'View-Diff', STOP_TEST_GEN = 'Stop-Test-Generation', STOP_BUILD = 'Stop-Build-Process', + PROVIDE_FEEDBACK = 'Provide-Feedback', } // TODO: Refactor the common functionality between Transform, FeatureDev, CWSPRChat, Scan and UTG to a new Folder. diff --git a/packages/core/src/amazonqTest/chat/views/connector/connector.ts b/packages/core/src/amazonqTest/chat/views/connector/connector.ts index 8b79ea654ed..86c7b446b97 100644 --- a/packages/core/src/amazonqTest/chat/views/connector/connector.ts +++ b/packages/core/src/amazonqTest/chat/views/connector/connector.ts @@ -80,6 +80,7 @@ export class ChatMessage extends UiMessage { readonly messageId?: string | undefined readonly messageType: ChatItemType readonly canBeVoted?: boolean + readonly buttons?: ChatItemButton[] readonly informationCard: ChatItemContent['informationCard'] override type: TestMessageType = 'chatMessage' @@ -90,6 +91,7 @@ export class ChatMessage extends UiMessage { this.messageId = props.messageId || undefined this.canBeVoted = props.canBeVoted || undefined this.informationCard = props.informationCard || undefined + this.buttons = props.buttons || undefined } }