From 1a9191f36a73d6a926c3474cec6797cf46506844 Mon Sep 17 00:00:00 2001 From: Corina <14900841+corinagum@users.noreply.github.com> Date: Tue, 6 Feb 2024 12:11:41 -0800 Subject: [PATCH] [JS] chore: #54 Add unit tests for PromptManager.ts (#1246) ## Linked issues closes: #54 - Update docstrings for PromptManager.ts - Add unit tests for PromptManager.ts New JS coverage: 74.63 ## Attestation Checklist - [x] My code follows the style guidelines of this project - I have checked for/fixed spelling, linting, and other errors - I have commented my code for clarity - I have made corresponding changes to the documentation (updating the doc strings in the code is sufficient) - My changes generate no new warnings - I have added tests that validates my changes, and provides sufficient test coverage. I have tested with: - Local testing - E2E testing in Teams - New and existing unit tests pass locally with my changes Co-authored-by: Corina Gum <> --- .../internals/testing/assets/test/README.md | 1 + .../assets/test/missingConfig/actions.json | 16 ++ .../assets/test/missingConfig/skprompt.txt | 1 + .../assets/test/missingPrompt/actions.json | 16 ++ .../assets/test/missingPrompt/config.json | 21 ++ .../testing/assets/test/unknown/actions.json | 16 ++ .../testing/assets/test/unknown/config.json | 21 ++ .../testing/assets/test/unknown/skprompt.txt | 1 + .../src/prompts/PromptManager.spec.ts | 184 ++++++++++++++++++ .../teams-ai/src/prompts/PromptManager.ts | 91 ++++----- 10 files changed, 324 insertions(+), 44 deletions(-) create mode 100644 js/packages/teams-ai/src/internals/testing/assets/test/README.md create mode 100644 js/packages/teams-ai/src/internals/testing/assets/test/missingConfig/actions.json create mode 100644 js/packages/teams-ai/src/internals/testing/assets/test/missingConfig/skprompt.txt create mode 100644 js/packages/teams-ai/src/internals/testing/assets/test/missingPrompt/actions.json create mode 100644 js/packages/teams-ai/src/internals/testing/assets/test/missingPrompt/config.json create mode 100644 js/packages/teams-ai/src/internals/testing/assets/test/unknown/actions.json create mode 100644 js/packages/teams-ai/src/internals/testing/assets/test/unknown/config.json create mode 100644 js/packages/teams-ai/src/internals/testing/assets/test/unknown/skprompt.txt diff --git a/js/packages/teams-ai/src/internals/testing/assets/test/README.md b/js/packages/teams-ai/src/internals/testing/assets/test/README.md new file mode 100644 index 000000000..22c0929fc --- /dev/null +++ b/js/packages/teams-ai/src/internals/testing/assets/test/README.md @@ -0,0 +1 @@ +The files in this folder `/test` are used for unit tests in `PromptManager.spec.ts`. The files in this folder are not used in the actual package. diff --git a/js/packages/teams-ai/src/internals/testing/assets/test/missingConfig/actions.json b/js/packages/teams-ai/src/internals/testing/assets/test/missingConfig/actions.json new file mode 100644 index 000000000..5e0c865cb --- /dev/null +++ b/js/packages/teams-ai/src/internals/testing/assets/test/missingConfig/actions.json @@ -0,0 +1,16 @@ +[ + { + "name": "SendCard", + "description": "Sends an adaptive card to the user", + "parameters": { + "type": "object", + "properties": { + "card": { + "type": "object", + "description": "The adaptive card to send" + } + }, + "required": ["card"] + } + } +] diff --git a/js/packages/teams-ai/src/internals/testing/assets/test/missingConfig/skprompt.txt b/js/packages/teams-ai/src/internals/testing/assets/test/missingConfig/skprompt.txt new file mode 100644 index 000000000..61287e832 --- /dev/null +++ b/js/packages/teams-ai/src/internals/testing/assets/test/missingConfig/skprompt.txt @@ -0,0 +1 @@ +You are a friendly assistant for Microsoft Teams. \ No newline at end of file diff --git a/js/packages/teams-ai/src/internals/testing/assets/test/missingPrompt/actions.json b/js/packages/teams-ai/src/internals/testing/assets/test/missingPrompt/actions.json new file mode 100644 index 000000000..5e0c865cb --- /dev/null +++ b/js/packages/teams-ai/src/internals/testing/assets/test/missingPrompt/actions.json @@ -0,0 +1,16 @@ +[ + { + "name": "SendCard", + "description": "Sends an adaptive card to the user", + "parameters": { + "type": "object", + "properties": { + "card": { + "type": "object", + "description": "The adaptive card to send" + } + }, + "required": ["card"] + } + } +] diff --git a/js/packages/teams-ai/src/internals/testing/assets/test/missingPrompt/config.json b/js/packages/teams-ai/src/internals/testing/assets/test/missingPrompt/config.json new file mode 100644 index 000000000..4d8e860d9 --- /dev/null +++ b/js/packages/teams-ai/src/internals/testing/assets/test/missingPrompt/config.json @@ -0,0 +1,21 @@ +{ + "schema": 1.1, + "description": "A bot that can manage a list of work items.", + "type": "completion", + "completion": { + "model": "gpt-3.5-turbo", + "completion_type": "chat", + "include_history": true, + "include_input": true, + "max_input_tokens": 2800, + "max_tokens": 1000, + "temperature": 0.2, + "top_p": 0.0, + "presence_penalty": 0.6, + "frequency_penalty": 0.0, + "stop_sequences": [] + }, + "augmentation": { + "augmentation_type": "monologue" + } +} diff --git a/js/packages/teams-ai/src/internals/testing/assets/test/unknown/actions.json b/js/packages/teams-ai/src/internals/testing/assets/test/unknown/actions.json new file mode 100644 index 000000000..5e0c865cb --- /dev/null +++ b/js/packages/teams-ai/src/internals/testing/assets/test/unknown/actions.json @@ -0,0 +1,16 @@ +[ + { + "name": "SendCard", + "description": "Sends an adaptive card to the user", + "parameters": { + "type": "object", + "properties": { + "card": { + "type": "object", + "description": "The adaptive card to send" + } + }, + "required": ["card"] + } + } +] diff --git a/js/packages/teams-ai/src/internals/testing/assets/test/unknown/config.json b/js/packages/teams-ai/src/internals/testing/assets/test/unknown/config.json new file mode 100644 index 000000000..4d8e860d9 --- /dev/null +++ b/js/packages/teams-ai/src/internals/testing/assets/test/unknown/config.json @@ -0,0 +1,21 @@ +{ + "schema": 1.1, + "description": "A bot that can manage a list of work items.", + "type": "completion", + "completion": { + "model": "gpt-3.5-turbo", + "completion_type": "chat", + "include_history": true, + "include_input": true, + "max_input_tokens": 2800, + "max_tokens": 1000, + "temperature": 0.2, + "top_p": 0.0, + "presence_penalty": 0.6, + "frequency_penalty": 0.0, + "stop_sequences": [] + }, + "augmentation": { + "augmentation_type": "monologue" + } +} diff --git a/js/packages/teams-ai/src/internals/testing/assets/test/unknown/skprompt.txt b/js/packages/teams-ai/src/internals/testing/assets/test/unknown/skprompt.txt new file mode 100644 index 000000000..61287e832 --- /dev/null +++ b/js/packages/teams-ai/src/internals/testing/assets/test/unknown/skprompt.txt @@ -0,0 +1 @@ +You are a friendly assistant for Microsoft Teams. \ No newline at end of file diff --git a/js/packages/teams-ai/src/prompts/PromptManager.spec.ts b/js/packages/teams-ai/src/prompts/PromptManager.spec.ts index 5f886fa4b..c3a95e2dc 100644 --- a/js/packages/teams-ai/src/prompts/PromptManager.spec.ts +++ b/js/packages/teams-ai/src/prompts/PromptManager.spec.ts @@ -1,5 +1,12 @@ import { strict as assert } from 'assert'; +import path from 'path'; + +import { DataSource } from '../dataSources'; import { TestPromptManager } from '../internals/testing/TestPromptManager'; +import { ConfiguredPromptManagerOptions, PromptManager } from './PromptManager'; +import { Message } from './Message'; +import { PromptTemplate } from './PromptTemplate'; +import { RenderedPromptSection } from './PromptSection'; describe('PromptManager', () => { describe('constructor', () => { @@ -10,6 +17,89 @@ describe('PromptManager', () => { }); }); + it('should return options', () => { + const options: ConfiguredPromptManagerOptions = { + promptsFolder: 'promptFolder', + role: 'system', + max_conversation_history_tokens: -1, + max_history_messages: 10, + max_input_tokens: -1 + }; + const prompts = new TestPromptManager(options); + const configuredOptions = prompts.options; + assert(configuredOptions !== undefined); + assert(configuredOptions.promptsFolder === 'promptFolder'); + assert(configuredOptions.role === 'system'); + assert(configuredOptions.max_conversation_history_tokens === -1); + assert(configuredOptions.max_history_messages === 10); + }); + + describe('addDataSource', () => { + it('should add a data source', () => { + const prompts = new TestPromptManager(); + const newDataSource: DataSource = { + name: 'test', + renderData: async (context, memory, tokenizer, maxTokens): Promise> => { + return Promise.resolve({ text: 'test', output: 'test', length: 1, tokens: 1, tooLong: false }); + } + }; + prompts.addDataSource(newDataSource); + assert(newDataSource !== undefined); + }); + + it('should throw an error on adding duplicate data source', () => { + const prompts = new TestPromptManager(); + const newDataSource: DataSource = { + name: 'test', + renderData: async (context, memory, tokenizer, maxTokens): Promise> => { + return Promise.resolve({ text: 'test', output: 'test', length: 1, tokens: 1, tooLong: false }); + } + }; + prompts.addDataSource(newDataSource); + assert(newDataSource !== undefined); + assert.throws(() => prompts.addDataSource(newDataSource)); + }); + }); + + describe('getDataSource', () => { + it('should get a data source', () => { + const prompts = new TestPromptManager(); + const newDataSource: DataSource = { + name: 'test', + renderData: async (context, memory, tokenizer, maxTokens): Promise> => { + return Promise.resolve({ text: 'test', output: 'test', length: 1, tokens: 1, tooLong: false }); + } + }; + prompts.addDataSource(newDataSource); + const dataSource = prompts.getDataSource('test'); + assert(dataSource !== undefined); + }); + + it('should throw an error on getting a data source that does not exist', () => { + const prompts = new TestPromptManager(); + assert.throws(() => prompts.getDataSource('test')); + }); + }); + + describe('hasDataSource', () => { + it('should return true when a data source exists', () => { + const prompts = new TestPromptManager(); + const newDataSource: DataSource = { + name: 'test', + renderData: async (context, memory, tokenizer, maxTokens): Promise> => { + return Promise.resolve({ text: 'test', output: 'test', length: 1, tokens: 1, tooLong: false }); + } + }; + prompts.addDataSource(newDataSource); + assert(prompts.hasDataSource('test')); + }); + + it('should return false when a data source does not exist', () => { + const prompts = new TestPromptManager(); + assert(!prompts.hasDataSource('test')); + }); + }); + describe('addFunction', () => { it('should add a function', () => { const prompts = new TestPromptManager(); @@ -50,4 +140,98 @@ describe('PromptManager', () => { assert.equal(prompts.hasFunction('test'), true); }); }); + + describe('Prompts', () => { + const newPrompt: PromptTemplate = { + name: 'test', + prompt: { + required: true, + tokens: -1, + renderAsMessages(context, memory, functions, tokenizer, maxTokens) { + return Promise.resolve({ + messages: [], + output: [] as Message[], + length: 1, + tokens: 1, + tooLong: false + }); + }, + renderAsText(context, memory, functions, tokenizer, maxTokens) { + return Promise.resolve({ text: 'test', output: 'test', length: 1, tokens: 1, tooLong: false }); + } + }, + config: { + completion: { + frequency_penalty: 0, + include_history: true, + include_input: true, + include_images: false, + max_tokens: 100, + max_input_tokens: 2048, + model: 'test', + presence_penalty: 0, + temperature: 0, + top_p: 1 + }, + schema: 1.1, + type: 'completion' + } + }; + it('addPrompt should add a prompt', async () => { + const prompts = new TestPromptManager(); + prompts.addPrompt(newPrompt); + assert.deepEqual(await prompts.getPrompt('test'), newPrompt); + assert.equal(await prompts.hasPrompt('test'), true); + }); + + it('should throw when adding a prompt that already exists', () => { + const prompts = new TestPromptManager(); + prompts.addPrompt(newPrompt); + assert.throws(() => prompts.addPrompt(newPrompt)); + }); + + it(`should resolve the prompt template files if the prompt name doesn't exist`, async () => { + const prompts = new PromptManager({ + promptsFolder: path.join(__dirname, '..', 'internals', 'testing', 'assets', 'test') + }); + const prompt = await prompts.getPrompt('unknown'); + assert.notEqual(prompt, null); + }); + + it(`should throw an error if config.json is missing when using getPrompt`, async () => { + const prompts = new PromptManager({ + promptsFolder: path.join(__dirname, '..', 'internals', 'testing', 'assets', 'test') + }); + assert.rejects(async () => { + await prompts.getPrompt('missingConfig'); + }); + }); + + it(`should throw an error if skprompt.txt is missing when using getPrompt`, async () => { + const prompts = new PromptManager({ + promptsFolder: path.join(__dirname, '..', 'internals', 'testing', 'assets', 'test') + }); + assert.rejects(async () => { + await prompts.getPrompt('missingPrompt'); + }); + }); + + it(`should assign role 'system' if role is missing`, async () => { + const prompts = new PromptManager({ + role: '', + promptsFolder: path.join(__dirname, '..', 'internals', 'testing', 'assets', 'test') + }); + assert.rejects(async () => { + await prompts.getPrompt('missingPrompt'); + }); + }); + + it(`should resolve the prompt template files if the prompt name doesn't exist`, async () => { + const prompts = new PromptManager({ + promptsFolder: path.join(__dirname, '..', 'internals', 'testing', 'assets', 'test') + }); + const result = await prompts.hasPrompt('unknown'); + assert.equal(result, true); + }); + }); }); diff --git a/js/packages/teams-ai/src/prompts/PromptManager.ts b/js/packages/teams-ai/src/prompts/PromptManager.ts index e78c06624..d02bfc874 100644 --- a/js/packages/teams-ai/src/prompts/PromptManager.ts +++ b/js/packages/teams-ai/src/prompts/PromptManager.ts @@ -6,30 +6,31 @@ * Licensed under the MIT License. */ -import { PromptFunctions, PromptFunction } from './PromptFunctions'; -import { PromptTemplate, CompletionConfig } from './PromptTemplate'; -import { Tokenizer } from '../tokenizers'; import { TurnContext } from 'botbuilder'; import * as fs from 'fs/promises'; import * as path from 'path'; -import { TemplateSection } from './TemplateSection'; + +import { MonologueAugmentation, SequenceAugmentation } from '../augmentations'; import { DataSource } from '../dataSources'; -import { PromptSection } from './PromptSection'; import { Memory } from '../MemoryFork'; -import { DataSourceSection } from './DataSourceSection'; -import { MonologueAugmentation, SequenceAugmentation } from '../augmentations'; +import { Tokenizer } from '../tokenizers'; import { ConversationHistory } from './ConversationHistory'; -import { UserMessage } from './UserMessage'; +import { CompletionConfig, PromptTemplate } from './PromptTemplate'; +import { DataSourceSection } from './DataSourceSection'; import { GroupSection } from './GroupSection'; import { Prompt } from './Prompt'; +import { PromptFunctions, PromptFunction } from './PromptFunctions'; +import { PromptSection } from './PromptSection'; +import { TemplateSection } from './TemplateSection'; import { UserInputMessage } from './UserInputMessage'; +import { UserMessage } from './UserMessage'; /** * Options used to configure the prompt manager. */ export interface PromptManagerOptions { /** - * Path to the filesystem folder containing all the applications prompts. + * Path to the filesystem folder containing all the application's prompts. */ promptsFolder: string; @@ -41,10 +42,10 @@ export interface PromptManagerOptions { role?: string; /** - * Optional. Maximum number of tokens to of conversation history to include in prompts. + * Optional. Maximum number of tokens of conversation history to include in prompts. * @remarks * The default is to let conversation history consume the remainder of the prompts - * `max_input_tokens` budget. Setting this a value greater then 1 will override that and + * `max_input_tokens` budget. Setting this to a value greater than 1 will override that and * all prompts will use a fixed token budget. */ max_conversation_history_tokens?: number; @@ -61,7 +62,7 @@ export interface PromptManagerOptions { /** * Optional. Maximum number of tokens user input to include in prompts. * @remarks - * This defaults to unlimited but can set to a value greater then `1` to limit the length of + * This defaults to unlimited but can be set to a value greater than `1` to limit the length of * user input included in prompts. For example, if set to `100` then the any user input over * 100 tokens in length will be truncated. */ @@ -83,7 +84,7 @@ export interface ConfiguredPromptManagerOptions { role: string; /** - * Maximum number of tokens to of conversation history to include in prompts. + * Maximum number of tokens of conversation history to include in prompts. */ max_conversation_history_tokens: number; @@ -93,7 +94,7 @@ export interface ConfiguredPromptManagerOptions { max_history_messages: number; /** - * Maximum number of tokens user input to include in prompts. + * Maximum number of tokens of user input to include in prompts. */ max_input_tokens: number; } @@ -122,7 +123,8 @@ export class PromptManager implements PromptFunctions { /** * Creates a new 'PromptManager' instance. - * @param options Options used to configure the prompt manager. + * @param {PromptManagerOptions} options - Options used to configure the prompt manager. + * @returns {PromptManager} A new prompt manager instance. */ public constructor(options: PromptManagerOptions) { this._options = Object.assign( @@ -138,6 +140,7 @@ export class PromptManager implements PromptFunctions { /** * Gets the configured prompt manager options. + * @returns {ConfiguredPromptManagerOptions} The configured prompt manager options. */ public get options(): ConfiguredPromptManagerOptions { return this._options; @@ -145,8 +148,8 @@ export class PromptManager implements PromptFunctions { /** * Registers a new data source with the prompt manager. - * @param dataSource Data source to add. - * @returns The prompt manager for chaining. + * @param {DataSource} dataSource - Data source to add. + * @returns {this} The prompt manager for chaining. */ public addDataSource(dataSource: DataSource): this { if (this._dataSources.has(dataSource.name)) { @@ -160,8 +163,8 @@ export class PromptManager implements PromptFunctions { /** * Looks up a data source by name. - * @param name Name of the data source to lookup. - * @returns The data source. + * @param {string} name - Name of the data source to lookup. + * @returns {DataSource} The data source. */ public getDataSource(name: string): DataSource { const dataSource = this._dataSources.get(name); @@ -174,8 +177,8 @@ export class PromptManager implements PromptFunctions { /** * Checks for the existence of a named data source. - * @param name Name of the data source to lookup. - * @returns True if the data source exists. + * @param {string} name - Name of the data source to lookup. + * @returns {boolean} True if the data source exists. */ public hasDataSource(name: string): boolean { return this._dataSources.has(name); @@ -183,9 +186,9 @@ export class PromptManager implements PromptFunctions { /** * Registers a new prompt template function with the prompt manager. - * @param name Name of the function to add. - * @param fn Function to add. - * @returns The prompt manager for chaining. + * @param {string} name - Name of the function to add. + * @param {PromptFunction} fn - Function to add. + * @returns {this} - The prompt manager for chaining. */ public addFunction(name: string, fn: PromptFunction): this { if (this._functions.has(name)) { @@ -199,8 +202,8 @@ export class PromptManager implements PromptFunctions { /** * Looks up a prompt template function by name. - * @param name Name of the function to lookup. - * @returns The function. + * @param {string} name - Name of the function to lookup. + * @returns {PromptFunction} The function. */ public getFunction(name: string): PromptFunction { const fn = this._functions.get(name); @@ -213,8 +216,8 @@ export class PromptManager implements PromptFunctions { /** * Checks for the existence of a named prompt template function. - * @param name Name of the function to lookup. - * @returns True if the function exists. + * @param {string} name Name of the function to lookup. + * @returns {boolean} True if the function exists. */ public hasFunction(name: string): boolean { return this._functions.has(name); @@ -222,12 +225,12 @@ export class PromptManager implements PromptFunctions { /** * Invokes a prompt template function by name. - * @param name Name of the function to invoke. - * @param context Turn context for the current turn of conversation with the user. - * @param memory An interface for accessing state values. - * @param tokenizer Tokenizer to use when rendering the prompt. - * @param args Arguments to pass to the function. - * @returns Value returned by the function. + * @param {string} name - Name of the function to invoke. + * @param {TurnContext} context - Turn context for the current turn of conversation with the user. + * @param {Memory} memory - An interface for accessing state values. + * @param {Tokenizer} tokenizer - Tokenizer to use when rendering the prompt. + * @param {string[]} args - Arguments to pass to the function. + * @returns {Promise} Value returned by the function. */ public invokeFunction( name: string, @@ -242,13 +245,13 @@ export class PromptManager implements PromptFunctions { /** * Registers a new prompt template with the prompt manager. - * @param prompt Prompt template to add. - * @returns The prompt manager for chaining. + * @param {PromptTemplate} prompt - Prompt template to add. + * @returns {this} The prompt manager for chaining. */ public addPrompt(prompt: PromptTemplate): this { if (this._prompts.has(prompt.name)) { throw new Error( - `The PromptManager.addPrompt() method was called with a previously registered prompt named "${name}".` + `The PromptManager.addPrompt() method was called with a previously registered prompt named "${prompt.name}".` ); } @@ -265,8 +268,8 @@ export class PromptManager implements PromptFunctions { * The template will be pre-parsed and cached for use when the template is rendered by name. * * Any augmentations will also be added to the template. - * @param name Name of the prompt to load. - * @returns The loaded and parsed prompt template. + * @param {string} name - Name of the prompt to load. + * @returns {Promise} The loaded and parsed prompt template. */ public async getPrompt(name: string): Promise { if (!this._prompts.has(name)) { @@ -351,8 +354,8 @@ export class PromptManager implements PromptFunctions { /** * Checks for the existence of a named prompt. - * @param name Name of the prompt to load. - * @returns True if the prompt exists. + * @param {string} name - Name of the prompt to load. + * @returns {boolean} True if the prompt exists. */ public async hasPrompt(name: string): Promise { if (!this._prompts.has(name)) { @@ -372,7 +375,7 @@ export class PromptManager implements PromptFunctions { } /** - * @param template + * @param {PromptTemplate} template - The prompt template to update. * @private */ private updateConfig(template: PromptTemplate): void { @@ -402,8 +405,8 @@ export class PromptManager implements PromptFunctions { } /** - * @param template - * @param sections + * @param {PromptTemplate} template - The prompt template to append augmentations to. + * @param {PromptSection[]} sections - The prompt sections to append augmentations to. * @private */ private appendAugmentations(template: PromptTemplate, sections: PromptSection[]): void {