Vision support and sample

js/packages/teams-ai/package.json

@@ -2,7 +2,7 @@
     "name": "@microsoft/teams-ai",
     "author": "Microsoft Corp.",
     "description": "SDK focused on building AI based applications for Microsoft Teams.",
-    "version": "1.0.0-preview.1",
+    "version": "1.0.0-preview.2",
     "license": "MIT",
     "keywords": [
         "botbuilder",

js/packages/teams-ai/src/AI.ts

@@ -474,17 +474,6 @@ export class AI<TState extends TurnState = TurnState> {
         start_time?: number,
         step_count?: number
     ): Promise<boolean> {
-        // Populate {{$temp.input}}
-        if (typeof state.temp.input != 'string') {
-            // Use the received activity text
-            state.temp.input = context.activity.text;
-        }
-
-        // Initialize {{$allOutputs}}
-        if (state.temp.actionOutputs == undefined) {
-            state.temp.actionOutputs = {};
-        }
-
         // Initialize start time and action count
         const { max_steps, max_time } = this._options;
         if (start_time === undefined) {
@@ -570,6 +559,7 @@ export class AI<TState extends TurnState = TurnState> {
             // Copy the actions output to the input
             state.temp.lastOutput = output;
             state.temp.input = output;
+            state.temp.input_files = [];
         }
 
         // Check for looping

js/packages/teams-ai/src/Application.ts

@@ -22,6 +22,7 @@ import { MessageExtensions } from './MessageExtensions';
 import { TaskModules, TaskModulesOptions } from './TaskModules';
 import { AuthenticationManager, AuthenticationOptions } from './authentication/Authentication';
 import { TurnState } from './TurnState';
+import { InputFileDownloader } from './InputFileDownloader';
 
 /**
  * @private
@@ -122,6 +123,11 @@ export interface ApplicationOptions<TState extends TurnState> {
      * Optional. Factory used to create a custom turn state instance.
      */
     turnStateFactory: () => TState;
+
+    /**
+     * Optional. Array of input file download plugins to use.
+     */
+    fileDownloaders?: InputFileDownloader<TState>[];
 }
 
 /**
@@ -570,6 +576,27 @@ export class Application<TState extends TurnState = TurnState> {
                     return false;
                 }
 
+                // Populate {{$temp.input}}
+                if (typeof state.temp.input != 'string') {
+                    // Use the received activity text
+                    state.temp.input = context.activity.text;
+                }
+
+                // Download any input files
+                if (Array.isArray(this._options.fileDownloaders) && this._options.fileDownloaders.length > 0) {
+                    const inputFiles = state.temp.input_files ?? [];
+                    for (let i = 0; i < this._options.fileDownloaders.length; i++) {
+                        const files = await this._options.fileDownloaders[i].downloadFiles(context, state);
+                        inputFiles.push(...files);
+                    }
+                    state.temp.input_files = inputFiles;
+                }
+
+                // Initialize {{$allOutputs}}
+                if (state.temp.actionOutputs == undefined) {
+                    state.temp.actionOutputs = {};
+                }
+
                 // Run any RouteSelectors in this._invokeRoutes first if the incoming Teams activity.type is "Invoke".
                 // Invoke Activities from Teams need to be responded to in less than 5 seconds.
                 if (context.activity.type === ActivityTypes.Invoke) {

js/packages/teams-ai/src/InputFileDownloader.ts

@@ -0,0 +1,43 @@
+/**
+ * @module teams-ai
+ */
+/**
+ * Copyright (c) Microsoft Corporation. All rights reserved.
+ * Licensed under the MIT License.
+ */
+
+import { TurnContext } from 'botbuilder';
+import { TurnState } from './TurnState';
+
+/**
+ * A plugin responsible for downloading files relative to the current users input.
+ * @template TState Optional. Type of application state.
+ */
+export interface InputFileDownloader<TState extends TurnState = TurnState> {
+    /**
+     * Download any files relative to the current users input.
+     * @param context Context for the current turn of conversation.
+     * @param state Application state for the current turn of conversation.
+     */
+    downloadFiles(context: TurnContext, state: TState): Promise<InputFile[]>;
+}
+
+/**
+ * A file sent by the user to the bot.
+ */
+export interface InputFile {
+    /**
+     * The downloaded content of the file.
+     */
+    content: Buffer;
+
+    /**
+     * The content type of the file.
+     */
+    contentType: string;
+
+    /**
+     * Optional. URL to the content of the file.
+     */
+    contentUrl?: string;
+}

js/packages/teams-ai/src/TeamsAttachmentDownloader.ts

@@ -0,0 +1,129 @@
+/**
+ * @module teams-ai
+ */
+/**
+ * Copyright (c) Microsoft Corporation. All rights reserved.
+ * Licensed under the MIT License.
+ */
+
+import axios, { AxiosInstance } from 'axios';
+import { Attachment, TurnContext } from 'botbuilder';
+import { TurnState } from './TurnState';
+import { InputFile, InputFileDownloader } from './InputFileDownloader';
+
+/**
+ * Options for the `TeamsAttachmentDownloader` class.
+ */
+export interface TeamsAttachmentDownloaderOptions {
+    /**
+     * The Microsoft App ID of the bot.
+     */
+    botAppId: string;
+
+    /**
+     * The Microsoft App Password of the bot.
+     */
+    botAppPassword: string;
+}
+
+/**
+ * Downloads attachments from Teams using the bots access token.
+ */
+export class TeamsAttachmentDownloader<TState extends TurnState = TurnState> implements InputFileDownloader<TState> {
+    private readonly _options: TeamsAttachmentDownloaderOptions;
+    private _httpClient: AxiosInstance;
+
+    /**
+     * Creates a new instance of the `TeamsAttachmentDownloader` class.
+     * @param options Options for the `TeamsAttachmentDownloader` class.
+     */
+    public constructor(options: TeamsAttachmentDownloaderOptions) {
+        this._options = options;
+        this._httpClient = axios.create();
+    }
+
+    /**
+     * Download any files relative to the current users input.
+     * @param context Context for the current turn of conversation.
+     * @param state Application state for the current turn of conversation.
+     */
+    public async downloadFiles(context: TurnContext, state: TState): Promise<InputFile[]> {
+        // Filter out HTML attachments
+        const attachments = context.activity.attachments?.filter((a) => !a.contentType.startsWith('text/html'));
+        if (!attachments || attachments.length === 0) {
+            return Promise.resolve([]);
+        }
+
+        // Download all attachments
+        const accessToken = await this.getAccessToken();
+        const files: InputFile[] = [];
+        for (const attachment of attachments) {
+            const file = await this.downloadFile(attachment, accessToken);
+            if (file) {
+                files.push(file);
+            }
+        }
+
+        return files;
+    }
+
+    /**
+     * @private
+     */
+    private async downloadFile(attachment: Attachment, accessToken: string): Promise<InputFile> {
+        if (attachment.contentUrl && attachment.contentUrl.startsWith('https://')) {
+            // Download file
+            const headers = {
+                'Authorization': `Bearer ${accessToken}`
+            };
+            const response = await this._httpClient.get(attachment.contentUrl, {
+                headers,
+                responseType: 'arraybuffer'
+            });
+
+            // Convert to a buffer
+            const content = Buffer.from(response.data, 'binary');
+
+            // Fixup content type
+            let contentType = attachment.contentType;
+            if (contentType === 'image/*') {
+                contentType = 'image/png';
+            }
+
+            // Return file
+            return {
+                content,
+                contentType,
+                contentUrl: attachment.contentUrl,
+            };
+        } else {
+            return {
+                content: Buffer.from(attachment.content),
+                contentType: attachment.contentType,
+                contentUrl: attachment.contentUrl,
+            };
+        }
+    }
+
+    /**
+     * @private
+     */
+    private async getAccessToken(): Promise<string> {
+        const headers = {
+            'Content-Type': 'application/x-www-form-urlencoded'
+        };
+        const body = `grant_type=client_credentials&client_id=${encodeURI(this._options.botAppId)}&client_secret=${encodeURI(this._options.botAppPassword)}&scope=https%3A%2F%2Fapi.botframework.com%2F.default`;
+        const token = await this._httpClient.post<JWTToken>('https://login.microsoftonline.com/botframework.com/oauth2/v2.0/token', body, { headers });
+        return token.data.access_token;
+    }
+}
+
+/**
+ * @private
+ */
+interface JWTToken {
+    token_type: string;
+    expires_in: number;
+    ext_expires_in: number;
+    access_token: string;
+}

js/packages/teams-ai/src/TurnState.ts

@@ -8,6 +8,7 @@
 
 import { TurnContext, Storage, StoreItems } from 'botbuilder';
 import { Memory } from './MemoryFork';
+import { InputFile } from './InputFileDownloader';
 
 /**
  * @private
@@ -50,14 +51,14 @@ export interface DefaultUserState {}
  */
 export interface DefaultTempState {
     /**
-     * Input passed to an AI prompt
+     * Input passed by the user to the AI Library
      */
     input: string;
 
     /**
-     * Formatted conversation history for embedding in an AI prompt
+     * Downloaded files passed by the user to the AI Library
      */
-    history: string;
+    input_files: InputFile[];
 
     /**
      * Output returned from the last executed action

js/packages/teams-ai/src/authentication/Authentication.spec.ts

@@ -48,7 +48,7 @@ describe('Authentication', () => {
         await state.load(context);
         state.temp = {
             input: '',
-            history: '',
+            input_files: [],
             lastOutput: '',
             actionOutputs: {},
             authTokens: {}
@@ -291,7 +291,7 @@ describe('AuthenticationManager', () => {
         await state.load(context);
         state.temp = {
             input: '',
-            history: '',
+            input_files: [],
             lastOutput: '',
             actionOutputs: {},
             authTokens: {}

js/packages/teams-ai/src/authentication/OAuthAdaptiveCardAuthentication.spec.ts

@@ -34,7 +34,7 @@ describe('AdaptiveCardAuthenticaion', () => {
         await state.load(context);
         state.temp = {
             input: '',
-            history: '',
+            input_files: [],
             lastOutput: '',
             actionOutputs: {},
             authTokens: {}

js/packages/teams-ai/src/authentication/OAuthBotAuthentication.spec.ts

@@ -39,7 +39,7 @@ describe('BotAuthentication', () => {
         await state.load(context);
         state.temp = {
             input: '',
-            history: '',
+            input_files: [],
             lastOutput: '',
             actionOutputs: {},
             authTokens: {}

js/packages/teams-ai/src/authentication/OAuthMessageExtensionAuthentication.spec.ts

@@ -33,7 +33,7 @@ describe('OAuthPromptMessageExtensionAuthentication', () => {
         await state.load(context);
         state.temp = {
             input: '',
-            history: '',
+            input_files: [],
             lastOutput: '',
             actionOutputs: {},
             authTokens: {}

js/packages/teams-ai/src/authentication/TeamsSsoBotAuthentication.spec.ts

@@ -41,7 +41,7 @@ describe('TeamsSsoBotAuthentication', () => {
         await state.load(context);
         state.temp = {
             input: '',
-            history: '',
+            input_files: [],
             lastOutput: '',
             actionOutputs: {},
             authTokens: {}

js/packages/teams-ai/src/authentication/TeamsSsoMessageExtensionAuthentication.spec.ts

@@ -46,7 +46,7 @@ describe('TeamsSsoMessageExtensionAuthentication', () => {
         await state.load(context);
         state.temp = {
             input: '',
-            history: '',
+            input_files: [],
             lastOutput: '',
             actionOutputs: {},
             authTokens: {}

js/packages/teams-ai/src/index.ts

@@ -19,9 +19,11 @@ export * from './validators';
 export * from './AdaptiveCards';
 export * from './AI';
 export * from './Application';
+export * from './InputFileDownloader';
 export * from './MemoryFork';
 export * from './MessageExtensions';
 export * from './TaskModules';
+export * from './TeamsAttachmentDownloader';
 export * from './TestTurnState';
 export * from './TurnState';
 export * from './Utilities';