Add onedrive integration

StrongMonkey · StrongMonkey · commit 0fec3f802e46 · 2024-09-06T02:48:41.000-07:00
Signed-off-by: Daishan Peng &lt;daishan@acorn.io&gt;
diff --git a/actions/knowledge/filehelper.ts b/actions/knowledge/filehelper.ts
@@ -29,13 +29,9 @@ export async function getFileOrFolderSizeInKB(
   return 0;
 }
 
-export async function getBasename(filePath: string): Promise<string> {
-  return path.basename(filePath);
-}
-
 export async function importFiles(
   files: string[],
-  type: 'local' | 'notion'
+  type: 'local' | 'notion' | 'onedrive'
 ): Promise<Map<string, FileDetail>> {
   const result: Map<string, FileDetail> = new Map();
 
diff --git a/actions/knowledge/knowledge.ts b/actions/knowledge/knowledge.ts
@@ -65,7 +65,7 @@ export async function ensureFiles(
     if (!fs.existsSync(filePath)) {
       if (file[1].type === 'local') {
         await fs.promises.copyFile(file[0], filePath);
-      } else if (file[1].type === 'notion') {
+      } else if (file[1].type === 'notion' || file[1].type === 'onedrive') {
         if (
           fs.existsSync(filePath) &&
           fs.lstatSync(filePath).isSymbolicLink()
@@ -78,7 +78,7 @@ export async function ensureFiles(
   }
 
   if (!updateOnly) {
-    for (const type of ['local', 'notion']) {
+    for (const type of ['local', 'notion', 'onedrive']) {
       if (!fs.existsSync(path.join(dir, type))) {
         continue;
       }
@@ -124,7 +124,7 @@ export async function getFiles(
   if (!fs.existsSync(dir)) {
     return result;
   }
-  for (const type of ['local', 'notion']) {
+  for (const type of ['local', 'notion', 'onedrive']) {
     if (!fs.existsSync(path.join(dir, type))) {
       continue;
     }
@@ -135,7 +135,7 @@ export async function getFiles(
         filePath = await fs.promises.readlink(filePath);
       }
       result.set(filePath, {
-        type: type as 'local' | 'notion',
+        type: type as any,
         fileName: file,
         size: await getFileOrFolderSizeInKB(path.join(dir, type, file)),
       });
diff --git a/actions/knowledge/notion.ts b/actions/knowledge/notion.ts
@@ -3,12 +3,7 @@
 import fs from 'fs';
 import path from 'path';
 import { WORKSPACE_DIR } from '@/config/env';
-import {
-  GPTScript,
-  PromptFrame,
-  Run,
-  RunEventType,
-} from '@gptscript-ai/gptscript';
+import { runSyncTool } from '@/actions/knowledge/tool';
 
 export async function isNotionConfigured() {
   return fs.existsSync(
@@ -22,37 +17,15 @@ export async function isNotionConfigured() {
   );
 }
 
-function readFilesRecursive(dir: string): string[] {
-  let results: string[] = [];
-
-  const list = fs.readdirSync(dir);
-  list.forEach((file) => {
-    if (file === 'metadata.json') return;
-    const filePath = path.join(dir, file);
-    const stat = fs.statSync(filePath);
-
-    if (stat && stat.isDirectory()) {
-      // Recursively read the directory
-      results = results.concat(readFilesRecursive(filePath));
-    } else {
-      // Add the file path to the results
-      results.push(filePath);
-    }
-  });
-
-  return results;
-}
-
 export async function getNotionFiles(): Promise<
   Map<string, { url: string; fileName: string }>
 > {
   const dir = path.join(WORKSPACE_DIR(), 'knowledge', 'integrations', 'notion');
-  const filePaths = readFilesRecursive(dir);
   const metadataFromFiles = fs.readFileSync(path.join(dir, 'metadata.json'));
   const metadata = JSON.parse(metadataFromFiles.toString());
   const result = new Map<string, { url: string; fileName: string }>();
-  for (const filePath of filePaths) {
-    const pageID = path.basename(path.dirname(filePath));
+  for (const pageID in metadata) {
+    const filePath = path.join(dir, pageID, metadata[pageID].filename);
     result.set(filePath, {
       url: metadata[pageID].url,
       fileName: path.basename(filePath),
@@ -63,28 +36,5 @@ export async function getNotionFiles(): Promise<
 }
 
 export async function runNotionSync(authed: boolean): Promise<void> {
-  const gptscript = new GPTScript({
-    DefaultModelProvider: 'github.com/gptscript-ai/gateway-provider',
-  });
-
-  const runningTool = await gptscript.run(
-    'github.com/gptscript-ai/knowledge-notion-integration',
-    {
-      prompt: true,
-    }
-  );
-  if (!authed) {
-    const handlePromptEvent = (runningTool: Run) => {
-      return new Promise<string>((resolve) => {
-        runningTool.on(RunEventType.Prompt, (data: PromptFrame) => {
-          resolve(data.id);
-        });
-      });
-    };
-
-    const id = await handlePromptEvent(runningTool);
-    await gptscript.promptResponse({ id, responses: {} });
-  }
-  await runningTool.text();
-  return;
+  return runSyncTool(authed, 'notion');
 }
diff --git a/actions/knowledge/onedrive.ts b/actions/knowledge/onedrive.ts
@@ -0,0 +1,45 @@
+'use server';
+import fs from 'fs';
+import path from 'path';
+import { WORKSPACE_DIR } from '@/config/env';
+import { runSyncTool } from '@/actions/knowledge/tool';
+
+export async function isOneDriveConfigured() {
+  return fs.existsSync(
+    path.join(
+      WORKSPACE_DIR(),
+      'knowledge',
+      'integrations',
+      'onedrive',
+      'metadata.json'
+    )
+  );
+}
+
+export async function getOneDriveFiles(): Promise<
+  Map<string, { url: string; fileName: string }>
+> {
+  const dir = path.join(
+    WORKSPACE_DIR(),
+    'knowledge',
+    'integrations',
+    'onedrive'
+  );
+  const metadataFromFiles = fs.readFileSync(path.join(dir, 'metadata.json'));
+  const metadata = JSON.parse(metadataFromFiles.toString());
+  const result = new Map<string, { url: string; fileName: string }>();
+  for (const documentID in metadata) {
+    result.set(path.join(dir, documentID, metadata[documentID].fileName), {
+      url: metadata[documentID].url,
+      fileName: metadata[documentID].fileName,
+    });
+  }
+  return result;
+}
+
+// syncFiles syncs all files only when they are selected
+// todo: we can stop syncing once file is no longer used by any other script
+
+export async function runOneDriveSync(authed: boolean): Promise<void> {
+  return runSyncTool(authed, 'onedrive');
+}
diff --git a/actions/knowledge/tool.ts b/actions/knowledge/tool.ts
@@ -0,0 +1,62 @@
+'use server';
+
+import {
+  GPTScript,
+  PromptFrame,
+  Run,
+  RunEventType,
+} from '@gptscript-ai/gptscript';
+import path from 'path';
+import { WORKSPACE_DIR } from '@/config/env';
+import fs from 'fs';
+
+export async function runSyncTool(
+  authed: boolean,
+  tool: 'notion' | 'onedrive'
+): Promise<void> {
+  const gptscript = new GPTScript({
+    DefaultModelProvider: 'github.com/gptscript-ai/gateway-provider',
+  });
+
+  let toolUrl = '';
+  if (tool === 'notion') {
+    toolUrl = 'github.com/gptscript-ai/knowledge-notion-integration@46a273e';
+  } else if (tool === 'onedrive') {
+    toolUrl = 'github.com/gptscript-ai/knowledge-onedrive-integration@a85a498';
+  }
+  const runningTool = await gptscript.run(toolUrl, {
+    prompt: true,
+  });
+  if (!authed) {
+    const handlePromptEvent = (runningTool: Run) => {
+      return new Promise<string>((resolve) => {
+        runningTool.on(RunEventType.Prompt, (data: PromptFrame) => {
+          resolve(data.id);
+        });
+      });
+    };
+
+    const id = await handlePromptEvent(runningTool);
+    await gptscript.promptResponse({ id, responses: {} });
+  }
+  await runningTool.text();
+  return;
+}
+
+export async function syncFiles(
+  selectedFiles: string[],
+  type: 'notion' | 'onedrive'
+): Promise<void> {
+  const dir = path.join(WORKSPACE_DIR(), 'knowledge', 'integrations', type);
+  const metadataFromFiles = fs.readFileSync(path.join(dir, 'metadata.json'));
+  const metadata = JSON.parse(metadataFromFiles.toString());
+  for (const file of selectedFiles) {
+    const documentID = path.basename(path.dirname(file));
+    const detail = metadata[documentID];
+    detail.sync = true;
+    metadata[documentID] = detail;
+  }
+  fs.writeFileSync(path.join(dir, 'metadata.json'), JSON.stringify(metadata));
+  await runSyncTool(true, type);
+  return;
+}
diff --git a/actions/knowledge/util.ts b/actions/knowledge/util.ts
@@ -1,7 +1,7 @@
 export interface FileDetail {
   fileName: string;
   size: number;
-  type: 'local' | 'notion';
+  type: 'local' | 'notion' | 'onedrive';
 }
 
 export function gatewayTool(): string {
diff --git a/components/edit/configure.tsx b/components/edit/configure.tsx
@@ -31,6 +31,7 @@ import { RiFoldersLine } from 'react-icons/ri';
 import FileModal from '@/components/knowledge/FileModal';
 import { gatewayTool } from '@/actions/knowledge/util';
 import { importFiles } from '@/actions/knowledge/filehelper';
+import { DiOnedrive } from 'react-icons/di';
 
 interface ConfigureProps {
   collapsed?: boolean;
@@ -191,14 +192,17 @@ const Configure: React.FC<ConfigureProps> = ({ collapsed }) => {
                     ([key, fileDetail], _) => (
                       <div key={key} className="flex space-x-2">
                         <div className="flex flex-row w-full border-2 justify-between truncate dark:border-zinc-700 text-sm pl-2 rounded-lg">
-                          <div className="flex items-center">
+                          <div className="flex items-center overflow-auto">
                             {fileDetail.type === 'local' && (
                               <RiFileSearchLine className="justify-start mr-2" />
                             )}
                             {fileDetail.type === 'notion' && (
                               <RiNotionFill className="justify-start mr-2" />
                             )}
-                            <div className="flex flex-row justify-start overflow-x-auto">
+                            {fileDetail.type === 'onedrive' && (
+                              <DiOnedrive className="justify-start mr-2" />
+                            )}
+                            <div className="flex flex-row justify-start overflow-auto">
                               <p className="capitalize text-left">
                                 {fileDetail.fileName}
                               </p>
diff --git a/components/knowledge/FileModal.tsx b/components/knowledge/FileModal.tsx
@@ -1,15 +1,20 @@
 import {
+  Avatar,
   Button,
   Modal,
   ModalBody,
   ModalContent,
   useDisclosure,
 } from '@nextui-org/react';
-import { Image } from '@nextui-org/image';
 import { BiPlus } from 'react-icons/bi';
 import { NotionFileModal } from '@/components/knowledge/Notion';
 import { isNotionConfigured, runNotionSync } from '@/actions/knowledge/notion';
 import { useState } from 'react';
+import { OnedriveFileModal } from '@/components/knowledge/OneDrive';
+import {
+  isOneDriveConfigured,
+  runOneDriveSync,
+} from '@/actions/knowledge/onedrive';
 
 interface FileModalProps {
   isOpen: boolean;
@@ -19,9 +24,12 @@ interface FileModalProps {
 
 const FileModal = ({ isOpen, onClose, handleAddFile }: FileModalProps) => {
   const notionModal = useDisclosure();
+  const onedriveModal = useDisclosure();
   const [isSyncing, setIsSyncing] = useState(false);
   const [notionConfigured, setNotionConfigured] = useState(false);
   const [notionSyncError, setNotionSyncError] = useState('');
+  const [oneDriveConfigured, setOneDriveConfigured] = useState(false);
+  const [oneDriveSyncError, setOneDriveSyncError] = useState('');
 
   const onClickNotion = async () => {
     onClose();
@@ -40,6 +48,23 @@ const FileModal = ({ isOpen, onClose, handleAddFile }: FileModalProps) => {
     }
   };
 
+  const onClickOnedrive = async () => {
+    onClose();
+    onedriveModal.onOpen();
+    const isConfigured = await isOneDriveConfigured();
+    if (!isConfigured) {
+      setIsSyncing(true);
+      try {
+        await runOneDriveSync(false);
+        setOneDriveConfigured(true);
+      } catch (e) {
+        setOneDriveSyncError((e as Error).toString());
+      } finally {
+        setIsSyncing(false);
+      }
+    }
+  };
+
   return (
     <>
       <Modal
@@ -68,11 +93,30 @@ const FileModal = ({ isOpen, onClose, handleAddFile }: FileModalProps) => {
               onClick={onClickNotion}
               className="flex w-full items-center justify-center gap-3 rounded-md bg-white px-3 py-2 text-sm font-semibold text-gray-900 shadow-sm ring-1 ring-inset ring-gray-300 hover:bg-gray-50 focus-visible:ring-transparent hover:cursor-pointer"
             >
-              <Image className="h-5 w-5" src="notion.svg" alt="Notion Icon" />
+              <Avatar
+                size="sm"
+                src="notion.svg"
+                alt="Notion Icon"
+                classNames={{ base: 'p-1.5 bg-white' }}
+              />
               <span className="text-sm font-semibold leading-6">
                 Sync From Notion
               </span>
             </Button>
+            <Button
+              onClick={onClickOnedrive}
+              className="flex w-full items-center justify-center gap-3 rounded-md bg-white px-3 py-2 text-sm font-semibold text-gray-900 shadow-sm ring-1 ring-inset ring-gray-300 hover:bg-gray-50 focus-visible:ring-transparent hover:cursor-pointer"
+            >
+              <Avatar
+                size="sm"
+                src="onedrive.svg"
+                alt="OneDrive Icon"
+                classNames={{ base: 'p-1.5 bg-white' }}
+              />
+              <span className="text-sm font-semibold leading-6">
+                Sync From OneDrive
+              </span>
+            </Button>
           </ModalBody>
         </ModalContent>
       </Modal>
@@ -86,6 +130,16 @@ const FileModal = ({ isOpen, onClose, handleAddFile }: FileModalProps) => {
         syncError={notionSyncError}
         setSyncError={setNotionSyncError}
       />
+      <OnedriveFileModal
+        isOpen={onedriveModal.isOpen}
+        onClose={onedriveModal.onClose}
+        isSyncing={isSyncing}
+        setIsSyncing={setIsSyncing}
+        onedriveConfigured={oneDriveConfigured}
+        setOnedriveConfigured={setOneDriveConfigured}
+        syncError={oneDriveSyncError}
+        setSyncError={setOneDriveSyncError}
+      />
     </>
   );
 };
diff --git a/components/knowledge/Notion.tsx b/components/knowledge/Notion.tsx
diff --git a/components/knowledge/OneDrive.tsx b/components/knowledge/OneDrive.tsx
diff --git a/public/onedrive.svg b/public/onedrive.svg

Original file line number	Diff line number	Diff line change
`@@ -1,7 +1,7 @@`
`1`	`1`	`export interface FileDetail {`
`2`	`2`	`fileName: string;`
`3`	`3`	`size: number;`
`4`		`- type: 'local' \| 'notion';`
	`4`	`+ type: 'local' \| 'notion' \| 'onedrive';`
`5`	`5`	`}`
`6`	`6`
`7`	`7`	`export function gatewayTool(): string {`