Skip to content

Commit

Permalink
Merge pull request #41 from meta-d/develop
Browse files Browse the repository at this point in the history
Version 2.5 ChatBI
  • Loading branch information
meta-d authored Aug 5, 2024
2 parents 9978a5d + 47d6737 commit e415624
Show file tree
Hide file tree
Showing 313 changed files with 13,949 additions and 12,375 deletions.
2 changes: 1 addition & 1 deletion .deploy/api/package.json
Original file line number Diff line number Diff line change
@@ -1,7 +1,7 @@
{
"name": "ocap-server",
"author": "Metad",
"version": "2.4.7",
"version": "2.5.0",
"scripts": {
"start": "nx serve",
"build": "nx build",
Expand Down
15 changes: 8 additions & 7 deletions .deploy/webapp/package.json
Original file line number Diff line number Diff line change
@@ -1,6 +1,6 @@
{
"name": "ocap",
"version": "2.4.7",
"version": "2.5",
"description": "",
"license": "AGPL-3.0",
"homepage": "https://mtda.cloud",
Expand Down Expand Up @@ -45,10 +45,11 @@
"@casl/angular": "^6.0.0",
"@datorama/akita": "^6.2.3",
"@duckdb/duckdb-wasm": "1.25.0",
"@langchain/community": "0.2.11",
"@langchain/core": "0.2.7",
"@langchain/langgraph": "0.0.26",
"@langchain/openai": "0.1.3",
"@langchain/community": "0.2.20",
"@langchain/core": "0.2.18",
"@langchain/langgraph": "0.0.31",
"@langchain/ollama": "^0.0.2",
"@langchain/openai": "0.2.4",
"@microsoft/fetch-event-source": "^2.0.1",
"@ng-matero/extensions": "^13.1.0",
"@ng-web-apis/common": "^2.0.1",
Expand All @@ -71,7 +72,6 @@
"@sentry/tracing": "^7.38.0",
"@swc/helpers": "0.5.3",
"@tinymce/tinymce-angular": "^6.0.1",
"ai": "^2.2.30",
"angular-gridster2": "^14.0.1",
"apache-arrow": "^9.0.0",
"axios": "1.6.8",
Expand All @@ -91,7 +91,7 @@
"immer": "^10.0.1",
"js-yaml": "^4.1.0",
"json5": "^2.2.3",
"langchain": "0.2.5",
"langchain": "0.2.10",
"lato-font": "^3.0.0",
"lodash": "^4.17.21",
"lodash-es": "^4.17.21",
Expand All @@ -108,6 +108,7 @@
"ngx-quill": "^16.1.2",
"ngxtension": "^3.1.2",
"noto-serif-sc": "^8.0.0",
"ollama": "^0.5.6",
"openai": "^4.6.0",
"prismjs": "^1.29.0",
"quill": "^1.3.7",
Expand Down
10 changes: 5 additions & 5 deletions apps/cloud/src/app/@core/copilot/agent-route.ts
Original file line number Diff line number Diff line change
@@ -1,13 +1,13 @@
import { BaseMessage, HumanMessage } from '@langchain/core/messages'
import { BaseChatModel } from '@langchain/core/language_models/chat_models'
import { HumanMessage } from '@langchain/core/messages'
import { ChatPromptTemplate, MessagesPlaceholder, TemplateFormat } from '@langchain/core/prompts'
import { Runnable, RunnableConfig } from '@langchain/core/runnables'
import { StructuredToolInterface } from '@langchain/core/tools'
import { PartialValues } from '@langchain/core/utils/types'
import { Runnable, RunnableConfig } from '@langchain/core/runnables'
import { ChatOpenAI } from '@langchain/openai'
import { createCopilotAgentState } from '@metad/copilot'
import { AgentState } from '@metad/copilot-angular'
import { AgentExecutor, createOpenAIToolsAgent } from 'langchain/agents'
import { z } from 'zod'
import { createCopilotAgentState } from '@metad/copilot'

type ZodAny = z.ZodObject<any, any, any, any>

Expand Down Expand Up @@ -48,7 +48,7 @@ export function createState() {
* @returns
*/
export async function createWorkerAgent<NewPartialVariableName extends string>(
llm: ChatOpenAI,
llm: BaseChatModel,
tools: StructuredToolInterface<ZodAny>[],
systemPrompt: string,
partialValues?: PartialValues<NewPartialVariableName>,
Expand Down
60 changes: 34 additions & 26 deletions apps/cloud/src/app/@core/copilot/example-vector-retriever.ts
Original file line number Diff line number Diff line change
Expand Up @@ -8,9 +8,9 @@ import {
VectorStoreRetrieverInterface,
VectorStoreRetrieverMMRSearchKwargs
} from '@langchain/core/vectorstores'
import { CopilotExampleService } from '../services/'
import { catchError, firstValueFrom, of, timeout } from 'rxjs'
import { SERVER_REQUEST_TIMEOUT } from '../config'
import { CopilotExampleService } from '../services/'

/**
* Type for options when adding a document to the VectorStore.
Expand All @@ -22,7 +22,7 @@ type AddDocumentOptions = Record<string, any>
* Class for performing document retrieval from a VectorStore. Can perform
* similarity search or maximal marginal relevance search.
*/
export class VectorStoreRetriever<V extends VectorStoreInterface = VectorStoreInterface>
export class ExampleVectorStoreRetriever<V extends VectorStoreInterface = VectorStoreInterface>
extends BaseRetriever
implements VectorStoreRetrieverInterface
{
Expand Down Expand Up @@ -78,32 +78,40 @@ export class VectorStoreRetriever<V extends VectorStoreInterface = VectorStoreIn
`The vector store backing this retriever, ${this._vectorstoreType()} does not support max marginal relevance search.`
)
}
return await firstValueFrom(this.service.maxMarginalRelevanceSearch(query, {
k: this.k,
filter: this.filter,
command: this.command,
role: this.role(),
...this.searchKwargs
}).pipe(
timeout(SERVER_REQUEST_TIMEOUT),
catchError((error) => {
return of([])
})
))
return await firstValueFrom(
this.service
.maxMarginalRelevanceSearch(query, {
k: this.k,
filter: this.filter,
command: this.command,
role: this.role(),
...this.searchKwargs
})
.pipe(
timeout(SERVER_REQUEST_TIMEOUT),
catchError((error) => {
return of([])
})
)
)
}

return await firstValueFrom(this.service.similaritySearch(query, {
command: this.command,
k: this.k,
filter: this.filter,
role: this.role(),
score: this.score
}).pipe(
timeout(SERVER_REQUEST_TIMEOUT),
catchError((error) => {
return of([])
})
))
return await firstValueFrom(
this.service
.similaritySearch(query, {
command: this.command,
k: this.k,
filter: this.filter,
role: this.role(),
score: this.score
})
.pipe(
timeout(SERVER_REQUEST_TIMEOUT),
catchError((error) => {
return of([])
})
)
)
}

async addDocuments(documents: DocumentInterface[], options?: AddDocumentOptions): Promise<string[] | void> {
Expand Down
14 changes: 9 additions & 5 deletions apps/cloud/src/app/@core/copilot/few-shot.ts
Original file line number Diff line number Diff line change
@@ -1,9 +1,13 @@
import { inject, Provider } from '@angular/core'
import { SemanticSimilarityExampleSelector } from '@langchain/core/example_selectors'
import { FewShotPromptTemplate, PromptTemplate } from '@langchain/core/prompts'
import { ExampleVectorStoreRetrieverInput, NgmCommandFewShotPromptToken, NgmCopilotService } from '@metad/copilot-angular'
import {
ExampleVectorStoreRetrieverInput,
NgmCommandFewShotPromptToken,
NgmCopilotService
} from '@metad/copilot-angular'
import { CopilotExampleService } from '../services/copilot-example.service'
import { VectorStoreRetriever } from './example-vector-retriever'
import { ExampleVectorStoreRetriever } from './example-vector-retriever'

export function injectAgentFewShotTemplate(command: string, fields?: ExampleVectorStoreRetrieverInput) {
const copilotService = inject(NgmCopilotService)
Expand All @@ -19,15 +23,15 @@ function createExampleFewShotPrompt(
fields?: ExampleVectorStoreRetrieverInput
) {
const examplePrompt = PromptTemplate.fromTemplate(
`Question: {{input}}
Answer: {{output}}`,
`Question: {{input}}
Answer: {{output}}`,
{
templateFormat: 'mustache'
}
)
return new FewShotPromptTemplate({
exampleSelector: new SemanticSimilarityExampleSelector({
vectorStoreRetriever: new VectorStoreRetriever(
vectorStoreRetriever: new ExampleVectorStoreRetriever(
{
...(fields ?? { vectorStore: null }),
vectorStore: null,
Expand Down
1 change: 1 addition & 0 deletions apps/cloud/src/app/@core/copilot/index.ts
Original file line number Diff line number Diff line change
Expand Up @@ -2,5 +2,6 @@ export * from './dimension-member-retriever'
export * from './example-vector-retriever'
export * from './few-shot'
export * from './checkpoint-saver'
export * from './references'
export * as Route from './agent-route'
export * as Plan from './agent-plan'
19 changes: 19 additions & 0 deletions apps/cloud/src/app/@core/copilot/references.ts
Original file line number Diff line number Diff line change
@@ -0,0 +1,19 @@
import { inject } from '@angular/core'
import { ExampleVectorStoreRetrieverInput, NgmCopilotService } from '@metad/copilot-angular'
import { CopilotExampleService } from '../services/copilot-example.service'
import { ExampleVectorStoreRetriever } from './example-vector-retriever'

export function injectExampleRetriever(command: string, fields?: ExampleVectorStoreRetrieverInput) {
const copilotService = inject(NgmCopilotService)
const copilotExampleService = inject(CopilotExampleService)

return new ExampleVectorStoreRetriever(
{
...(fields ?? { vectorStore: null }),
vectorStore: null,
command,
role: copilotService.role
},
copilotExampleService
)
}
79 changes: 79 additions & 0 deletions apps/cloud/src/app/@core/services/chatbi-conversation.service.ts
Original file line number Diff line number Diff line change
@@ -0,0 +1,79 @@
import { HttpClient } from '@angular/common/http'
import { inject, Injectable } from '@angular/core'
import { API_PREFIX, SystemPrivacyFields } from '@metad/cloud/state'
import { CopilotChatMessage } from '@metad/copilot'
import { Indicator } from '@metad/ocap-core'
import { omit } from 'lodash-es'
import { NGXLogger } from 'ngx-logger'
import { BehaviorSubject, map } from 'rxjs'
import { IChatBIConversation } from '../types'

const API_CHATBI_CONVERSATION = API_PREFIX + '/chatbi-conversation'

export interface ChatbiConverstion {
id?: string
key: string
name: string
modelId: string
dataSource: string
entity: string
command: string
messages: CopilotChatMessage[]
indicators: Indicator[]
}

@Injectable({ providedIn: 'root' })
export class ChatBIConversationService {
readonly #logger = inject(NGXLogger)
readonly httpClient = inject(HttpClient)

readonly #refresh = new BehaviorSubject<void>(null)

getMy() {
return this.httpClient
.get<{ items: IChatBIConversation[] }>(API_CHATBI_CONVERSATION + '/my')
.pipe(map(({ items }) => items.map(convertChatBIConversationResult)))
}

getById(id: string) {
return this.httpClient
.get<IChatBIConversation>(API_CHATBI_CONVERSATION + '/' + id)
.pipe(map(convertChatBIConversationResult))
}

upsert(entity: Partial<ChatbiConverstion>) {
return entity.id
? this.httpClient
.put<ChatbiConverstion>(`${API_CHATBI_CONVERSATION}/${entity.id}`, convertChatBIConversation(entity))
.pipe(map(() => entity as ChatbiConverstion))
: this.httpClient
.post<ChatbiConverstion>(API_CHATBI_CONVERSATION, convertChatBIConversation(entity))
.pipe(map((result) => convertChatBIConversationResult(result)))
}

delete(id: string) {
return this.httpClient.delete(`${API_CHATBI_CONVERSATION}/${id}`)
}

refresh() {
this.#refresh.next()
}
}

export function convertChatBIConversation(input: Partial<ChatbiConverstion>) {
return {
...omit(input, 'messages', 'indicators'),
options: {
messages: input.messages,
indicators: input.indicators,
}
} as IChatBIConversation
}

export function convertChatBIConversationResult(result: IChatBIConversation) {
return {
...omit(result, 'options', ...SystemPrivacyFields),
messages: result.options?.messages || [],
indicators: result.options?.indicators
} as ChatbiConverstion
}
Loading

0 comments on commit e415624

Please sign in to comment.