-
Notifications
You must be signed in to change notification settings - Fork 4.5k
Commit
This commit does not belong to any branch on this repository, and may belong to a fork outside of the repository.
- Loading branch information
Showing
1 changed file
with
84 additions
and
130 deletions.
There are no files selected for viewing
This file contains bidirectional Unicode text that may be interpreted or compiled differently than what appears below. To review, open the file in an editor that reveals hidden Unicode characters.
Learn more about bidirectional Unicode characters
| Original file line number | Diff line number | Diff line change |
|---|---|---|
| @@ -1,146 +1,100 @@ | ||
| import { IAgentRuntime, Plugin, logger } from "@elizaos/core"; | ||
| import { IAgentRuntime, Plugin, logger, ModelClass } from "@elizaos/core"; | ||
| import { Readable } from "node:stream"; | ||
| import { ModelClass } from "@elizaos/core"; | ||
| import { prependWavHeader } from "./utils.ts"; | ||
|
|
||
| function getVoiceSettings(runtime: IAgentRuntime) { | ||
| const getSetting = (key: string, fallback = "") => | ||
| process.env[key] || runtime.getSetting(key) || fallback; | ||
|
|
||
| return { | ||
| elevenlabsApiKey: | ||
| process.env.ELEVENLABS_XI_API_KEY || | ||
| runtime.getSetting("ELEVENLABS_XI_API_KEY"), | ||
| elevenlabsVoiceId: | ||
| process.env.ELEVENLABS_VOICE_ID || | ||
| runtime.getSetting("ELEVENLABS_VOICE_ID"), | ||
| elevenlabsModel: | ||
| process.env.ELEVENLABS_MODEL_ID || | ||
| runtime.getSetting("ELEVENLABS_MODEL_ID") || | ||
| "eleven_monolingual_v1", | ||
| elevenlabsStability: | ||
| process.env.ELEVENLABS_VOICE_STABILITY || | ||
| runtime.getSetting("ELEVENLABS_VOICE_STABILITY") || | ||
| "0.5", | ||
| elevenStreamingLatency: | ||
| process.env.ELEVENLABS_OPTIMIZE_STREAMING_LATENCY || | ||
| runtime.getSetting("ELEVENLABS_OPTIMIZE_STREAMING_LATENCY") || | ||
| "0", | ||
| elevenlabsOutputFormat: | ||
| process.env.ELEVENLABS_OUTPUT_FORMAT || | ||
| runtime.getSetting("ELEVENLABS_OUTPUT_FORMAT") || | ||
| "pcm_16000", | ||
| elevenlabsVoiceSimilarity: | ||
| process.env.ELEVENLABS_VOICE_SIMILARITY_BOOST || | ||
| runtime.getSetting("ELEVENLABS_VOICE_SIMILARITY_BOOST") || | ||
| "0.75", | ||
| elevenlabsVoiceStyle: | ||
| process.env.ELEVENLABS_VOICE_STYLE || | ||
| runtime.getSetting("ELEVENLABS_VOICE_STYLE") || | ||
| "0", | ||
| elevenlabsVoiceUseSpeakerBoost: | ||
| process.env.ELEVENLABS_VOICE_USE_SPEAKER_BOOST || | ||
| runtime.getSetting("ELEVENLABS_VOICE_USE_SPEAKER_BOOST") || | ||
| "true", | ||
| apiKey: getSetting("ELEVENLABS_XI_API_KEY"), | ||
| voiceId: getSetting("ELEVENLABS_VOICE_ID"), | ||
| model: getSetting("ELEVENLABS_MODEL_ID", "eleven_monolingual_v1"), | ||
| stability: getSetting("ELEVENLABS_VOICE_STABILITY", "0.5"), | ||
| latency: getSetting("ELEVENLABS_OPTIMIZE_STREAMING_LATENCY", "0"), | ||
| outputFormat: getSetting("ELEVENLABS_OUTPUT_FORMAT", "pcm_16000"), | ||
| similarity: getSetting("ELEVENLABS_VOICE_SIMILARITY_BOOST", "0.75"), | ||
| style: getSetting("ELEVENLABS_VOICE_STYLE", "0"), | ||
| speakerBoost: getSetting("ELEVENLABS_VOICE_USE_SPEAKER_BOOST", "true"), | ||
| }; | ||
| } | ||
|
|
||
| async function fetchSpeech(runtime: IAgentRuntime, text: string) { | ||
| const settings = getVoiceSettings(runtime); | ||
| try { | ||
| const response = await fetch( | ||
| `https://api.elevenlabs.io/v1/text-to-speech/${settings.voiceId}/stream?optimize_streaming_latency=${settings.latency}&output_format=${settings.outputFormat}`, | ||
| { | ||
| method: "POST", | ||
| headers: { | ||
| "Content-Type": "application/json", | ||
| "xi-api-key": settings.apiKey, | ||
| }, | ||
| body: JSON.stringify({ | ||
| model_id: settings.model, | ||
| text, | ||
| voice_settings: { | ||
| similarity_boost: settings.similarity, | ||
| stability: settings.stability, | ||
| style: settings.style, | ||
| use_speaker_boost: settings.speakerBoost, | ||
| }, | ||
| }), | ||
| } | ||
| ); | ||
| if (response.status !== 200) { | ||
| const errorBodyString = await response.text(); | ||
| const errorBody = JSON.parse(errorBodyString); | ||
|
|
||
| if (response.status === 401 && errorBody.detail?.status === "quota_exceeded") { | ||
| logger.log("ElevenLabs quota exceeded"); | ||
| throw new Error("QUOTA_EXCEEDED"); | ||
| } | ||
| throw new Error(`Received status ${response.status} from Eleven Labs API: ${JSON.stringify(errorBody)}`); | ||
| } | ||
| return Readable.fromWeb(response.body); | ||
| } catch (error) { | ||
| logger.error(error); | ||
| return new Readable({ read() {} }); | ||
| } | ||
| } | ||
|
|
||
| export const elevenLabsPlugin: Plugin = { | ||
| name: "elevenLabs", | ||
| description: "ElevenLabs plugin", | ||
|
|
||
| models: { | ||
| [ModelClass.TEXT_TO_SPEECH]: async ( | ||
| runtime: IAgentRuntime, | ||
| text: string | null | ||
| ) => { | ||
| const { | ||
| elevenlabsApiKey, | ||
| elevenlabsVoiceId, | ||
| elevenlabsModel, | ||
| elevenlabsStability, | ||
| elevenStreamingLatency, | ||
| elevenlabsOutputFormat, | ||
| elevenlabsVoiceSimilarity, | ||
| elevenlabsVoiceStyle, | ||
| elevenlabsVoiceUseSpeakerBoost, | ||
| } = getVoiceSettings(runtime); | ||
|
|
||
| try { | ||
| const response = await fetch( | ||
| `https://api.elevenlabs.io/v1/text-to-speech/${elevenlabsVoiceId}/stream?optimize_streaming_latency=${elevenStreamingLatency}&output_format=${elevenlabsOutputFormat}`, | ||
| { | ||
| method: "POST", | ||
| headers: { | ||
| "Content-Type": "application/json", | ||
| "xi-api-key": elevenlabsApiKey, | ||
| }, | ||
| body: JSON.stringify({ | ||
| model_id: elevenlabsModel, | ||
| text: text, | ||
| voice_settings: { | ||
| similarity_boost: elevenlabsVoiceSimilarity, | ||
| stability: elevenlabsStability, | ||
| style: elevenlabsVoiceStyle, | ||
| use_speaker_boost: elevenlabsVoiceUseSpeakerBoost, | ||
| }, | ||
| }), | ||
| } | ||
| ); | ||
|
|
||
| const status = response.status; | ||
| if (status !== 200) { | ||
| const errorBodyString = await response.text(); | ||
| const errorBody = JSON.parse(errorBodyString); | ||
|
|
||
| // Check for quota exceeded error | ||
| if (status === 401 && errorBody.detail?.status === "quota_exceeded") { | ||
| logger.log("ElevenLabs quota exceeded"); | ||
| throw new Error("QUOTA_EXCEEDED"); | ||
| } | ||
|
|
||
| throw new Error( | ||
| `Received status ${status} from Eleven Labs API: ${errorBodyString}` | ||
| ); | ||
| } | ||
|
|
||
| if (response) { | ||
| const webStream = ReadableStream.from( | ||
| response.body as ReadableStream | ||
| ); | ||
| const reader = webStream.getReader(); | ||
|
|
||
| const readable = new Readable({ | ||
| read() { | ||
| reader.read().then(({ done, value }) => { | ||
| if (done) { | ||
| this.push(null); | ||
| } else { | ||
| this.push(value); | ||
| } | ||
| }); | ||
| }, | ||
| }); | ||
|
|
||
| if (elevenlabsOutputFormat.startsWith("pcm_")) { | ||
| const sampleRate = Number.parseInt( | ||
| elevenlabsOutputFormat.substring(4) | ||
| ); | ||
| const withHeader = prependWavHeader( | ||
| readable, | ||
| 1024 * 1024 * 100, | ||
| sampleRate, | ||
| 1, | ||
| 16 | ||
| ); | ||
| return withHeader; | ||
| } else { | ||
| return readable; | ||
| } | ||
| } else { | ||
| return new Readable({ | ||
| read() {}, | ||
| }); | ||
| } | ||
| } catch (error) {} | ||
| [ModelClass.TEXT_TO_SPEECH]: async (runtime, text) => { | ||
| const stream = await fetchSpeech(runtime, text); | ||
| return getVoiceSettings(runtime).outputFormat.startsWith("pcm_") | ||
| ? prependWavHeader(stream, 1024 * 1024 * 100, parseInt(getVoiceSettings(runtime).outputFormat.slice(4)), 1, 16) | ||
| : stream; | ||
| }, | ||
| }, | ||
| tests: [ | ||
| { | ||
| name: "test eleven labs", | ||
| tests: [ | ||
| { | ||
| name: "Eleven Labs API key validation", | ||
| fn: async (runtime: IAgentRuntime) => { | ||
| if (!getVoiceSettings(runtime).apiKey) { | ||
| throw new Error("Missing API key: Please provide a valid Eleven Labs API key."); | ||
| } | ||
| }, | ||
| }, | ||
| { | ||
| name: "Eleven Labs API response", | ||
| fn: async (runtime: IAgentRuntime) => { | ||
| try { | ||
| await fetchSpeech(runtime, "test"); | ||
| } catch(error) { | ||
| throw new Error(`Failed to fetch speech from Eleven Labs API: ${error.message || "Unknown error occurred"}`); | ||
| } | ||
|
|
||
| }, | ||
| }, | ||
| ], | ||
| }, | ||
| ], | ||
| }; | ||
| export default elevenLabsPlugin; |