[draft] async text mode

examples/voice_agents/sms_agent.py

@@ -0,0 +1,151 @@
+import logging
+import os
+
+import aiohttp
+from dotenv import load_dotenv
+
+from livekit.agents import (
+    Agent,
+    AgentServer,
+    AgentSession,
+    EffectCall,
+    JobContext,
+    JobExecutorType,
+    RunContext,
+    TextMessageContext,
+    cli,
+)
+from livekit.agents.beta.workflows import GetEmailTask
+from livekit.agents.llm import Instructions, ToolFlag, function_tool
+from livekit.plugins import silero
+from livekit.plugins.turn_detector.multilingual import MultilingualModel
+
+logger = logging.getLogger("basic-agent")
+
+load_dotenv()
+
+PORT = int(os.getenv("PORT", 8081))
+
+
+class MyAgent(Agent):
+    BASE_INSTRUCTIONS = (
+        "Your name is Kelly. You would interact with users via {modality}. "
+        "with that in mind keep your responses concise and to the point. "
+        "{modality_specific}"
+        "You are curious and friendly, and have a sense of humor. "
+        "When user want to register for the weather event, call the `register_for_weather` function. "
+    )
+
+    def __init__(self) -> None:
+        super().__init__(
+            instructions=Instructions(
+                audio=self.BASE_INSTRUCTIONS.format(
+                    modality="voice",
+                    modality_specific="do not use emojis, asterisks, markdown, or other special characters in your responses.",
+                ),
+                text=self.BASE_INSTRUCTIONS.format(
+                    modality="text",
+                    modality_specific="you can use emojis to make your responses more engaging.",
+                ),
+            ),
+        )
+
+    @function_tool
+    async def get_weather(
+        self,
+        latitude: str,
+        longitude: str,
+    ):
+        """Called when the user asks about the weather. This function will return the weather for
+        the given location. When given a location, please estimate the latitude and longitude of the
+        location and do not ask the user for them.
+
+        Args:
+            latitude: The latitude of the location
+            longitude: The longitude of the location
+        """
+
+        logger.info(f"getting weather for {latitude}, {longitude}")
+        self.session.say("I'm getting the weather for you...")
+
+        url = f"https://api.open-meteo.com/v1/forecast?latitude={latitude}&longitude={longitude}&current=temperature_2m"
+        weather_data = {}
+        async with aiohttp.ClientSession() as session:
+            async with session.get(url) as response:
+                if response.status == 200:
+                    data = await response.json()
+                    # response from the function call is returned to the LLM
+                    weather_data = {
+                        "temperature": data["current"]["temperature_2m"],
+                        "temperature_unit": "Celsius",
+                    }
+                else:
+                    raise Exception(f"Failed to get weather data, status code: {response.status}")
+
+        return weather_data
+
+    @function_tool(flags=ToolFlag.DURABLE)
+    async def register_for_weather(self, context: RunContext):
+        """Called when the user wants to register for the weather event."""
+        logger.info("register_for_weather called")
+
+        chat_ctx = self.chat_ctx.copy(
+            exclude_function_call=True, exclude_instructions=True, exclude_config_update=True
+        )
+        email_result = await EffectCall(
+            GetEmailTask(
+                chat_ctx=chat_ctx,
+                setup_fnc=lambda agent: agent.configure(llm="openai/gpt-4.1"),
+            )
+        )
+        email_address = email_result.email_address
+
+        logger.info(f"User's email address: {email_address}")
+
+        return "You are now registered for the weather event."
+        # context.session.say("You are now registered for the weather event.")
+
+
+server = AgentServer(port=PORT, job_executor_type=JobExecutorType.THREAD)
+
+
+@server.text_handler(endpoint="weather")
+async def text_handler(ctx: TextMessageContext):
+    logger.info(f"text message received: {ctx.text}")
+
+    session = AgentSession(
+        llm="openai/gpt-4.1-mini",
+        # state_passphrase="my-secret-passphrase",
+    )
+
+    start_result = await session.start(
+        agent=ctx.session_state if ctx.session_state else MyAgent(),
+        capture_run=True,
+        wait_run_state=False,
+    )
+    async for ev in start_result:
+        await ctx.send_response(ev)
+
+    logger.info(f"running session with text input: {ctx.text}")
+    async for ev in session.run(user_input=ctx.text):
+        await ctx.send_response(ev)
+
+
+@server.rtc_session()
+async def entrypoint(ctx: JobContext):
+    session = AgentSession(
+        stt="deepgram/nova-3",
+        llm="openai/gpt-4.1-mini",
+        tts="cartesia/sonic-2:9626c31c-bec5-4cca-baa8-f8ba9e84c8bc",
+        turn_detection=MultilingualModel(),
+        vad=silero.VAD.load(),
+        preemptive_generation=True,
+    )
+
+    await session.start(agent=MyAgent(), room=ctx.room)
+
+    session.generate_reply(instructions="greeting the user", input_modality="audio")
+
+
+if __name__ == "__main__":
+    cli.run_app(server)

livekit-agents/livekit/agents/__init__.py

@@ -29,12 +29,14 @@
     AssignmentTimeoutError,
     create_api_error_from_http,
 )
+from .durable_scheduler import DurableScheduler, EffectCall
 from .job import (
     AutoSubscribe,
     JobContext,
     JobExecutorType,
     JobProcess,
     JobRequest,
+    TextMessageContext,
     get_job_context,
 )
 from .language import LanguageCode
@@ -55,6 +57,7 @@
     StopResponse,
     ToolContext,
     ToolError,
+    ToolFlag,
     function_tool,
 )
 from .plugin import Plugin
@@ -133,11 +136,13 @@ def __getattr__(name: str) -> typing.Any:
     "JobProcess",
     "JobContext",
     "JobRequest",
+    "TextMessageContext",
     "get_job_context",
     "JobExecutorType",
     "AutoSubscribe",
     "FunctionTool",
     "function_tool",
+    "ToolFlag",
     "ProviderTool",
     "ChatContext",
     "ChatItem",
@@ -215,6 +220,8 @@ def __getattr__(name: str) -> typing.Any:
     "FunctionCallEvent",
     "FunctionCallOutputEvent",
     "AgentHandoffEvent",
+    "DurableScheduler",
+    "EffectCall",
 ]
 
 # Cleanup docs of unexported modules

livekit-agents/livekit/agents/_exceptions.py

@@ -1,5 +1,7 @@
 from __future__ import annotations
 
+from livekit.protocol.agent_pb import agent_text
+
 
 class AssignmentTimeoutError(Exception):
     """Raised when accepting a job but not receiving an assignment within the specified timeout.
@@ -102,6 +104,28 @@ class CLIError(Exception):
     pass
 
 
+class TextMessageError(Exception):
+    def __init__(
+        self,
+        message: str,
+        code: agent_text.TextMessageErrorCode = agent_text.INTERNAL_ERROR,
+    ) -> None:
+        super().__init__(message)
+        self._message = message
+        self._code = code
+
+    @property
+    def message(self) -> str:
+        return self._message
+
+    @property
+    def code(self) -> agent_text.TextMessageErrorCode:
+        return self._code
+
+    def to_proto(self) -> agent_text.TextMessageError:
+        return agent_text.TextMessageError(message=self._message, code=self._code)
+
+
 def create_api_error_from_http(
     message: str = "",
     *,

livekit-agents/livekit/agents/beta/workflows/address.py

@@ -1,7 +1,8 @@
 from __future__ import annotations
 
+from collections.abc import Callable
 from dataclasses import dataclass
-from typing import TYPE_CHECKING
+from typing import TYPE_CHECKING, Any
 
 from ... import llm, stt, tts, vad
 from ...llm import Instructions
@@ -13,6 +14,7 @@
 from ...voice.speech_handle import SpeechHandle
 
 if TYPE_CHECKING:
+    from ...voice.agent import Agent, _AgentState
     from ...voice.audio_recognition import TurnDetectionMode
 
 
@@ -75,7 +77,14 @@ def __init__(
         tts: NotGivenOr[tts.TTS | None] = NOT_GIVEN,
         allow_interruptions: NotGivenOr[bool] = NOT_GIVEN,
         require_confirmation: NotGivenOr[bool] = NOT_GIVEN,
+        *,
+        setup_fnc: Callable[[Agent], None] | None = None,
     ) -> None:
+        self._init_kwargs = {
+            "extra_instructions": extra_instructions,
+            "allow_interruptions": allow_interruptions,
+            "require_confirmation": require_confirmation,
+        }
         confirmation_instructions = (
             "Call `confirm_address` after the user confirmed the address is correct."
         )
@@ -106,12 +115,25 @@ def __init__(
             llm=llm,
             tts=tts,
             allow_interruptions=allow_interruptions,
+            setup_fnc=setup_fnc,
         )
 
         self._current_address = ""
         self._require_confirmation = require_confirmation
         self._address_update_speech_handle: SpeechHandle | None = None
 
+    def export_init_kwargs(self) -> dict[str, Any]:
+        return self._init_kwargs
+
+    def _snapshot(self) -> _AgentState:
+        state = super()._snapshot()
+        state.extra_state["current_address"] = self._current_address
+        return state
+
+    def _restore(self, state: _AgentState) -> None:
+        super()._restore(state)
+        self._current_address = state.extra_state["current_address"]
+
     async def on_enter(self) -> None:
         self.session.generate_reply(instructions="Ask the user to provide their address.")
 

livekit-agents/livekit/agents/beta/workflows/email_address.py

@@ -1,8 +1,9 @@
 from __future__ import annotations
 
 import re
+from collections.abc import Callable
 from dataclasses import dataclass
-from typing import TYPE_CHECKING
+from typing import TYPE_CHECKING, Any
 
 from ... import llm, stt, tts, vad
 from ...llm import Instructions
@@ -14,6 +15,7 @@
 from ...voice.speech_handle import SpeechHandle
 
 if TYPE_CHECKING:
+    from ...voice.agent import Agent, _AgentState
     from ...voice.audio_recognition import TurnDetectionMode
 
 EMAIL_REGEX = (
@@ -72,7 +74,14 @@ def __init__(
         tts: NotGivenOr[tts.TTS | None] = NOT_GIVEN,
         allow_interruptions: NotGivenOr[bool] = NOT_GIVEN,
         require_confirmation: NotGivenOr[bool] = NOT_GIVEN,
+        *,
+        setup_fnc: Callable[[Agent], None] | None = None,
     ) -> None:
+        self._init_kwargs = {
+            "extra_instructions": extra_instructions,
+            "allow_interruptions": allow_interruptions,
+            "require_confirmation": require_confirmation,
+        }
         confirmation_instructions = (
             "Call `confirm_email_address` after the user confirmed the email address is correct."
         )
@@ -103,6 +112,7 @@ def __init__(
             llm=llm,
             tts=tts,
             allow_interruptions=allow_interruptions,
+            setup_fnc=setup_fnc,
         )
 
         self._current_email = ""
@@ -111,6 +121,9 @@ def __init__(
         # used to ignore the call to confirm_email_address in case the LLM is hallucinating and not asking for user confirmation
         self._email_update_speech_handle: SpeechHandle | None = None
 
+    def export_init_kwargs(self) -> dict[str, Any]:
+        return self._init_kwargs
+
     async def on_enter(self) -> None:
         self.session.generate_reply(instructions="Ask the user to provide an email address.")
 
@@ -173,3 +186,12 @@ def _confirmation_required(self, ctx: RunContext) -> bool:
         if is_given(self._require_confirmation):
             return self._require_confirmation
         return ctx.speech_handle.input_details.modality == "audio"
+
+    def _snapshot(self) -> _AgentState:
+        state = super()._snapshot()
+        state.extra_state["current_email"] = self._current_email
+        return state
+
+    def _restore(self, state: _AgentState) -> None:
+        super()._restore(state)
+        self._current_email = state.extra_state["current_email"]