feat(collection): add optional speech for WebXR and desktop controls

dimos/cli/commands/imitation.py

@@ -25,12 +25,14 @@
 
 from dimos.cli.imitation_inspect import print_inspection
 from dimos.constants import DIMOS_PROJECT_ROOT, STATE_DIR
+from dimos.imitation.collection.prompts import CollectionSpeech
 from dimos.imitation.collection.recording import RecordingSchema
 from dimos.imitation.dataprep.build import inspect_dataset, inspect_recording
 from dimos.imitation.dataprep.core import OutputConfig
 from dimos.imitation.dataprep.lerobot import run_lerobot_dataprep
 from dimos.imitation.tui import CollectionApp, CollectionSession, RolloutApp, RolloutSession
 from dimos.porcelain.dimos import Dimos
+from dimos.stream.audio.tts.kokoro import KokoroTTSConfig
 from dimos.utils.cache import cache_usage_guard
 
 imitation_app = typer.Typer(help="Operate running collection/policy modules and prepare datasets")
@@ -48,16 +50,28 @@ def _require_new_path(path: Path) -> Path:
 
 
 @imitation_app.command()
-def collect() -> None:
+def collect(
+    tts: bool = typer.Option(False, "--tts", help="Speak recording feedback on this computer"),
+) -> None:
     """Attach episode controls to a blueprint started with dimos run."""
     driver = None
+    app = None
     try:
         driver = Dimos.connect()
-        CollectionApp(CollectionSession(driver)).run()
+        session = CollectionSession(driver)
+        speech = None
+        if tts:
+            typer.echo("Preparing recording speech...")
+            speech = CollectionSpeech(KokoroTTSConfig(enabled=True))
+            speech.prepare()
+        app = CollectionApp(session, speech=speech)
+        app.run()
     except Exception as exc:
         typer.echo(f"Collection controls failed: {exc}", err=True)
         raise typer.Exit(1) from exc
     finally:
+        if app is not None:
+            app.stop_audio()
         if driver is not None:
             driver.stop()
 

dimos/imitation/collection/prompts.py

@@ -0,0 +1,82 @@
+# Copyright 2026 Dimensional Inc.
+#
+# Licensed under the Apache License, Version 2.0 (the "License");
+# you may not use this file except in compliance with the License.
+# You may obtain a copy of the License at
+#
+#     http://www.apache.org/licenses/LICENSE-2.0
+#
+# Unless required by applicable law or agreed to in writing, software
+# distributed under the License is distributed on an "AS IS" BASIS,
+# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+# See the License for the specific language governing permissions and
+# limitations under the License.
+
+"""Spoken feedback for confirmed collection transitions."""
+
+from dimos.msgs.imitation_msgs.EpisodeStatus import EpisodeStatus
+from dimos.stream.audio.tts.kokoro import KokoroTTS, KokoroTTSConfig
+
+RECORDING_PROMPTS = {
+    "start": "Recording started",
+    "save": "Episode saved",
+    "discard": "Recording canceled",
+}
+
+
+class CollectionPrompts:
+    def __init__(self) -> None:
+        self.previous: EpisodeStatus | None = None
+
+    def update(self, status: EpisodeStatus) -> str | None:
+        previous = self.previous
+        self.previous = status
+        if previous is not None and (
+            status.state,
+            status.last_event,
+            status.episodes_saved,
+            status.episodes_discarded,
+        ) == (
+            previous.state,
+            previous.last_event,
+            previous.episodes_saved,
+            previous.episodes_discarded,
+        ):
+            return None
+        if status.last_event == "start" and status.state == "recording":
+            return RECORDING_PROMPTS["start"]
+        if previous is not None:
+            if status.last_event == "save" and status.episodes_saved > previous.episodes_saved:
+                return RECORDING_PROMPTS["save"]
+            if (
+                status.last_event == "discard"
+                and status.episodes_discarded > previous.episodes_discarded
+            ):
+                return RECORDING_PROMPTS["discard"]
+        return None
+
+
+class CollectionSpeech:
+    """Prepare collection feedback once, then select WAVs for confirmed transitions."""
+
+    def __init__(self, config: KokoroTTSConfig) -> None:
+        self._config = config
+        self._prompts = CollectionPrompts()
+        self._audio: dict[str, bytes] = {}
+
+    def prepare(self) -> None:
+        if not self._config.enabled:
+            return
+        speech = KokoroTTS(self._config)
+        try:
+            speech.prepare()
+            self._audio = {
+                phrase: speech.synthesize(phrase) for phrase in RECORDING_PROMPTS.values()
+            }
+        finally:
+            # All feedback is cached; collection needs no live inference engine.
+            speech.close()
+
+    def update(self, status: EpisodeStatus, *, snapshot: bool = False) -> bytes | None:
+        phrase = self._prompts.update(status)
+        return self._audio.get(phrase) if phrase is not None and not snapshot else None

dimos/imitation/collection/test_prompts.py

@@ -0,0 +1,75 @@
+# Copyright 2026 Dimensional Inc.
+#
+# Licensed under the Apache License, Version 2.0 (the "License");
+# you may not use this file except in compliance with the License.
+# You may obtain a copy of the License at
+#
+#     http://www.apache.org/licenses/LICENSE-2.0
+#
+# Unless required by applicable law or agreed to in writing, software
+# distributed under the License is distributed on an "AS IS" BASIS,
+# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+# See the License for the specific language governing permissions and
+# limitations under the License.
+
+import pytest
+
+from dimos.imitation.collection.prompts import CollectionSpeech
+from dimos.msgs.imitation_msgs.EpisodeStatus import EpisodeStatus
+from dimos.stream.audio.tts.kokoro import KokoroTTSConfig
+
+
+def test_confirmed_transitions_use_prepared_audio_and_ignore_duplicate_polls(speech):
+    prompts, engine = speech
+    engine.close.assert_called_once()
+    engine.synthesize.reset_mock()
+    events = [
+        ("init", "idle", 0, 0, None),
+        ("save", "idle", 0, 0, None),
+        ("discard", "idle", 0, 0, None),
+        ("start", "recording", 0, 0, b"Recording started"),
+        ("save", "idle", 1, 0, b"Episode saved"),
+        ("start", "recording", 1, 0, b"Recording started"),
+        ("discard", "idle", 1, 1, b"Recording canceled"),
+        ("start", "recording", 1, 1, b"Recording started"),
+        ("start", "recording", 2, 1, b"Recording started"),
+    ]
+    for ts, (event, state, saved, discarded, expected) in enumerate(events):
+        status = EpisodeStatus(
+            ts=ts, last_event=event, state=state, episodes_saved=saved, episodes_discarded=discarded
+        )
+        assert prompts.update(status) == expected
+        assert prompts.update(status) is None
+        # RPC polling refreshes timestamps even when the episode has not changed.
+        assert prompts.update(status.model_copy(update={"ts": status.ts + 0.5})) is None
+    engine.synthesize.assert_not_called()
+
+
+@pytest.fixture
+def speech(mocker):
+    engine = mocker.patch(
+        "dimos.imitation.collection.prompts.KokoroTTS", autospec=True
+    ).return_value
+    engine.synthesize.side_effect = lambda phrase: phrase.encode()
+    speech = CollectionSpeech(KokoroTTSConfig(enabled=True))
+    speech.prepare()
+    return speech, engine
+
+
+def test_failed_preparation_releases_engine(mocker):
+    engine = mocker.patch(
+        "dimos.imitation.collection.prompts.KokoroTTS", autospec=True
+    ).return_value
+    engine.synthesize.side_effect = RuntimeError("synthesis failed")
+    speech = CollectionSpeech(KokoroTTSConfig(enabled=True))
+    with pytest.raises(RuntimeError, match="synthesis failed"):
+        speech.prepare()
+    engine.close.assert_called_once_with()
+    assert (
+        speech.update(
+            EpisodeStatus(
+                ts=1, state="recording", last_event="start", episodes_saved=0, episodes_discarded=0
+            )
+        )
+        is None
+    )

dimos/imitation/test_tui.py

@@ -21,10 +21,12 @@
 from dimos.core.global_config import GlobalConfig
 from dimos.core.module import Module
 from dimos.imitation.collection.episode_monitor import EpisodeCommand, EpisodeControlSpec
+from dimos.imitation.collection.prompts import CollectionSpeech
 from dimos.imitation.policy.module import RolloutControlSpec
 from dimos.imitation.tui import CollectionApp, CollectionSession, RolloutApp, RolloutSession
 from dimos.msgs.imitation_msgs.EpisodeStatus import EpisodeStatus
 from dimos.porcelain.dimos import Dimos
+from dimos.stream.audio.tts.kokoro import KokoroTTSConfig
 
 
 @pytest.fixture
@@ -172,3 +174,57 @@ def test_rollout_disconnect_disables_commands_without_stopping_policy(mocker):
         driver.stop.assert_called_once_with()
     finally:
         session.close()
+
+
+async def test_collection_speech_follows_confirmed_status_and_stops_on_detach(
+    collection_session, mocker
+):
+    session, _, monitor = collection_session
+    engine = mocker.patch(
+        "dimos.imitation.collection.prompts.KokoroTTS", autospec=True
+    ).return_value
+    engine.synthesize.side_effect = lambda phrase: phrase.encode()
+    speech = CollectionSpeech(KokoroTTSConfig(enabled=True))
+    speech.prepare()
+    player = mocker.patch("dimos.imitation.tui.WavPlayer", autospec=True).return_value
+    monitor.get_status.return_value = EpisodeStatus(
+        ts=1, state="recording", last_event="start", episodes_saved=0, episodes_discarded=0
+    )
+    app = CollectionApp(session, speech=speech)
+    mocker.patch.object(app, "set_interval")
+    async with app.run_test(size=(80, 24)):
+        player.play.assert_not_called()
+        monitor.command.return_value = EpisodeStatus(
+            ts=2, state="idle", last_event="save", episodes_saved=1, episodes_discarded=0
+        )
+        app.action_toggle_recording()
+        player.play.assert_called_once_with(b"Episode saved")
+        monitor.get_status.return_value = monitor.command.return_value
+        app._poll()
+        player.play.assert_called_once_with(b"Episode saved")
+        monitor.get_status.return_value = EpisodeStatus(
+            ts=3, state="recording", last_event="start", episodes_saved=1, episodes_discarded=0
+        )
+        app._poll()
+        assert player.play.call_args.args == (b"Recording started",)
+        monitor.get_status.return_value = monitor.get_status.return_value.model_copy(
+            update={"ts": 3.5}
+        )
+        app._poll()
+        assert player.play.call_count == 2
+        monitor.command.return_value = EpisodeStatus(
+            ts=4, state="idle", last_event="discard", episodes_saved=1, episodes_discarded=1
+        )
+        player.play.side_effect = RuntimeError("No output device")
+        app.action_discard()
+        assert not app._disconnected
+        assert app._status.episodes_discarded == 1
+        assert "Audio unavailable" in str(app.query_one("#message", Static).render())
+    player.stop.assert_called_once_with()
+
+
+def test_disabled_collection_does_not_open_audio(collection_session, mocker):
+    session, _, _ = collection_session
+    player = mocker.patch("dimos.imitation.tui.WavPlayer", autospec=True)
+    CollectionApp(session)
+    player.assert_not_called()

dimos/imitation/tui.py

@@ -25,9 +25,11 @@
 
 from dimos.cli import theme
 from dimos.imitation.collection.episode_monitor import EpisodeCommand, EpisodeControlSpec
+from dimos.imitation.collection.prompts import CollectionSpeech
 from dimos.imitation.policy.module import RolloutControlSpec, RolloutStatus
 from dimos.msgs.imitation_msgs.EpisodeStatus import EpisodeStatus
 from dimos.porcelain.dimos import Dimos
+from dimos.stream.audio.wav_player import WavPlayer
 
 
 class CollectionSession:
@@ -87,11 +89,22 @@ class CollectionApp(App[None]):
         Binding("ctrl+c", "quit", "Detach", show=False),
     ]
 
-    def __init__(self, session: CollectionSession, title: str = "Collection") -> None:
+    def __init__(
+        self,
+        session: CollectionSession,
+        title: str = "Collection",
+        *,
+        speech: CollectionSpeech | None = None,
+    ) -> None:
         super().__init__()
         self._session = session
         self._title = title
         self._status = session.get_status()
+        self._speech = speech
+        self._player = WavPlayer() if speech is not None else None
+        self._audio_error: str | None = None
+        if speech is not None:
+            speech.update(self._status, snapshot=True)
         self._message = "Reset the scene, then start a take."
         self._disconnected = False
         self._recording_started_at: float | None = None
@@ -118,8 +131,13 @@ def on_mount(self) -> None:
         self.set_interval(0.25, self._poll)
 
     def on_unmount(self) -> None:
+        self.stop_audio()
         self._session.close()
 
+    def stop_audio(self) -> None:
+        if self._player is not None:
+            self._player.stop()
+
     @staticmethod
     def _format_elapsed(seconds: float) -> str:
         minutes, seconds = divmod(max(seconds, 0.0), 60.0)
@@ -133,6 +151,14 @@ def _set_status(self, status: EpisodeStatus) -> None:
             self._recording_started_at = time.monotonic()
         elif not recording:
             self._recording_started_at = None
+        if self._speech is not None and self._player is not None:
+            audio = self._speech.update(status)
+            if audio is not None:
+                try:
+                    self._player.play(audio)
+                    self._audio_error = None
+                except Exception as exc:
+                    self._audio_error = f"Audio unavailable: {exc}"
 
     def _refresh(self) -> None:
         recording = self._status.state == "recording"
@@ -159,7 +185,7 @@ def _refresh(self) -> None:
             else "Reset the scene. Press Space when the demonstration begins."
         )
         self.query_one("#guidance", Static).update(guidance)
-        self.query_one("#message", Static).update(self._message)
+        self.query_one("#message", Static).update(self._audio_error or self._message)
         toggle = self.query_one("#toggle", Button)
         toggle.label = "Save episode" if recording else "Start recording"
         toggle.variant = "error" if recording else "success"
@@ -174,7 +200,9 @@ def _poll(self) -> None:
             self._refresh()
         except Exception as exc:
             self._message = f"Connection error: {exc}"
+            self._audio_error = None
             self._disconnected = True
+            self.stop_audio()
             self._session.close()
             self._refresh()