toxic proxy & bring back tts tests

.github/workflows/tests.yml

@@ -3,98 +3,29 @@ name: tests
 on:
   push:
     branches:
-      # - main
-      - 0.x
+      - main
   pull_request:
     branches:
-      # - main
-      - 0.x
+      - main
   workflow_dispatch:
 
 jobs:
   tests:
-    # Don't run tests for PRs on forks
+    # don't run tests for PRs on forks
     if: github.event_name != 'pull_request' || github.event.pull_request.head.repo.fork == false
     strategy:
       fail-fast: false
       matrix:
-        os: [
-            # disabled Intel Macs due to pytorch 2.3+ not supporting it
-            # macos-14-large,
-            macos-latest,
-            windows-latest,
-            ubuntu-latest,
-            ubuntu-24.04-arm,
-          ]
-        python_version: ["3.9", "3.12"]
-        test_group: ["base"]
-        include:
-          # Include llm, stt, and tts tests only on Ubuntu 20.04 with Python 3.9
-          - os: ubuntu-latest
-            python_version: "3.12"
-            test_group: llm
-          - os: ubuntu-latest
-            python_version: "3.12"
-            test_group: stt
-          - os: ubuntu-latest
-            python_version: "3.12"
-            test_group: tts
+        plugin: [cartesia, aws, azure, deepgram, elevenlabs, google, groq, neuphonic, playai, resemble, rime]
 
-    runs-on: ${{ matrix.os }}
-    name: ${{ matrix.test_group }} — ${{ matrix.os }}) (py${{ matrix.python_version }})
+    runs-on: ubuntu-latest
+    name: livekit-plugins-${{ matrix.plugin }}
     steps:
       - uses: actions/checkout@v4
-        with:
-          submodules: true
-          lfs: true
-
-      - name: Cache packages
-        uses: actions/cache@v4
-        with:
-          path: |
-            /var/cache/apt/archives
-            ~/Library/Caches/Homebrew
-            C:\ProgramData\chocolatey\lib\ffmpeg
-          key: ${{ runner.os }}-cache
-          restore-keys: |
-            ${{ runner.os }}-cache
-
-      - uses: actions/setup-python@v5
-        with:
-          python-version: ${{ matrix.python_version }}
-          cache: "pip"
-
-      - name: Install ffmpeg (Linux)
-        if: ${{ startsWith(matrix.os, 'ubuntu') }}
-        run: sudo apt-get update && sudo apt-get install -y ffmpeg
-
-      # Azure plugin fails with OpenSSL3, and Ubuntu 22.04 does not include libssl1.1 in its repos
-      - name: Install libssl 1.1 (Linux 22.04)
-        if: ${{ matrix.os == 'ubuntu-24.04-arm' }}
-        run: |
-          wget https://old-releases.ubuntu.com/ubuntu/pool/main/o/openssl/libssl1.1_1.1.1-1ubuntu2.1_arm64.deb
-          wget https://old-releases.ubuntu.com/ubuntu/pool/main/o/openssl/libssl-dev_1.1.1-1ubuntu2.1_arm64.deb
-          sudo dpkg -i libssl1.1_1.1.1-1ubuntu2.1_arm64.deb
-          sudo dpkg -i libssl-dev_1.1.1-1ubuntu2.1_arm64.deb
-
-      - name: Install ffmpeg (macOS)
-        if: ${{ startsWith(matrix.os, 'macos') }}
-        run: brew install ffmpeg
-
-      - name: Install ffmpeg (Windows)
-        if: ${{ startsWith(matrix.os, 'windows') }}
-        run: choco install ffmpeg
-
-      - name: Install packages
-        shell: bash
-        run: |
-          pip install pytest pytest-asyncio pytest-timeout './livekit-agents[codecs]' psutil
-          pip install -r ./tests/test-requirements.txt
-          ./livekit-plugins/install_local.sh
-
       - name: Run tests
         shell: bash
         env:
+          PLUGIN: ${{ matrix.plugin }}
           LIVEKIT_URL: ${{ secrets.LIVEKIT_URL }}
           LIVEKIT_API_KEY: ${{ secrets.LIVEKIT_API_KEY }}
           LIVEKIT_API_SECRET: ${{ secrets.LIVEKIT_API_SECRET }}
@@ -115,31 +46,9 @@ jobs:
           RIME_API_KEY: ${{ secrets.RIME_API_KEY }}
           SPEECHMATICS_API_KEY: ${{ secrets.SPEECHMATICS_API_KEY }}
           GOOGLE_APPLICATION_CREDENTIALS: google.json
-          PYTEST_ADDOPTS: "--color=yes"
           AWS_ACCESS_KEY_ID: ${{ secrets.AWS_ACCESS_KEY_ID }}
           AWS_SECRET_ACCESS_KEY: ${{ secrets.AWS_SECRET_ACCESS_KEY }}
           NEUPHONIC_API_KEY: ${{ secrets.NEUPHONIC_API_KEY }}
           RESEMBLE_API_KEY: ${{ secrets.RESEMBLE_API_KEY }}
         working-directory: tests
-        run: |
-          echo "$GOOGLE_CREDENTIALS_JSON" > google.json
-
-          case "${{ matrix.test_group }}" in
-            base)
-              test_files="test_aio.py test_tokenizer.py test_vad.py test_ipc.py test_tts_fallback.py test_stt_fallback.py test_message_change.py test_build_func_desc.py test_create_func.py test_connection_pool.py"
-              ;;
-            llm)
-              test_files="test_llm.py"
-              ;;
-            stt)
-              test_files="test_stt.py"
-              ;;
-            tts)
-              test_files="test_tts.py"
-              ;;
-            *)
-              echo "Unknown test group: ${{ matrix.test_group }}"
-              exit 1
-              ;;
-          esac
-          pytest $test_files
+        run: make test

livekit-agents/livekit/agents/llm/llm.py

@@ -225,7 +225,7 @@
             val = await self._event_aiter.__anext__()
         except StopAsyncIteration:
             if not self._task.cancelled() and (exc := self._task.exception()):
-                raise exc from None
+                raise exc
 
             raise StopAsyncIteration from None
 

livekit-agents/livekit/agents/stt/stt.py

@@ -337,7 +337,7 @@
             val = await self._event_aiter.__anext__()
         except StopAsyncIteration:
             if not self._task.cancelled() and (exc := self._task.exception()):
-                raise exc from None
+                raise exc
 
             raise StopAsyncIteration from None
 

livekit-agents/livekit/agents/tts/tts.py

@@ -250,7 +250,7 @@
             val = await self._event_aiter.__anext__()
         except StopAsyncIteration:
             if not self._synthesize_task.cancelled() and (exc := self._synthesize_task.exception()):
-                raise exc from None
+                raise exc
 
             raise StopAsyncIteration from None
 
@@ -373,7 +373,6 @@
                 cancelled=self._task.cancelled(),
                 label=self._tts._label,
                 streamed=True,
-                error=None,
             )
             self._tts.emit("metrics_collected", metrics)
 
@@ -446,7 +445,7 @@
             val = await self._event_aiter.__anext__()
         except StopAsyncIteration:
             if not self._task.cancelled() and (exc := self._task.exception()):
-                raise exc from None
+                raise exc
 
             raise StopAsyncIteration from None
 

livekit-agents/livekit/agents/vad.py

@@ -151,7 +151,7 @@
         self._input_ch.close()
 
     async def aclose(self) -> None:
-        """Close ths stream immediately"""
+        """Close the stream immediately"""
         self._input_ch.close()
         await aio.cancel_and_wait(self._task)
         self._event_ch.close()
@@ -162,7 +162,7 @@
             val = await self._event_aiter.__anext__()
         except StopAsyncIteration:
             if not self._task.cancelled() and (exc := self._task.exception()):
-                raise exc from None
+                raise exc
 
             raise StopAsyncIteration from None
 

livekit-plugins/livekit-plugins-aws/livekit/plugins/aws/tts.py

@@ -10,11 +10,12 @@
 # See the License for the specific language governing permissions and
 # limitations under the License.
 
 from __future__ import annotations
 
 import asyncio
 from dataclasses import dataclass
 
+import boto3
 import aioboto3
 import aiohttp
 
@@ -182,14 +183,14 @@
                     finally:
                         await utils.aio.gracefully_cancel(push_task)
 
-        except asyncio.TimeoutError as e:
-            raise APITimeoutError() from e
+        except asyncio.TimeoutError:
+            raise APITimeoutError() from None
         except aiohttp.ClientResponseError as e:
             raise APIStatusError(
                 message=e.message,
                 status_code=e.status,
                 request_id=request_id,
                 body=None,
-            ) from e
+            ) from None
         except Exception as e:
             raise APIConnectionError() from e

livekit-plugins/livekit-plugins-cartesia/livekit/plugins/cartesia/tts.py

@@ -272,15 +272,15 @@ async def _run(self) -> None:
                 for frame in bstream.flush():
                     emitter.push(frame)
                 emitter.flush()
-        except asyncio.TimeoutError as e:
-            raise APITimeoutError() from e
+        except asyncio.TimeoutError:
+            raise APITimeoutError() from None
         except aiohttp.ClientResponseError as e:
             raise APIStatusError(
                 message=e.message,
                 status_code=e.status,
                 request_id=None,
                 body=None,
-            ) from e
+            ) from None
         except Exception as e:
             raise APIConnectionError() from e
 

livekit-plugins/livekit-plugins-groq/livekit/plugins/groq/services.py

@@ -1,3 +1,5 @@
+from __future__ import annotations
+
 import os
 
 import openai

pyproject.toml

@@ -66,7 +66,6 @@ convention = "google"
 [tool.pytest.ini_options]
 asyncio_mode = "auto"
 asyncio_default_fixture_loop_scope = "function"
-timeout = 120
 addopts = ["--import-mode=importlib", "--ignore=examples"]
 
 

tests/Dockerfile.tests

@@ -0,0 +1,9 @@
+FROM python:3.9-slim
+
+ENV PYTHONUNBUFFERED=1
+
+RUN apt-get update && apt-get install -y curl strace procps
+
+RUN pip install --no-cache-dir uv
+
+WORKDIR /app

tests/Dockerfile.toxiproxy

@@ -0,0 +1,19 @@
+FROM golang:1.23-alpine AS builder
+
+RUN apk add --no-cache git make
+
+WORKDIR /build
+
+RUN git clone https://github.com/Shopify/toxiproxy.git .
+
+RUN make build
+
+RUN ls -al dist
+
+FROM alpine:3.18
+
+RUN apk add --no-cache ca-certificates
+RUN apk add --no-cache curl
+
+COPY --from=builder /build/dist/toxiproxy-server /usr/local/bin/toxiproxy-server
+COPY --from=builder /build/dist/toxiproxy-cli /usr/local/bin/toxiproxy-cli

tests/Makefile

@@ -0,0 +1,29 @@
+PLUGIN ?=
+
+.PHONY: test up down
+
+up:
+	@if [ -f ../.env ]; then \
+	  echo "Found .env file. Using it..."; \
+	  docker compose --env-file ../.env build; \
+	  docker compose --env-file ../.env up -d; \
+	else \
+	  echo "No .env file found. Running without it..."; \
+	  docker compose build; \
+	  docker compose up -d; \
+	fi
+
+down:
+	docker compose down
+
+test: up
+	@docker compose exec app bash -c "\
+	  until curl -sf http://toxiproxy:8474/proxies; do \
+	    echo 'Waiting for toxiproxy...'; \
+	    sleep 1; \
+	  done"
+	echo 'Toxiproxy is ready'
+
+	docker compose exec app uv sync --all-extras --dev
+	docker compose exec -e PLUGIN="$(PLUGIN)" app uv run pytest -s --color=yes --tb=short tests/test_tts.py --show-capture=all
+	$(MAKE) down