feat(whisperx): add whisperx backend for transcription with speaker diarization

.github/workflows/backend.yml

@@ -91,6 +91,19 @@ jobs:
             dockerfile: "./backend/Dockerfile.python"
             context: "./"
             ubuntu-version: '2404'
+          - build-type: ''
+            cuda-major-version: ""
+            cuda-minor-version: ""
+            platforms: 'linux/amd64'
+            tag-latest: 'auto'
+            tag-suffix: '-cpu-whisperx'
+            runs-on: 'ubuntu-latest'
+            base-image: "ubuntu:24.04"
+            skip-drivers: 'true'
+            backend: "whisperx"
+            dockerfile: "./backend/Dockerfile.python"
+            context: "./"
+            ubuntu-version: '2404'
           # CUDA 12 builds
           - build-type: 'cublas'
             cuda-major-version: "12"
@@ -266,6 +279,19 @@ jobs:
             cuda-minor-version: "8"
             platforms: 'linux/amd64'
             tag-latest: 'auto'
+            tag-suffix: '-gpu-nvidia-cuda-12-whisperx'
+            runs-on: 'ubuntu-latest'
+            base-image: "ubuntu:24.04"
+            skip-drivers: 'false'
+            backend: "whisperx"
+            dockerfile: "./backend/Dockerfile.python"
+            context: "./"
+            ubuntu-version: '2404'
+          - build-type: 'cublas'
+            cuda-major-version: "12"
+            cuda-minor-version: "9"
+            platforms: 'linux/amd64'
+            tag-latest: 'auto'
             tag-suffix: '-gpu-nvidia-cuda-12-coqui'
             runs-on: 'ubuntu-latest'
             base-image: "ubuntu:24.04"
@@ -574,6 +600,19 @@ jobs:
             dockerfile: "./backend/Dockerfile.python"
             context: "./"
             ubuntu-version: '2404'
+          - build-type: 'cublas'
+            cuda-major-version: "13"
+            cuda-minor-version: "0"
+            platforms: 'linux/amd64'
+            tag-latest: 'auto'
+            tag-suffix: '-gpu-nvidia-cuda-13-whisperx'
+            runs-on: 'ubuntu-latest'
+            base-image: "ubuntu:24.04"
+            skip-drivers: 'false'
+            backend: "whisperx"
+            dockerfile: "./backend/Dockerfile.python"
+            context: "./"
+            ubuntu-version: '2404'
           - build-type: 'cublas'
             cuda-major-version: "13"
             cuda-minor-version: "0"
@@ -836,6 +875,19 @@ jobs:
             dockerfile: "./backend/Dockerfile.python"
             context: "./"
             ubuntu-version: '2404'
+          - build-type: 'hipblas'
+            cuda-major-version: ""
+            cuda-minor-version: ""
+            platforms: 'linux/amd64'
+            tag-latest: 'auto'
+            tag-suffix: '-gpu-rocm-hipblas-whisperx'
+            runs-on: 'bigger-runner'
+            base-image: "rocm/dev-ubuntu-24.04:6.4.4"
+            skip-drivers: 'false'
+            backend: "whisperx"
+            dockerfile: "./backend/Dockerfile.python"
+            context: "./"
+            ubuntu-version: '2404'
           - build-type: 'hipblas'
             cuda-major-version: ""
             cuda-minor-version: ""

Makefile

@@ -1,5 +1,5 @@
 # Disable parallel execution for backend builds
-.NOTPARALLEL: backends/diffusers backends/llama-cpp backends/piper backends/stablediffusion-ggml backends/whisper backends/faster-whisper backends/silero-vad backends/local-store backends/huggingface backends/rfdetr backends/kitten-tts backends/kokoro backends/chatterbox backends/llama-cpp-darwin backends/neutts build-darwin-python-backend build-darwin-go-backend backends/mlx backends/diffuser-darwin backends/mlx-vlm backends/mlx-audio backends/stablediffusion-ggml-darwin backends/vllm backends/vllm-omni backends/moonshine backends/pocket-tts backends/qwen-tts backends/qwen-asr backends/voxcpm
+.NOTPARALLEL: backends/diffusers backends/llama-cpp backends/piper backends/stablediffusion-ggml backends/whisper backends/faster-whisper backends/silero-vad backends/local-store backends/huggingface backends/rfdetr backends/kitten-tts backends/kokoro backends/chatterbox backends/llama-cpp-darwin backends/neutts build-darwin-python-backend build-darwin-go-backend backends/mlx backends/diffuser-darwin backends/mlx-vlm backends/mlx-audio backends/stablediffusion-ggml-darwin backends/vllm backends/vllm-omni backends/moonshine backends/pocket-tts backends/qwen-tts backends/qwen-asr backends/voxcpm backends/whisperx
 
 GOCMD=go
 GOTEST=$(GOCMD) test
@@ -318,6 +318,7 @@ prepare-test-extra: protogen-python
 	$(MAKE) -C backend/python/qwen-tts
 	$(MAKE) -C backend/python/qwen-asr
 	$(MAKE) -C backend/python/voxcpm
+	$(MAKE) -C backend/python/whisperx
 
 test-extra: prepare-test-extra
 	$(MAKE) -C backend/python/transformers test
@@ -331,6 +332,7 @@ test-extra: prepare-test-extra
 	$(MAKE) -C backend/python/qwen-tts test
 	$(MAKE) -C backend/python/qwen-asr test
 	$(MAKE) -C backend/python/voxcpm test
+	$(MAKE) -C backend/python/whisperx test
 
 DOCKER_IMAGE?=local-ai
 DOCKER_AIO_IMAGE?=local-ai-aio
@@ -465,6 +467,7 @@ BACKEND_POCKET_TTS = pocket-tts|python|.|false|true
 BACKEND_QWEN_TTS = qwen-tts|python|.|false|true
 BACKEND_QWEN_ASR = qwen-asr|python|.|false|true
 BACKEND_VOXCPM = voxcpm|python|.|false|true
+BACKEND_WHISPERX = whisperx|python|.|false|true
 
 # Helper function to build docker image for a backend
 # Usage: $(call docker-build-backend,BACKEND_NAME,DOCKERFILE_TYPE,BUILD_CONTEXT,PROGRESS_FLAG,NEEDS_BACKEND_ARG)
@@ -512,12 +515,13 @@ $(eval $(call generate-docker-build-target,$(BACKEND_POCKET_TTS)))
 $(eval $(call generate-docker-build-target,$(BACKEND_QWEN_TTS)))
 $(eval $(call generate-docker-build-target,$(BACKEND_QWEN_ASR)))
 $(eval $(call generate-docker-build-target,$(BACKEND_VOXCPM)))
+$(eval $(call generate-docker-build-target,$(BACKEND_WHISPERX)))
 
 # Pattern rule for docker-save targets
 docker-save-%: backend-images
 	docker save local-ai-backend:$* -o backend-images/$*.tar
 
-docker-build-backends: docker-build-llama-cpp docker-build-rerankers docker-build-vllm docker-build-vllm-omni docker-build-transformers docker-build-diffusers docker-build-kokoro docker-build-faster-whisper docker-build-coqui docker-build-chatterbox docker-build-vibevoice docker-build-moonshine docker-build-pocket-tts docker-build-qwen-tts docker-build-qwen-asr docker-build-voxcpm
+docker-build-backends: docker-build-llama-cpp docker-build-rerankers docker-build-vllm docker-build-vllm-omni docker-build-transformers docker-build-diffusers docker-build-kokoro docker-build-faster-whisper docker-build-coqui docker-build-chatterbox docker-build-vibevoice docker-build-moonshine docker-build-pocket-tts docker-build-qwen-tts docker-build-qwen-asr docker-build-voxcpm docker-build-whisperx
 
 ########################################################
 ### Mock Backend for E2E Tests

backend/backend.proto

@@ -299,6 +299,7 @@ message TranscriptSegment {
   int64 end = 3;
   string text = 4;
   repeated int32 tokens = 5;
+  string speaker = 6;
 }
 
 message GenerateImageRequest {

backend/index.yaml

@@ -302,6 +302,25 @@
     default: "cpu-moonshine"
     nvidia-cuda-13: "cuda13-moonshine"
     nvidia-cuda-12: "cuda12-moonshine"
+- &whisperx
+  description: |
+    WhisperX provides fast automatic speech recognition with word-level timestamps, speaker diarization,
+    and forced alignment. Built on faster-whisper and pyannote-audio for high-accuracy transcription
+    with speaker identification.
+  urls:
+    - https://github.com/m-bain/whisperX
+  tags:
+    - speech-to-text
+    - diarization
+    - whisperx
+  license: BSD-4-Clause
+  name: "whisperx"
+  capabilities:
+    nvidia: "cuda12-whisperx"
+    amd: "rocm-whisperx"
+    default: "cpu-whisperx"
+    nvidia-cuda-13: "cuda13-whisperx"
+    nvidia-cuda-12: "cuda12-whisperx"
 - &kokoro
   icon: https://avatars.githubusercontent.com/u/166769057?v=4
   description: |
@@ -1417,6 +1436,55 @@
   uri: "quay.io/go-skynet/local-ai-backends:master-gpu-nvidia-cuda-13-moonshine"
   mirrors:
     - localai/localai-backends:master-gpu-nvidia-cuda-13-moonshine
+## whisperx
+- !!merge <<: *whisperx
+  name: "whisperx-development"
+  capabilities:
+    nvidia: "cuda12-whisperx-development"
+    amd: "rocm-whisperx-development"
+    default: "cpu-whisperx-development"
+    nvidia-cuda-13: "cuda13-whisperx-development"
+    nvidia-cuda-12: "cuda12-whisperx-development"
+- !!merge <<: *whisperx
+  name: "cpu-whisperx"
+  uri: "quay.io/go-skynet/local-ai-backends:latest-cpu-whisperx"
+  mirrors:
+    - localai/localai-backends:latest-cpu-whisperx
+- !!merge <<: *whisperx
+  name: "cpu-whisperx-development"
+  uri: "quay.io/go-skynet/local-ai-backends:master-cpu-whisperx"
+  mirrors:
+    - localai/localai-backends:master-cpu-whisperx
+- !!merge <<: *whisperx
+  name: "cuda12-whisperx"
+  uri: "quay.io/go-skynet/local-ai-backends:latest-gpu-nvidia-cuda-12-whisperx"
+  mirrors:
+    - localai/localai-backends:latest-gpu-nvidia-cuda-12-whisperx
+- !!merge <<: *whisperx
+  name: "cuda12-whisperx-development"
+  uri: "quay.io/go-skynet/local-ai-backends:master-gpu-nvidia-cuda-12-whisperx"
+  mirrors:
+    - localai/localai-backends:master-gpu-nvidia-cuda-12-whisperx
+- !!merge <<: *whisperx
+  name: "rocm-whisperx"
+  uri: "quay.io/go-skynet/local-ai-backends:latest-gpu-rocm-hipblas-whisperx"
+  mirrors:
+    - localai/localai-backends:latest-gpu-rocm-hipblas-whisperx
+- !!merge <<: *whisperx
+  name: "rocm-whisperx-development"
+  uri: "quay.io/go-skynet/local-ai-backends:master-gpu-rocm-hipblas-whisperx"
+  mirrors:
+    - localai/localai-backends:master-gpu-rocm-hipblas-whisperx
+- !!merge <<: *whisperx
+  name: "cuda13-whisperx"
+  uri: "quay.io/go-skynet/local-ai-backends:latest-gpu-nvidia-cuda-13-whisperx"
+  mirrors:
+    - localai/localai-backends:latest-gpu-nvidia-cuda-13-whisperx
+- !!merge <<: *whisperx
+  name: "cuda13-whisperx-development"
+  uri: "quay.io/go-skynet/local-ai-backends:master-gpu-nvidia-cuda-13-whisperx"
+  mirrors:
+    - localai/localai-backends:master-gpu-nvidia-cuda-13-whisperx
 ## coqui
 
 - !!merge <<: *coqui

backend/python/whisperx/Makefile

@@ -0,0 +1,16 @@
+.DEFAULT_GOAL := install
+
+.PHONY: install
+install:
+	bash install.sh
+
+.PHONY: protogen-clean
+protogen-clean:
+	$(RM) backend_pb2_grpc.py backend_pb2.py
+
+.PHONY: clean
+clean: protogen-clean
+	rm -rf venv __pycache__
+
+test: install
+	bash test.sh