SableClient · TomOdellSheetMusic · Aug 12, 2026
diff --git a/sdk/main.ts b/sdk/main.ts
@@ -84,6 +84,8 @@ interface MatrixRTCSdk {
       connection: Connection | null;
       membership: CallMembership;
       participant: LocalParticipant | RemoteParticipant | null;
+      speaking: boolean;
+      audioLevel: number;
     }[]
   >;
   /**
@@ -93,7 +95,17 @@ interface MatrixRTCSdk {
     connection: Connection | null;
     membership: CallMembership;
     participant: LocalParticipant | null;
+    speaking: boolean;
+    audioLevel: number;
   } | null>;
+  activeSpeakers$: Behavior<
+    {
+      connection: Connection | null;
+      membership: CallMembership;
+      participant: LocalParticipant | RemoteParticipant | null;
+      audioLevel: number;
+    }[]
+  >;
   /** Use the LocalMemberConnectionState returned from `join` for a more detailed connection state  */
   connected$: Behavior<boolean>;
   sendData?: (data: unknown) => Promise<void>;
@@ -302,6 +314,87 @@ export async function createMatrixRTCSdk(
 
   logger.info("createMatrixRTCSdk done");
 
+ const voiceActivityForMember$ = (member: {
+    userId: string;
+    membership$: Behavior<CallMembership>;
+  }): Observable<{ speaking: boolean; audioLevel: number }> =>
+    combineLatest([member.membership$, callViewModel.userMedia$]).pipe(
+      switchMap(([membership, mediaItems]) => {
+        const media = mediaItems.find(
+          (m) =>
+            m.userId === member.userId &&
+            m.id.startsWith(`${member.userId}:${membership.deviceId}:`),
+        );
+        return media
+          ? combineLatest([media.voiceActivity$, media.audioLevel$]).pipe(
+              map(([speaking, audioLevel]) => ({ speaking, audioLevel })),
+            )
+          : of({ speaking: false, audioLevel: 0 });
+      }),
+    );
+
+  const localMember$ = scope.behavior(
+    callViewModel.localMatrixLivekitMember$.pipe(
+      tap((member) => logger.info("localMatrixLivekitMember$ next: ", member)),
+      switchMap((member) => {
+        if (member === null) return of(null);
+        return combineLatest([
+          member.connection$,
+          member.membership$,
+          member.participant.value$,
+          voiceActivityForMember$(member),
+        ]).pipe(
+          map(([connection, membership, participant, voice]) => ({
+            connection,
+            membership,
+            participant,
+            speaking: voice.speaking,
+            audioLevel: voice.audioLevel,
+          })),
+        );
+      }),
+      tap((member) => logger.info("localMember$ next: ", member)),
+    ),
+  );
+
+  const remoteMembers$ = scope.behavior(
+    callViewModel.remoteMatrixLivekitMembers$.pipe(
+      switchMap((members) => {
+        const listOfMemberObservables = members.map((member) =>
+          combineLatest([
+            member.connection$,
+            member.membership$,
+            member.participant.value$,
+            voiceActivityForMember$(member),
+          ]).pipe(
+            map(([connection, membership, participant, voice]) => ({
+              connection,
+              membership,
+              participant,
+              speaking: voice.speaking,
+              audioLevel: voice.audioLevel,
+            })),
+            // using shareReplay instead of a Behavior here because the behavior would need
+            // a tricky scope.end() setup.
+            shareReplay({ bufferSize: 1, refCount: true }),
+          ),
+        );
+        return combineLatest(listOfMemberObservables);
+      }),
+    ),
+    [],
+  );
+  const activeSpeakers$ = scope.behavior(
+    combineLatest([localMember$, remoteMembers$]).pipe(
+      map(([local, remote]) =>
+        [...(local && local.speaking ? [local] : []), ...remote].filter(
+          (m) => m.speaking,
+        ),
+      ),
+    ),
+    [],
+  );
+
   return {
     join: (): void => {
       // first lets try making the widget sticky
@@ -317,53 +410,10 @@ export async function createMatrixRTCSdk(
       scope.end();
     },
     data$,
-    localMember$: scope.behavior(
-      callViewModel.localMatrixLivekitMember$.pipe(
-        tap((member) =>
-          logger.info("localMatrixLivekitMember$ next: ", member),
-        ),
-        switchMap((member) => {
-          if (member === null) return of(null);
-          return combineLatest([
-            member.connection$,
-            member.membership$,
-            member.participant.value$,
-          ]).pipe(
-            map(([connection, membership, participant]) => ({
-              connection,
-              membership,
-              participant,
-            })),
-          );
-        }),
-        tap((member) => logger.info("localMember$ next: ", member)),
-      ),
-    ),
+    localMember$,
     connected$: callViewModel.connected$,
-    remoteMembers$: scope.behavior(
-      callViewModel.remoteMatrixLivekitMembers$.pipe(
-        switchMap((members) => {
-          const listOfMemberObservables = members.map((member) =>
-            combineLatest([
-              member.connection$,
-              member.membership$,
-              member.participant.value$,
-            ]).pipe(
-              map(([connection, membership, participant]) => ({
-                connection,
-                membership,
-                participant,
-              })),
-              // using shareReplay instead of a Behavior here because the behavior would need
-              // a tricky scope.end() setup.
-              shareReplay({ bufferSize: 1, refCount: true }),
-            ),
-          );
-          return combineLatest(listOfMemberObservables);
-        }),
-      ),
-      [],
-    ),
+    remoteMembers$,
+    activeSpeakers$,
     sendData,
     sendRoomMessage,
   };

diff --git a/src/room/InCallView.tsx b/src/room/InCallView.tsx
@@ -28,7 +28,7 @@ import { useTranslation } from "react-i18next";
 import { Header, LeftNav, RightNav, RoomHeaderInfo } from "../Header";
 import { HeaderStyle, useUrlParams } from "../UrlParams";
 import { useCallViewKeyboardShortcuts } from "../useCallViewKeyboardShortcuts";
-import { widget } from "../widget";
+import { widget, ElementWidgetActions } from "../widget";
 import styles from "./InCallView.module.css";
 import { GridTile } from "../tile/GridTile";
 import { SettingsModal, defaultSettingsTab } from "../settings/SettingsModal";
@@ -142,6 +142,21 @@ export const ActiveCall: FC<ActiveCallProps> = (props) => {
 
     vm.leave$.pipe(scope.bind()).subscribe(props.onLeft);
 
+    // Forward currently speaking user IDs to the host client
+    if (widget) {
+      const widgetApi = widget.api;
+      vm.activeSpeakers$.pipe(scope.bind()).subscribe((speakers) => {
+        const userIds = speakers
+          .map((m) => m.userId)
+          .filter((id): id is string => typeof id === "string" && id !== "");
+        widgetApi.transport
+          .send(ElementWidgetActions.ActiveSpeakers, { userIds })
+          .catch((e) =>
+            rootLogger.error("Failed to send active speakers action", e),
+          );
+      });
+    }
+
     return (): void => {
       scope.end();
     };

diff --git a/src/state/CallViewModel/CallViewModel.ts b/src/state/CallViewModel/CallViewModel.ts
@@ -312,6 +312,8 @@ export interface CallViewModel {
   /** use the layout instead, this is just for the sdk export. */
   remoteMatrixLivekitMembers$: Behavior<RemoteMatrixLivekitMember[]>;
   localMatrixLivekitMember$: Behavior<LocalMatrixLivekitMember | null>;
+  /** All user media (local + remote) with their live speaking status */
+  userMedia$: Behavior<WrappedUserMediaViewModel[]>;
   /** List of participants raising their hand */
   handsRaised$: Behavior<Record<string, RaisedHandInfo>>;
   /** List of reactions. Keys are: membership.membershipId (currently predefined as: `${membershipEvent.userId}:${membershipEvent.deviceId}`)*/
@@ -353,6 +355,7 @@ export interface CallViewModel {
   showSpotlightIndicators$: Behavior<boolean>;
   showSpeakingIndicators$: Behavior<boolean>;
   showNameTags$: Behavior<boolean>;
+  activeSpeakers$: Behavior<UserMediaViewModel[]>;
   spotlightExpanded$: Behavior<boolean>;
   toggleSpotlightExpanded$: Behavior<(() => void) | null>;
   gridMode$: Behavior<GridMode>;
@@ -939,6 +942,24 @@ export function createCallViewModel$(
       }, undefined),
     ),
   );
+  // All active speakers in a call
+  const activeSpeakers$ = scope.behavior<UserMediaViewModel[]>(
+    userMedia$.pipe(
+      switchMap((mediaItems) =>
+        mediaItems.length === 0
+          ? of([])
+          : combineLatest(
+              mediaItems.map((m) =>
+                m.voiceActivity$.pipe(map((v) => [m, v] as const)),
+              ),
+            ),
+      ),
+      map((mediaItems) =>
+        mediaItems.filter(([, v]) => v).map(([m]) => m),
+      ),
+      distinctUntilChanged(shallowEquals),
+    ),
+  );
 
   const grid$ = scope.behavior<UserMediaViewModel[]>(
     userMedia$.pipe(
@@ -1783,6 +1804,7 @@ export function createCallViewModel$(
     setGridMode: setGridMode,
     layout$: layout$,
     localMatrixLivekitMember$,
+    userMedia$,
     remoteMatrixLivekitMembers$: scope.behavior(
       remoteMatrixLivekitMembers$.pipe(
         map((members) => members.value),
@@ -1803,6 +1825,7 @@ export function createCallViewModel$(
     showSpotlightIndicators$: showSpotlightIndicators$,
     showSpeakingIndicators$: showSpeakingIndicators$,
     showNameTags$,
+    activeSpeakers$,
     showHeader$: showHeader$,
     showFooter$: showFooter$,
     settingsOpen$: settingsOpen$,

diff --git a/src/state/media/RemoteUserMediaViewModel.ts b/src/state/media/RemoteUserMediaViewModel.ts
@@ -66,6 +66,20 @@ export function createRemoteUserMedia(
         ),
       ),
     ),
+    audioLevel$: scope.behavior(
+      pretendToBeDisconnected$.pipe(
+        switchMap((disconnected) =>
+          disconnected ? of(0) : baseUserMedia.audioLevel$,
+        ),
+      ),
+    ),
+    voiceActivity$: scope.behavior(
+      pretendToBeDisconnected$.pipe(
+        switchMap((disconnected) =>
+          disconnected ? of(false) : baseUserMedia.voiceActivity$,
+        ),
+      ),
+    ),
     videoEnabled$: scope.behavior(
       pretendToBeDisconnected$.pipe(
         switchMap((disconnected) =>

diff --git a/src/state/media/UserMediaViewModel.ts b/src/state/media/UserMediaViewModel.ts
@@ -29,6 +29,10 @@ import {
   type MemberMediaInputs,
   type BaseMemberMediaViewModel,
 } from "./MemberMediaViewModel";
+import {
+  observeSpeakingFromLevel$,
+  observeTrackAudioLevel$,
+} from "./observeAudioLevel";
 import { type RemoteUserMediaViewModel } from "./RemoteUserMediaViewModel";
 import { type ObservableScope } from "../ObservableScope";
 import { showConnectionStats } from "../../settings/settings";
@@ -45,6 +49,8 @@ export type UserMediaViewModel =
 export interface BaseUserMediaViewModel extends BaseMemberMediaViewModel {
   type: "user";
   speaking$: Behavior<boolean>;
+  audioLevel$: Behavior<number>;
+  voiceActivity$: Behavior<boolean>;
   audioEnabled$: Behavior<boolean>;
   videoEnabled$: Behavior<boolean>;
   videoFit$: Behavior<"cover" | "contain">;
@@ -106,6 +112,21 @@ export function createBaseUserMedia(
   >(undefined);
 
   const videoSize$ = videoSizeFromParticipant$(participant$);
+
+  // Client-side voice activity detection using the audio track itself
+  const audioLevel$ = scope.behavior(
+    participant$.pipe(
+      switchMap((p) => {
+        if (!p) return of(0);
+        return observeTrackAudioLevel$(
+          observeParticipantMedia(p).pipe(
+            map((m) => m.microphoneTrack?.track),
+          ),
+        );
+      }),
+    ),
+  );
+
   return {
     ...createMemberMedia(scope, {
       ...inputs,
@@ -125,6 +146,8 @@ export function createBaseUserMedia(
         ),
       ),
     ),
+    audioLevel$,
+    voiceActivity$: scope.behavior(observeSpeakingFromLevel$(audioLevel$)),
     audioEnabled$: scope.behavior(
       media$.pipe(map((m) => m?.microphoneTrack?.isMuted === false)),
     ),