Skip to content
Open
Show file tree
Hide file tree
Changes from all commits
Commits
File filter

Filter by extension

Filter by extension

Conversations
Failed to load comments.
Loading
Jump to
Jump to file
Failed to load files.
Loading
Diff view
Diff view
90 changes: 90 additions & 0 deletions src/components/playground/DataChannelLog.tsx
Original file line number Diff line number Diff line change
@@ -0,0 +1,90 @@
import React, { useMemo } from "react";

export type DataChannelLogEntry = {
id: string;
timestamp: number;
topic?: string;
participantIdentity?: string;
participantName?: string;
kind: "reliable" | "lossy" | "unknown";
payload: string;
payloadFormat: "json" | "text" | "binary";
};

type DataChannelLogProps = {
entries: DataChannelLogEntry[];
onClear: () => void;
};

const noEntriesMessage = "No data events received yet.";

export const DataChannelLog: React.FC<DataChannelLogProps> = ({
entries,
onClear,
}: DataChannelLogProps) => {
const sortedEntries = useMemo(
() => [...entries].sort((a, b) => a.timestamp - b.timestamp),
[entries],
);

return (
<div className="flex flex-col h-full w-full text-xs text-gray-300">
<div className="flex items-center justify-between">
<span className="text-[10px] uppercase tracking-widest text-gray-500">
Room data events
</span>
<button
className="text-[10px] uppercase tracking-widest text-gray-500 hover:text-gray-300 active:text-gray-200 transition-colors"
onClick={onClear}
type="button"
>
Clear
</button>
</div>
<div className="mt-2 flex-1 overflow-y-auto pr-1">
{sortedEntries.length === 0 ? (
<div className="flex h-full items-center justify-center text-gray-600">
{noEntriesMessage}
</div>
) : (
<ul className="flex flex-col gap-2">
{sortedEntries.map((entry: DataChannelLogEntry) => (
<li
key={entry.id}
className="flex flex-col gap-2 rounded-sm border border-gray-800 bg-gray-950/70 p-3"
>
<div className="flex flex-wrap items-center gap-x-3 gap-y-1 text-[11px] text-gray-400">
<span className="font-medium text-gray-300">
{new Date(entry.timestamp).toLocaleTimeString()}
</span>
<span className="rounded-sm bg-gray-900 px-2 py-0.5 text-[10px] uppercase tracking-wider text-gray-400">
{entry.kind}
</span>
{entry.topic && (
<span className="rounded-sm bg-gray-900 px-2 py-0.5 text-[10px] uppercase tracking-wider text-gray-400">
{entry.topic}
</span>
)}
{entry.payloadFormat !== "text" && (
<span className="rounded-sm bg-gray-900 px-2 py-0.5 text-[10px] uppercase tracking-wider text-gray-400">
{entry.payloadFormat}
</span>
)}
{(entry.participantName || entry.participantIdentity) && (
<span className="text-gray-500">
{entry.participantName || entry.participantIdentity}
</span>
)}
</div>
<pre className="max-h-48 overflow-y-auto whitespace-pre-wrap break-words rounded-sm bg-black/40 p-2 font-mono text-[11px] leading-relaxed text-gray-200">
{entry.payload}
</pre>
</li>
))}
</ul>
)}
</div>
</div>
);
};

181 changes: 151 additions & 30 deletions src/components/playground/Playground.tsx
Original file line number Diff line number Diff line change
Expand Up @@ -26,13 +26,24 @@ import {
useRoomContext,
useParticipantAttributes,
} from "@livekit/components-react";
import { ConnectionState, LocalParticipant, Track } from "livekit-client";
import {
ConnectionState,
DataPacket_Kind,
LocalParticipant,
Participant,
RoomEvent,
Track,
} from "livekit-client";
import { QRCodeSVG } from "qrcode.react";
import { ReactNode, useCallback, useEffect, useMemo, useState } from "react";
import tailwindTheme from "../../lib/tailwindTheme.preval";
import { EditableNameValueRow } from "@/components/config/NameValueRow";
import { AttributesInspector } from "@/components/config/AttributesInspector";
import { RpcPanel } from "./RpcPanel";
import {
DataChannelLog,
DataChannelLogEntry,
} from "./DataChannelLog";

export interface PlaygroundMeta {
name: string;
Expand All @@ -55,6 +66,7 @@ export default function Playground({
const { config, setUserSettings } = useConfig();
const { name } = useRoomInfo();
const [transcripts, setTranscripts] = useState<ChatMessageType[]>([]);
const [dataEvents, setDataEvents] = useState<DataChannelLogEntry[]>([]);
const { localParticipant } = useLocalParticipant();

const voiceAssistant = useVoiceAssistant();
Expand All @@ -66,6 +78,9 @@ export default function Playground({
const [rpcMethod, setRpcMethod] = useState("");
const [rpcPayload, setRpcPayload] = useState("");
const [showRpc, setShowRpc] = useState(false);
const handleClearDataEvents = useCallback(() => {
setDataEvents([]);
}, []);

useEffect(() => {
if (roomState === ConnectionState.Connected) {
Expand Down Expand Up @@ -93,31 +108,105 @@ export default function Playground({
({ source }) => source === Track.Source.Microphone,
);

const onDataReceived = useCallback(
(msg: any) => {
if (msg.topic === "transcription") {
const decoded = JSON.parse(
new TextDecoder("utf-8").decode(msg.payload),
);
let timestamp = new Date().getTime();
if ("timestamp" in decoded && decoded.timestamp > 0) {
timestamp = decoded.timestamp;
const onDataReceived = useCallback((msg: any) => {
if (msg.topic === "transcription") {
const decoded = JSON.parse(
new TextDecoder("utf-8").decode(msg.payload),
);
let timestamp = new Date().getTime();
if ("timestamp" in decoded && decoded.timestamp > 0) {
timestamp = decoded.timestamp;
}
setTranscripts((prev) => [
...prev,
{
name: "You",
message: decoded.text,
timestamp: timestamp,
isSelf: true,
},
]);
}
}, []);

useDataChannel(onDataReceived);
useEffect(() => {
if (!room) {
return;
}

const decoder = new TextDecoder("utf-8");

const handleDataReceived = (
payload: Uint8Array,
participant?: Participant,
kind?: DataPacket_Kind,
topic?: string,
) => {
const timestamp = Date.now();
let payloadText = "";
let payloadFormat: DataChannelLogEntry["payloadFormat"] = "binary";

try {
payloadText = decoder.decode(payload);
if (payloadText.length > 0) {
payloadFormat = "text";
const trimmed = payloadText.trim();
if (
(trimmed.startsWith("{") && trimmed.endsWith("}")) ||
(trimmed.startsWith("[") && trimmed.endsWith("]"))
) {
try {
const parsed = JSON.parse(payloadText);
payloadText = JSON.stringify(parsed, null, 2);
payloadFormat = "json";
} catch {
payloadFormat = "text";
}
}
}
setTranscripts([
...transcripts,
} catch (error) {
payloadText = Array.from(payload)
.map((byte) => byte.toString(16).padStart(2, "0"))
.join(" ");
payloadFormat = "binary";
}

const kindLabel: DataChannelLogEntry["kind"] =
kind === DataPacket_Kind.RELIABLE
? "reliable"
: kind === DataPacket_Kind.LOSSY
? "lossy"
: "unknown";

setDataEvents((prev) => {
const next = [
...prev,
{
name: "You",
message: decoded.text,
timestamp: timestamp,
isSelf: true,
id: `${timestamp}-${Math.random().toString(16).slice(2)}`,
timestamp,
topic,
participantIdentity: participant?.identity,
participantName: participant?.name,
kind: kindLabel,
payload: payloadText,
payloadFormat,
},
]);
}
},
[transcripts],
);
];
const maxEntries = 200;
if (next.length > maxEntries) {
return next.slice(next.length - maxEntries);
}
return next;
});
};

useDataChannel(onDataReceived);
room.on(RoomEvent.DataReceived, handleDataReceived);

return () => {
room.off(RoomEvent.DataReceived, handleDataReceived);
};
}, [room]);

const videoTileContent = useMemo(() => {
const videoFitClassName = `object-${config.video_fit || "contain"}`;
Expand Down Expand Up @@ -228,6 +317,14 @@ export default function Playground({
voiceAssistant.audioTrack,
voiceAssistant.agent,
]);
const dataEventsContent = useMemo(() => {
return (
<DataChannelLog
entries={dataEvents}
onClear={handleClearDataEvents}
/>
);
}, [dataEvents, handleClearDataEvents]);

const handleRpcCall = useCallback(async () => {
if (!voiceAssistant.agent || !room) {
Expand Down Expand Up @@ -536,6 +633,11 @@ export default function Playground({
});
}

mobileTabs.push({
title: "Events",
content: <div className="h-full">{dataEventsContent}</div>,
});

mobileTabs.push({
title: "Settings",
content: (
Expand Down Expand Up @@ -574,8 +676,21 @@ export default function Playground({
initialTab={mobileTabs.length - 1}
/>
</div>

{/* Data Events - Left Column */}
{config.settings.outputs.data_events && (
<PlaygroundTile
title="Data Events"
className="hidden lg:flex h-full basis-3/12 flex-col"
childrenClassName="!items-stretch"
>
{dataEventsContent}
</PlaygroundTile>
)}

{/* Video/Audio - Center-Left Column */}
<div
className={`flex-col grow basis-1/2 gap-4 h-full hidden lg:${
className={`flex-col grow basis-1/3 gap-4 h-full hidden lg:${
!config.settings.outputs.audio && !config.settings.outputs.video
? "hidden"
: "flex"
Expand All @@ -601,18 +716,24 @@ export default function Playground({
)}
</div>

{/* Chat - Center-Right Column */}
{config.settings.chat && (
<PlaygroundTile
title="Chat"
className="h-full grow basis-1/4 hidden lg:flex"
>
{chatTileContent}
</PlaygroundTile>
<div className="hidden h-full basis-3/12 flex-col gap-4 lg:flex">
<PlaygroundTile
title="Chat"
className="h-full grow"
childrenClassName="!items-stretch"
>
{chatTileContent}
</PlaygroundTile>
</div>
)}

{/* Settings - Right Column */}
<PlaygroundTile
padding={false}
backgroundColor="gray-950"
className="h-full w-full basis-1/4 items-start overflow-y-auto hidden max-w-[480px] lg:flex"
className="h-full w-full basis-3/12 items-start overflow-y-auto hidden lg:flex"
childrenClassName="h-full grow items-start"
>
{settingsTileContent}
Expand Down
9 changes: 7 additions & 2 deletions src/components/playground/SettingsDropdown.tsx
Original file line number Diff line number Diff line change
Expand Up @@ -31,6 +31,11 @@ const settingsDropdown: SettingValue[] = [
type: "outputs",
key: "audio",
},
{
title: "Show data events",
type: "outputs",
key: "data_events",
},

{
title: "---",
Expand Down Expand Up @@ -68,7 +73,7 @@ export const SettingsDropdown = () => {
const key = setting.key as "camera" | "mic" | "screen";
return config.settings.inputs[key];
} else if (setting.type === "outputs") {
const key = setting.key as "video" | "audio";
const key = setting.key as "video" | "audio" | "data_events";
return config.settings.outputs[key];
}

Expand All @@ -85,7 +90,7 @@ export const SettingsDropdown = () => {
} else if (setting.type === "inputs") {
newSettings.inputs[setting.key as "camera" | "mic" | "screen"] = newValue;
} else if (setting.type === "outputs") {
newSettings.outputs[setting.key as "video" | "audio"] = newValue;
newSettings.outputs[setting.key as "video" | "audio" | "data_events"] = newValue;
}
setUserSettings(newSettings);
};
Expand Down
Loading