feat(voice): underline the transcript by its packets

src/components/voice-demo.tsx

@@ -8,10 +8,10 @@ import "@moq/publish/element";
 import type MoqPublish from "@moq/publish/element";
 import "@moq/watch/element";
 import type MoqWatch from "@moq/watch/element";
-import { createEffect, createSignal, For, Index, onCleanup, Show } from "solid-js";
+import { createSignal, onCleanup, Show } from "solid-js";
 import { type AgentRecord, CallLog } from "./voice/call";
 import Cat from "./voice/cat";
-import Timeline from "./voice/timeline";
+import Transcript, { type Turn } from "./voice/transcript";
 
 type Turnstile = {
 	render: (
@@ -115,9 +115,6 @@ const REFUSALS = new Set<string>(["disabled", "unavailable", "challenge", "rate_
 
 type Phase = "idle" | "checking" | "challenge" | "admitting" | "connecting" | "live";
 
-// A run of speech by one side. `interim` is your words the recognizer may still revise.
-type Turn = { who: "you" | "ai"; text: string; interim: string };
-
 // Everything one call holds, so teardown is one place.
 type Call = {
 	gen: number;
@@ -187,10 +184,6 @@ function refusal(body: unknown): Outcome {
 	return typeof code === "string" && REFUSALS.has(code) ? (code as Outcome) : "unavailable";
 }
 
-function words(text: string): string[] {
-	return text.split(/\s+/).filter(Boolean);
-}
-
 function clock(secs: number): string {
 	return `${Math.floor(secs / 60)}:${String(secs % 60).padStart(2, "0")}`;
 }
@@ -205,7 +198,7 @@ export default function VoiceDemo() {
 	// The agent's decoded audio, before volume, for the cat to lip-sync to.
 	const [voice, setVoice] = createSignal<AudioNode>();
 	// The last call's measurements, kept after it ends so the reader can look them over.
-	const [timeline, setTimeline] = createSignal<CallLog>();
+	const [log, setLog] = createSignal<CallLog>();
 
 	// Bumped by every start and teardown, so an await that resumes after Stop sees it is stale
 	// and backs out instead of resurrecting the call.
@@ -214,14 +207,6 @@ export default function VoiceDemo() {
 	let widget: { api: Turnstile; id: string } | undefined;
 	let turnstileContainer!: HTMLDivElement;
 
-	// The transcript follows new words unless the reader has scrolled up to reread.
-	let transcriptList: HTMLOListElement | undefined;
-	let following = true;
-	createEffect(() => {
-		transcript();
-		if (transcriptList && following) transcriptList.scrollTop = transcriptList.scrollHeight;
-	});
-
 	const removeWidget = () => {
 		widget?.api.remove(widget.id);
 		widget = undefined;
@@ -243,7 +228,6 @@ export default function VoiceDemo() {
 		if (call) {
 			for (const timer of call.timers) clearTimeout(timer);
 			clearInterval(call.ticker);
-			call.log.ended = performance.now();
 			// The agent hangs up at once on this, instead of waiting for us to redial.
 			send(call, { label: "moq-transport", type: "session-ending" });
 			const unmount = () => setMounted((m) => (m === call ? undefined : m));
@@ -317,7 +301,7 @@ export default function VoiceDemo() {
 			"",
 		);
 		const call: Call = { gen, session, timers: [], epoch, messages: [], log: new CallLog() };
-		setTimeline(call.log);
+		setLog(call.log);
 		current = call;
 		// Queued until the publisher's broadcast exists, so the agent reads it first.
 		send(call, {
@@ -448,9 +432,16 @@ export default function VoiceDemo() {
 			effect.cleanup(() => setVoice((v) => (v === root ? undefined : v)));
 		});
 		effects.run((effect) => {
-			const media = effect.get(el.audio.out.timestamp);
-			const playing = !effect.get(el.audio.out.stalled);
-			call.log.playhead(performance.now(), media, playing);
+			call.log.playhead(effect.get(el.audio.out.timestamp));
+		});
+		// The decoder reports a frame only once playback nears it, so a reply streamed ahead shows up
+		// here first: the frames are already in the buffer.
+		effects.run((effect) => {
+			call.log.buffer(effect.get(el.audio.out.buffered));
+		});
+		// The ring stalls before the first reply, when one runs dry, and on a barge-in's reset.
+		effects.run((effect) => {
+			call.log.replying(!effect.get(el.audio.out.stalled), performance.now());
 		});
 		effects.spawn(async () => {
 			const arrivals = el.audio.arrivals();
@@ -459,7 +450,7 @@ export default function VoiceDemo() {
 				const next = await arrivals.next();
 				if (!next) return;
 				if (next.kind === "frame") call.log.received(next.timestamp, next.arrival, next.late);
-				else call.log.gap(next.timestamp, next.duration);
+				else call.log.gap(next.timestamp, next.duration, performance.now());
 			}
 		});
 		effects.run((effect) => {
@@ -558,40 +549,7 @@ export default function VoiceDemo() {
 
 			<div ref={turnstileContainer} classList={{ "mt-4": phase() === "challenge" }} />
 
-			<Show when={transcript().length > 0}>
-				<ol
-					ref={(el) => {
-						transcriptList = el;
-						following = true;
-					}}
-					onScroll={(e) => {
-						const el = e.currentTarget;
-						following = el.scrollHeight - el.scrollTop - el.clientHeight < 24;
-					}}
-					class="mt-4 mb-0 max-h-80 list-none space-y-3 overflow-y-auto p-0"
-					aria-label="Transcript"
-				>
-					<For each={transcript()}>
-						{(turn) => (
-							<li class="flex gap-3">
-								<span class="w-8 shrink-0 pt-0.5 text-right font-mono text-xs text-slate-500">
-									{turn.who === "ai" ? "AI" : "you"}
-								</span>
-								<p class="m-0 flex flex-wrap gap-x-1.5 font-bold">
-									<Index each={words(turn.text)}>
-										{(word) => <span class={turn.who === "ai" ? "text-green-400" : "text-slate-100"}>{word()}</span>}
-									</Index>
-									<Index each={words(turn.interim)}>{(word) => <span class="text-slate-500">{word()}</span>}</Index>
-								</p>
-							</li>
-						)}
-					</For>
-				</ol>
-			</Show>
-
-			<Show when={timeline()} keyed>
-				{(log) => <Timeline call={log} />}
-			</Show>
+			<Transcript turns={transcript()} log={log()} live={mounted() !== undefined} />
 
 			<Show when={mounted()} keyed>
 				{(call) => {

src/components/voice/call.test.ts

@@ -1,5 +1,5 @@
 import { describe, expect, test } from "bun:test";
-import { CallLog } from "./call";
+import { align, CallLog } from "./call";
 
 // Millisecond ticks, so media times read as written.
 const arrival = (capture: number, arrive: number) => ({
@@ -86,7 +86,7 @@ describe("publish", () => {
 		const [old, claimed] = [log.reply.find((f) => f.media === 2020), log.reply.find((f) => f.media === 2000)];
 		expect(old?.track).toBe("bot-audio");
 		expect(claimed?.track).toBe("bot-audio-1");
-		expect(claimed && log.downLateness(claimed)).toBe(40);
+		expect(claimed?.published).toBe(4000);
 	});
 
 	test("a claimed frame sheds the retired track's discard", () => {
@@ -107,7 +107,7 @@ describe("publish", () => {
 		expect(old?.discarded).toBe(true);
 		expect(claimed?.track).toBe("bot-audio-1");
 		expect(claimed?.discarded).toBeFalsy();
-		expect(claimed?.play).toBeUndefined();
+		expect(claimed?.played).toBeFalsy();
 	});
 
 	test("a frame of a track retired before it arrived never plays", () => {
@@ -125,10 +125,9 @@ describe("publish", () => {
 		const late = log.reply.find((f) => f.media === 3000);
 		expect(late?.track).toBe("bot-audio");
 		expect(late?.discarded).toBe(true);
-		expect(late?.play).toBeDefined();
 	});
 
-	test("a frame of a track retired before any reply arrived is placed at the retirement", () => {
+	test("a frame of a track retired before any reply arrived never plays", () => {
 		const log = new CallLog();
 		log.agent(arrival(0, 1010));
 		log.agent(publish("bot-audio", 0, 5000, 1100));
@@ -144,7 +143,6 @@ describe("publish", () => {
 
 		const frame = log.reply.find((f) => f.media === 0);
 		expect(frame?.discarded).toBe(true);
-		expect(frame?.play).toBeDefined();
 	});
 
 	test("an older publish still holds media past a newer, shorter one", () => {
@@ -156,24 +154,183 @@ describe("publish", () => {
 	});
 });
 
-describe("exchange", () => {
-	test("starts with the earlier of your latest statement and the agent's latest reply", () => {
+const word = (speaker: string, text: string, start: number, end: number, track = "mic") => ({
+	version: 2,
+	type: "word",
+	speaker,
+	track,
+	text,
+	timescale: 1000,
+	start_ticks: start,
+	end_ticks: end,
+});
+
+describe("sent", () => {
+	test("marks a frame that reached the agent after a later one", () => {
+		const log = new CallLog();
+		log.agent(arrival(0, 1010));
+		log.agent(arrival(40, 1050));
+		// Retransmitted: captured before the frame that beat it.
+		log.agent(arrival(20, 1200));
+
+		expect(log.sent({ text: "hi", start: 0, end: 60 })).toEqual(["delivered", "reordered", "delivered"]);
+	});
+
+	test("a frame the agent has not heard yet is in flight", () => {
+		const log = new CallLog();
+		log.agent(arrival(0, 1010));
+		log.agent(arrival(40, 1050));
+
+		expect(log.sent({ text: "hi", start: 0, end: 60 })).toEqual(["delivered", "flight", "delivered"]);
+	});
+
+	test("a frame that ends where a slot falls does not hold it", () => {
+		const log = new CallLog();
+		log.agent(arrival(0, 1010));
+		log.agent(arrival(40, 1050));
+
+		// The slots fall at 20 and 40: the missing frame and the one at 40.
+		expect(log.sent({ text: "hi", start: 10, end: 50 })).toEqual(["flight", "delivered"]);
+	});
+});
+
+describe("heard", () => {
+	test("buffered ahead of the playhead, delivered once played", () => {
+		const log = synced();
+		log.received(20, 120, false);
+		log.buffer([{ start: 40, end: 80 }]);
+		log.playhead(0);
+
+		expect(log.heard({ text: "hi", start: 0, end: 100, track: "bot-audio" })).toEqual([
+			"delivered",
+			"buffered",
+			"buffered",
+			"buffered",
+			"flight",
+		]);
+	});
+
+	test("a retired track's word is interrupted, arrived or not", () => {
 		const log = synced();
-		const record = (type: string, at: number, fields: object) => ({
+		log.received(20, 120, false);
+		log.agent({
 			version: 2,
-			type,
+			type: "retire",
+			track: "bot-audio",
 			timescale: 1000,
-			at_ns: at * 1e6,
-			...fields,
+			unplayed_ticks: 5000,
+			at_ns: 1200 * 1e6,
 		});
-		// The agent starts replying at 3000 local, then you interrupt: speech from 5000, confirmed at 5200.
-		log.agent(record("llm", 4000, { event: "request" }));
-		log.agent(record("vad", 6210, { speaking: true, media_ticks: 5200, confirm_ticks: 200 }));
-		expect(log.exchange()).toBe(3000);
-		log.agent(record("turn", 6250, { speaking: true, media_ticks: 5240 }));
-		expect(log.exchange()).toBe(3000);
-		// Its next reply is to your interruption, so the exchange now starts where you did.
-		log.agent(record("llm", 7000, { event: "request" }));
-		expect(log.exchange()).toBe(5000);
+
+		expect(log.heard({ text: "hi", start: 20, end: 60, track: "bot-audio" })).toEqual(["interrupted", "interrupted"]);
+	});
+
+	test("ignores another track's frame at the same media time", () => {
+		const log = synced();
+		log.received(2000, 2500, false);
+		log.agent(publish("bot-audio-1", 2000, 3000, 4000));
+
+		expect(log.heard({ text: "hi", start: 2000, end: 2020, track: "bot-audio-1" })).toEqual(["flight"]);
+	});
+
+	test("a retired track's gap is not the replacement's loss", () => {
+		const log = synced();
+		log.gap(200, 100, 150);
+		log.agent({
+			version: 2,
+			type: "retire",
+			track: "bot-audio",
+			timescale: 1000,
+			unplayed_ticks: 4800,
+			at_ns: 1200 * 1e6,
+		});
+		log.agent(publish("bot-audio-1", 200, 1000, 1300));
+		log.buffer([{ start: 200, end: 1000 }]);
+
+		expect(log.heard({ text: "hi", start: 200, end: 240, track: "bot-audio-1" })).toEqual(["buffered", "buffered"]);
+		expect(log.heard({ text: "hi", start: 200, end: 240, track: "bot-audio" })).toEqual(["interrupted", "interrupted"]);
+	});
+});
+
+describe("pending", () => {
+	const word = (type: string, text: string, start: number, end: number, at = 0) => ({
+		version: 2,
+		type,
+		speaker: "user",
+		timescale: 1000,
+		text,
+		start_ticks: start,
+		end_ticks: end,
+		at_ns: at * 1e6,
+	});
+
+	// Your mic from 1000, the agent hearing each frame 10 ms after it was captured, up to 1200.
+	function talking(): CallLog {
+		const log = new CallLog();
+		for (let capture = 1000; capture < 1200; capture += 20) log.agent(arrival(capture, capture + 10));
+		return log;
+	}
+
+	test("runs from your first packet to now, delivered then in flight", () => {
+		const packets = talking().pending(1300);
+		expect(packets?.length).toBe(15);
+		expect(packets?.slice(0, 10).every((p) => p === "delivered")).toBe(true);
+		expect(packets?.slice(10).every((p) => p === "flight")).toBe(true);
+	});
+
+	test("gives way to words, interim or final", () => {
+		const log = talking();
+		log.agent(word("interim", "hel", 1000, 1100, 1));
+		expect(log.pending(1300)?.length).toBe(10);
+		log.agent(word("word", "hello", 1000, 1160));
+		expect(log.interims).toEqual([]);
+		expect(log.pending(1300)?.length).toBe(7);
+	});
+
+	test("a newer interim result replaces the last one", () => {
+		const log = talking();
+		log.agent(word("interim", "hel", 1000, 1100, 1));
+		log.agent(word("interim", "hello", 1000, 1150, 2));
+		expect(log.interims.map((w) => w.text)).toEqual(["hello"]);
+	});
+
+	test("hides while the reply plays, and starts over when it stops", () => {
+		const log = talking();
+		log.agent(word("word", "hello", 1000, 1160));
+		log.replying(true, 1500);
+		expect(log.pending(2000)).toBeUndefined();
+		log.replying(false, 3000);
+		expect(log.pending(3200)?.length).toBe(10);
+	});
+});
+
+describe("align", () => {
+	test("matches across case and punctuation, and skips words either side left out", () => {
+		const log = new CallLog();
+		for (const [i, text] of ["hello", "there", "how", "are", "you"].entries()) {
+			log.agent(word("user", text, i * 100, i * 100 + 80));
+		}
+		// The transcript has "um", which was never timed; `voice.json` has "there", which the transcript lost.
+		const matched = align(["Hello,", "um", "how", "are", "you?"], log.userWords);
+
+		expect(matched.map((w) => w?.text)).toEqual(["hello", undefined, "how", "are", "you"]);
+	});
+
+	test("meets up again past a longer stretch only on two words in a row", () => {
+		const log = new CallLog();
+		for (const [i, text] of ["one", "two", "three", "four", "the", "dog", "hello", "world"].entries()) {
+			log.agent(word("user", text, i * 100, i * 100 + 80));
+		}
+		const matched = align(["the", "cat", "hello", "world"], log.userWords);
+
+		expect(matched.map((w) => w?.text)).toEqual([undefined, undefined, "hello", "world"]);
+	});
+
+	test("a restated window does not repeat a word", () => {
+		const log = new CallLog();
+		log.agent(word("user", "hello", 0, 80));
+		log.agent(word("user", "hello", 0, 80));
+
+		expect(log.userWords).toHaveLength(1);
 	});
 });