diff --git a/src/components/voice-demo.tsx b/src/components/voice-demo.tsx index bddfc5e..990cbbb 100644 --- a/src/components/voice-demo.tsx +++ b/src/components/voice-demo.tsx @@ -8,10 +8,10 @@ import "@moq/publish/element"; import type MoqPublish from "@moq/publish/element"; import "@moq/watch/element"; import type MoqWatch from "@moq/watch/element"; -import { createEffect, createSignal, For, Index, onCleanup, Show } from "solid-js"; +import { createSignal, onCleanup, Show } from "solid-js"; import { type AgentRecord, CallLog } from "./voice/call"; import Cat from "./voice/cat"; -import Timeline from "./voice/timeline"; +import Transcript, { type Turn } from "./voice/transcript"; type Turnstile = { render: ( @@ -115,9 +115,6 @@ const REFUSALS = new Set(["disabled", "unavailable", "challenge", "rate_ type Phase = "idle" | "checking" | "challenge" | "admitting" | "connecting" | "live"; -// A run of speech by one side. `interim` is your words the recognizer may still revise. -type Turn = { who: "you" | "ai"; text: string; interim: string }; - // Everything one call holds, so teardown is one place. type Call = { gen: number; @@ -187,10 +184,6 @@ function refusal(body: unknown): Outcome { return typeof code === "string" && REFUSALS.has(code) ? (code as Outcome) : "unavailable"; } -function words(text: string): string[] { - return text.split(/\s+/).filter(Boolean); -} - function clock(secs: number): string { return `${Math.floor(secs / 60)}:${String(secs % 60).padStart(2, "0")}`; } @@ -205,7 +198,7 @@ export default function VoiceDemo() { // The agent's decoded audio, before volume, for the cat to lip-sync to. const [voice, setVoice] = createSignal(); // The last call's measurements, kept after it ends so the reader can look them over. - const [timeline, setTimeline] = createSignal(); + const [log, setLog] = createSignal(); // Bumped by every start and teardown, so an await that resumes after Stop sees it is stale // and backs out instead of resurrecting the call. @@ -214,14 +207,6 @@ export default function VoiceDemo() { let widget: { api: Turnstile; id: string } | undefined; let turnstileContainer!: HTMLDivElement; - // The transcript follows new words unless the reader has scrolled up to reread. - let transcriptList: HTMLOListElement | undefined; - let following = true; - createEffect(() => { - transcript(); - if (transcriptList && following) transcriptList.scrollTop = transcriptList.scrollHeight; - }); - const removeWidget = () => { widget?.api.remove(widget.id); widget = undefined; @@ -243,7 +228,6 @@ export default function VoiceDemo() { if (call) { for (const timer of call.timers) clearTimeout(timer); clearInterval(call.ticker); - call.log.ended = performance.now(); // The agent hangs up at once on this, instead of waiting for us to redial. send(call, { label: "moq-transport", type: "session-ending" }); const unmount = () => setMounted((m) => (m === call ? undefined : m)); @@ -317,7 +301,7 @@ export default function VoiceDemo() { "", ); const call: Call = { gen, session, timers: [], epoch, messages: [], log: new CallLog() }; - setTimeline(call.log); + setLog(call.log); current = call; // Queued until the publisher's broadcast exists, so the agent reads it first. send(call, { @@ -448,9 +432,16 @@ export default function VoiceDemo() { effect.cleanup(() => setVoice((v) => (v === root ? undefined : v))); }); effects.run((effect) => { - const media = effect.get(el.audio.out.timestamp); - const playing = !effect.get(el.audio.out.stalled); - call.log.playhead(performance.now(), media, playing); + call.log.playhead(effect.get(el.audio.out.timestamp)); + }); + // The decoder reports a frame only once playback nears it, so a reply streamed ahead shows up + // here first: the frames are already in the buffer. + effects.run((effect) => { + call.log.buffer(effect.get(el.audio.out.buffered)); + }); + // The ring stalls before the first reply, when one runs dry, and on a barge-in's reset. + effects.run((effect) => { + call.log.replying(!effect.get(el.audio.out.stalled), performance.now()); }); effects.spawn(async () => { const arrivals = el.audio.arrivals(); @@ -459,7 +450,7 @@ export default function VoiceDemo() { const next = await arrivals.next(); if (!next) return; if (next.kind === "frame") call.log.received(next.timestamp, next.arrival, next.late); - else call.log.gap(next.timestamp, next.duration); + else call.log.gap(next.timestamp, next.duration, performance.now()); } }); effects.run((effect) => { @@ -558,40 +549,7 @@ export default function VoiceDemo() {
- 0}> -
    { - transcriptList = el; - following = true; - }} - onScroll={(e) => { - const el = e.currentTarget; - following = el.scrollHeight - el.scrollTop - el.clientHeight < 24; - }} - class="mt-4 mb-0 max-h-80 list-none space-y-3 overflow-y-auto p-0" - aria-label="Transcript" - > - - {(turn) => ( -
  1. - - {turn.who === "ai" ? "AI" : "you"} - -

    - - {(word) => {word()}} - - {(word) => {word()}} -

    -
    2. - )} -     -
- - - - {(log) => } - + {(call) => { diff --git a/src/components/voice/call.test.ts b/src/components/voice/call.test.ts index 5f5f865..6ff80f0 100644 --- a/src/components/voice/call.test.ts +++ b/src/components/voice/call.test.ts @@ -1,5 +1,5 @@ import { describe, expect, test } from "bun:test"; -import { CallLog } from "./call"; +import { align, CallLog } from "./call"; // Millisecond ticks, so media times read as written. const arrival = (capture: number, arrive: number) => ({ @@ -86,7 +86,7 @@ describe("publish", () => { const [old, claimed] = [log.reply.find((f) => f.media === 2020), log.reply.find((f) => f.media === 2000)]; expect(old?.track).toBe("bot-audio"); expect(claimed?.track).toBe("bot-audio-1"); - expect(claimed && log.downLateness(claimed)).toBe(40); + expect(claimed?.published).toBe(4000); }); test("a claimed frame sheds the retired track's discard", () => { @@ -107,7 +107,7 @@ describe("publish", () => { expect(old?.discarded).toBe(true); expect(claimed?.track).toBe("bot-audio-1"); expect(claimed?.discarded).toBeFalsy(); - expect(claimed?.play).toBeUndefined(); + expect(claimed?.played).toBeFalsy(); }); test("a frame of a track retired before it arrived never plays", () => { @@ -125,10 +125,9 @@ describe("publish", () => { const late = log.reply.find((f) => f.media === 3000); expect(late?.track).toBe("bot-audio"); expect(late?.discarded).toBe(true); - expect(late?.play).toBeDefined(); }); - test("a frame of a track retired before any reply arrived is placed at the retirement", () => { + test("a frame of a track retired before any reply arrived never plays", () => { const log = new CallLog(); log.agent(arrival(0, 1010)); log.agent(publish("bot-audio", 0, 5000, 1100)); @@ -144,7 +143,6 @@ describe("publish", () => { const frame = log.reply.find((f) => f.media === 0); expect(frame?.discarded).toBe(true); - expect(frame?.play).toBeDefined(); }); test("an older publish still holds media past a newer, shorter one", () => { @@ -156,24 +154,183 @@ describe("publish", () => { }); }); -describe("exchange", () => { - test("starts with the earlier of your latest statement and the agent's latest reply", () => { +const word = (speaker: string, text: string, start: number, end: number, track = "mic") => ({ + version: 2, + type: "word", + speaker, + track, + text, + timescale: 1000, + start_ticks: start, + end_ticks: end, +}); + +describe("sent", () => { + test("marks a frame that reached the agent after a later one", () => { + const log = new CallLog(); + log.agent(arrival(0, 1010)); + log.agent(arrival(40, 1050)); + // Retransmitted: captured before the frame that beat it. + log.agent(arrival(20, 1200)); + + expect(log.sent({ text: "hi", start: 0, end: 60 })).toEqual(["delivered", "reordered", "delivered"]); + }); + + test("a frame the agent has not heard yet is in flight", () => { + const log = new CallLog(); + log.agent(arrival(0, 1010)); + log.agent(arrival(40, 1050)); + + expect(log.sent({ text: "hi", start: 0, end: 60 })).toEqual(["delivered", "flight", "delivered"]); + }); + + test("a frame that ends where a slot falls does not hold it", () => { + const log = new CallLog(); + log.agent(arrival(0, 1010)); + log.agent(arrival(40, 1050)); + + // The slots fall at 20 and 40: the missing frame and the one at 40. + expect(log.sent({ text: "hi", start: 10, end: 50 })).toEqual(["flight", "delivered"]); + }); +}); + +describe("heard", () => { + test("buffered ahead of the playhead, delivered once played", () => { + const log = synced(); + log.received(20, 120, false); + log.buffer([{ start: 40, end: 80 }]); + log.playhead(0); + + expect(log.heard({ text: "hi", start: 0, end: 100, track: "bot-audio" })).toEqual([ + "delivered", + "buffered", + "buffered", + "buffered", + "flight", + ]); + }); + + test("a retired track's word is interrupted, arrived or not", () => { const log = synced(); - const record = (type: string, at: number, fields: object) => ({ + log.received(20, 120, false); + log.agent({ version: 2, - type, + type: "retire", + track: "bot-audio", timescale: 1000, - at_ns: at * 1e6, - ...fields, + unplayed_ticks: 5000, + at_ns: 1200 * 1e6, }); - // The agent starts replying at 3000 local, then you interrupt: speech from 5000, confirmed at 5200. - log.agent(record("llm", 4000, { event: "request" })); - log.agent(record("vad", 6210, { speaking: true, media_ticks: 5200, confirm_ticks: 200 })); - expect(log.exchange()).toBe(3000); - log.agent(record("turn", 6250, { speaking: true, media_ticks: 5240 })); - expect(log.exchange()).toBe(3000); - // Its next reply is to your interruption, so the exchange now starts where you did. - log.agent(record("llm", 7000, { event: "request" })); - expect(log.exchange()).toBe(5000); + + expect(log.heard({ text: "hi", start: 20, end: 60, track: "bot-audio" })).toEqual(["interrupted", "interrupted"]); + }); + + test("ignores another track's frame at the same media time", () => { + const log = synced(); + log.received(2000, 2500, false); + log.agent(publish("bot-audio-1", 2000, 3000, 4000)); + + expect(log.heard({ text: "hi", start: 2000, end: 2020, track: "bot-audio-1" })).toEqual(["flight"]); + }); + + test("a retired track's gap is not the replacement's loss", () => { + const log = synced(); + log.gap(200, 100, 150); + log.agent({ + version: 2, + type: "retire", + track: "bot-audio", + timescale: 1000, + unplayed_ticks: 4800, + at_ns: 1200 * 1e6, + }); + log.agent(publish("bot-audio-1", 200, 1000, 1300)); + log.buffer([{ start: 200, end: 1000 }]); + + expect(log.heard({ text: "hi", start: 200, end: 240, track: "bot-audio-1" })).toEqual(["buffered", "buffered"]); + expect(log.heard({ text: "hi", start: 200, end: 240, track: "bot-audio" })).toEqual(["interrupted", "interrupted"]); + }); +}); + +describe("pending", () => { + const word = (type: string, text: string, start: number, end: number, at = 0) => ({ + version: 2, + type, + speaker: "user", + timescale: 1000, + text, + start_ticks: start, + end_ticks: end, + at_ns: at * 1e6, + }); + + // Your mic from 1000, the agent hearing each frame 10 ms after it was captured, up to 1200. + function talking(): CallLog { + const log = new CallLog(); + for (let capture = 1000; capture < 1200; capture += 20) log.agent(arrival(capture, capture + 10)); + return log; + } + + test("runs from your first packet to now, delivered then in flight", () => { + const packets = talking().pending(1300); + expect(packets?.length).toBe(15); + expect(packets?.slice(0, 10).every((p) => p === "delivered")).toBe(true); + expect(packets?.slice(10).every((p) => p === "flight")).toBe(true); + }); + + test("gives way to words, interim or final", () => { + const log = talking(); + log.agent(word("interim", "hel", 1000, 1100, 1)); + expect(log.pending(1300)?.length).toBe(10); + log.agent(word("word", "hello", 1000, 1160)); + expect(log.interims).toEqual([]); + expect(log.pending(1300)?.length).toBe(7); + }); + + test("a newer interim result replaces the last one", () => { + const log = talking(); + log.agent(word("interim", "hel", 1000, 1100, 1)); + log.agent(word("interim", "hello", 1000, 1150, 2)); + expect(log.interims.map((w) => w.text)).toEqual(["hello"]); + }); + + test("hides while the reply plays, and starts over when it stops", () => { + const log = talking(); + log.agent(word("word", "hello", 1000, 1160)); + log.replying(true, 1500); + expect(log.pending(2000)).toBeUndefined(); + log.replying(false, 3000); + expect(log.pending(3200)?.length).toBe(10); + }); +}); + +describe("align", () => { + test("matches across case and punctuation, and skips words either side left out", () => { + const log = new CallLog(); + for (const [i, text] of ["hello", "there", "how", "are", "you"].entries()) { + log.agent(word("user", text, i * 100, i * 100 + 80)); + } + // The transcript has "um", which was never timed; `voice.json` has "there", which the transcript lost. + const matched = align(["Hello,", "um", "how", "are", "you?"], log.userWords); + + expect(matched.map((w) => w?.text)).toEqual(["hello", undefined, "how", "are", "you"]); + }); + + test("meets up again past a longer stretch only on two words in a row", () => { + const log = new CallLog(); + for (const [i, text] of ["one", "two", "three", "four", "the", "dog", "hello", "world"].entries()) { + log.agent(word("user", text, i * 100, i * 100 + 80)); + } + const matched = align(["the", "cat", "hello", "world"], log.userWords); + + expect(matched.map((w) => w?.text)).toEqual([undefined, undefined, "hello", "world"]); + }); + + test("a restated window does not repeat a word", () => { + const log = new CallLog(); + log.agent(word("user", "hello", 0, 80)); + log.agent(word("user", "hello", 0, 80)); + + expect(log.userWords).toHaveLength(1); }); }); diff --git a/src/components/voice/call.ts b/src/components/voice/call.ts index 967d975..e484e60 100644 --- a/src/components/voice/call.ts +++ b/src/components/voice/call.ts @@ -1,4 +1,4 @@ -// Everything measured about one live call, on this page's clock (`performance.now()` milliseconds). +// Everything measured about one live call's audio, on this page's clock (`performance.now()` ms). // // Three sources feed it: // - Your mic frames: `@moq/publish` stamps each with its capture time on this clock, so the media @@ -10,23 +10,22 @@ // The agent's clock is placed on ours NTP-style, from the fastest transit each way: request frames // bound the offset from one side and reply frames from the other, so it needs no clock agreement. -// A frame is late for WebRTC once it arrives this much later than the fastest one did: the initial -// target of libwebrtc's audio jitter buffer (NetEQ's kStartDelayMs in -// modules/audio_coding/neteq/delay_manager.cc), which only grows after it has already dropped audio. -export const WEBRTC_MS = 80; - -// The agent's Opus frames, and the duration drawn for a reply frame. +// The agent's Opus frames, and the span each packet of a word covers. export const FRAME_MS = 20; -// How much history to keep, in milliseconds. -const KEEP_MS = 60_000; - /** A `voice.json` record. Unknown types are skipped; see moq.pro agent/VOICE.md. */ export type AgentRecord = { version?: number; type?: string; [key: string]: unknown }; +/** + * One frame of a word, as far as it has travelled. `delivered` is at the agent for your words and + * played for its own; `buffered` is a reply frame here ahead of the playhead. + */ +export type Packet = "delivered" | "buffered" | "reordered" | "flight" | "interrupted" | "lost"; + export interface MicFrame { capture: number; // local arrive: number; // agent + reordered: boolean; // reached the agent after a frame captured later: retransmitted or reordered } export interface ReplyFrame { @@ -37,32 +36,17 @@ export interface ReplyFrame { // playhead was, so two tracks can hold the same media time. track?: string; published?: number; // agent - play?: number; // local, frozen once it played or was interrupted + played?: boolean; discarded?: boolean; } export interface Word { text: string; - start: number; + start: number; // local capture time for yours, reply media time for the agent's end: number; - at?: number; // agent, when STT settled on it track?: string; } -export interface Mark { - at: number; // agent - kind: "vad" | "turn" | "llm" | "tts" | "retire"; - label: string; - media?: number; // local, for user-side marks: what the agent had heard -} - -export interface Span { - start: number; // agent - end?: number; // agent, open while running - kind: "llm" | "tts"; - first?: number; // agent: first text or audio -} - interface Publish { track: string; start: number; // reply media @@ -84,40 +68,61 @@ function agentMs(record: AgentRecord, field: string): number | undefined { } // Insert keeping `list` sorted by `key`. Records arrive in about time order, so scan from the back. -function insert(list: T[], item: T, key: (t: T) => number) { +// Returns where it went. +function insert(list: T[], item: T, key: (t: T) => number): number { let i = list.length; while (i > 0 && key(list[i - 1]) > key(item)) i--; list.splice(i, 0, item); + return i; +} + +// The first index whose key is above `value`, in a list sorted by `key`. +function above(list: T[], value: number, key: (t: T) => number): number { + let lo = 0; + let hi = list.length; + while (lo < hi) { + const mid = (lo + hi) >> 1; + if (key(list[mid]) <= value) lo = mid + 1; + else hi = mid; + } + return lo; +} + +// Each FRAME_MS of a word, the middle of the frame it should fall in. +function slots(word: Word): number[] { + const count = Math.max(1, Math.round((word.end - word.start) / FRAME_MS)); + return Array.from({ length: count }, (_, i) => word.start + (i + 0.5) * FRAME_MS); } export class CallLog { mic: MicFrame[] = []; reply: ReplyFrame[] = []; - gaps: { media: number; duration: number }[] = []; + gaps: { media: number; duration: number; track?: string }[] = []; + // Reply media the player holds but has not played: received, decoded or not. + buffered: { start: number; end: number }[] = []; + // In the order the agent settled on them, which is the order they were said. userWords: Word[] = []; - interims: Word[] = []; agentWords: Word[] = []; - marks: Mark[] = []; - spans: Span[] = []; - // User speech as the VAD decided it, in local capture time. - speech: { start: number; end?: number }[] = []; + // The STT's latest interim result for your words since its last final ones. + interims: Word[] = []; + #interimAt?: number; + // Whether the reply is playing, and when it last stopped. + #replying = false; + #replyStopped?: number; publishes: Publish[] = []; // The longest publish, which bounds how far back one can still hold a media time. #longest = 0; - // Retired tracks and when they retired, on the local clock: their later arrivals never play. - #retired = new Map(); + // Tracks a barge-in retired: their later arrivals never play. + #retired = new Set(); + // A reconnect restates the window, so a word can come twice. + #seen = new Set(); + // The reply's media time only moves forward, across track changes too. + #head?: number; // The fastest transit each way, each including the clock offset. minUp?: number; // agent - local minDown?: number; // local - agent - // Where the reply's playhead was, and when. - #anchor?: { local: number; media: number }; - #playing = false; - #head?: { now: number; media: number | undefined }; - - ended?: number; // local - /** Agent time minus local time, once both directions have been measured. */ get offset(): number | undefined { if (this.minUp === undefined || this.minDown === undefined) return undefined; @@ -130,23 +135,6 @@ export class CallLog { return (this.minUp + this.minDown) / 2; } - /** An agent time on the local clock, once the offset is known. */ - local(agent: number): number | undefined { - const offset = this.offset; - return offset === undefined ? undefined : agent - offset; - } - - /** How much later than the fastest a request frame reached the agent. */ - upLateness(frame: MicFrame): number { - return frame.arrive - frame.capture - (this.minUp ?? 0); - } - - /** How much later than the fastest a reply frame reached us, once its publish is known. */ - downLateness(frame: ReplyFrame): number | undefined { - if (frame.published === undefined) return undefined; - return frame.arrive - frame.published - (this.minDown ?? 0); - } - /** Ingest one `voice.json` record. */ agent(record: AgentRecord) { if (record.version !== 2) return; @@ -155,58 +143,42 @@ export class CallLog { const capture = ms(record, "media_ticks"); const arrive = agentMs(record, "arrival_ns"); if (capture === undefined || arrive === undefined) return; - insert(this.mic, { capture, arrive }, (f) => f.capture); + const frame = { capture, arrive, reordered: false }; + const at = insert(this.mic, frame, (f) => f.capture); + for (let i = at + 1; i < this.mic.length && !frame.reordered; i++) { + frame.reordered = this.mic[i].arrive < arrive; + } const up = arrive - capture; if (this.minUp === undefined || up < this.minUp) this.minUp = up; return; } case "word": { - const word = this.#word(record); - if (!word) return; + const start = ms(record, "start_ticks"); + const end = ms(record, "end_ticks"); + if (typeof record.text !== "string" || start === undefined || end === undefined) return; + const track = typeof record.track === "string" ? record.track : undefined; + const key = `${record.speaker}:${track}:${start}:${record.text}`; + if (this.#seen.has(key)) return; + this.#seen.add(key); + const word = { text: record.text, start, end, track }; if (record.speaker === "user") { - insert(this.userWords, word, (w) => w.start); - // The final word supersedes what the interims heard there. - this.interims = this.interims.filter((i) => i.end <= word.start || i.start >= word.end); - } else if (record.speaker === "agent") { - insert(this.agentWords, word, (w) => w.start); - } + this.userWords.push(word); + // A final result settles the interims it covers. + this.interims = this.interims.filter((w) => w.start >= end); + } else if (record.speaker === "agent") this.agentWords.push(word); return; } case "interim": { - const word = this.#word(record); - if (!word) return; - // A later interim revises the ones it overlaps. - this.interims = this.interims.filter((i) => i.end <= word.start || i.start >= word.end); - if (!this.userWords.some((w) => w.start < word.end && w.end > word.start)) { - insert(this.interims, word, (w) => w.start); - } - return; - } - case "vad": { - const at = agentMs(record, "at_ns"); - const media = ms(record, "media_ticks"); - const confirm = ms(record, "confirm_ticks") ?? 0; - if (at === undefined || media === undefined) return; - const speaking = record.speaking === true; - const change = media - confirm; - const open = this.speech.at(-1); - if (speaking && (!open || open.end !== undefined)) this.speech.push({ start: change }); - if (!speaking && open && open.end === undefined) open.end = change; - this.marks.push({ at, kind: "vad", label: speaking ? "speech" : "silence", media }); - return; - } - case "turn": { - const at = agentMs(record, "at_ns"); - if (at === undefined) return; - const media = ms(record, "media_ticks"); - this.marks.push({ at, kind: "turn", label: record.speaking === true ? "your turn" : "my turn", media }); - return; - } - case "llm": - case "tts": { + const start = ms(record, "start_ticks"); + const end = ms(record, "end_ticks"); const at = agentMs(record, "at_ns"); - if (at === undefined) return; - this.#span(record.type, record.event, at); + if (typeof record.text !== "string" || start === undefined || end === undefined || at === undefined) return; + // One result's words share its time, and each result restates the words since the last final. + if (at !== this.#interimAt) { + this.#interimAt = at; + this.interims = []; + } + this.interims.push({ text: record.text, start, end }); return; } case "publish": { @@ -228,45 +200,16 @@ export class CallLog { return; } case "retire": { - const at = agentMs(record, "at_ns"); - const unplayed = ms(record, "unplayed_ticks"); - if (at === undefined) return; - const label = unplayed ? `barge-in, dropped ${(unplayed / 1000).toFixed(1)}s` : "barge-in"; - this.marks.push({ at, kind: "retire", label }); - // What the retired track still held never plays; draw it where it would have. if (typeof record.track !== "string") return; - const now = this.local(at) ?? performance.now(); - this.#retired.set(record.track, now); + this.#retired.add(record.track); for (const frame of this.reply) { - if (frame.track === record.track) this.#discard(frame, now); + if (frame.track === record.track) this.#discard(frame); } return; } } } - #word(record: AgentRecord): Word | undefined { - const start = ms(record, "start_ticks"); - const end = ms(record, "end_ticks"); - if (typeof record.text !== "string" || start === undefined || end === undefined) return undefined; - const track = typeof record.track === "string" ? record.track : undefined; - return { text: record.text, start, end, at: agentMs(record, "at_ns"), track }; - } - - #span(kind: "llm" | "tts", event: unknown, at: number) { - const open = this.spans.findLast((s) => s.kind === kind && s.end === undefined); - if (event === "request") { - // TTS requests each sentence; one span covers the whole reply until it ends. Each LLM request - // is a new statement, and supersedes one an interruption left without an end. - if (open && kind === "llm") open.end = at; - if (!open || kind === "llm") this.spans.push({ kind, start: at }); - } else if (event === "first_text" || event === "first_audio") { - if (open && open.first === undefined) open.first = at; - } else if (event === "end") { - if (open) open.end = at; - } - } - /** Whether a publish at agent time `at` can have been sent before local time `arrive`. */ #sentBefore(at: number, arrive: number): boolean { // Trusting the offset's midpoint would turn away the faster frames that would refine it. @@ -277,10 +220,7 @@ export class CallLog { #published(frame: ReplyFrame, publish: Publish) { // A frame claimed from a retired track is not what that retirement discarded. - if (frame.track !== undefined && frame.track !== publish.track && frame.discarded) { - frame.discarded = false; - frame.play = undefined; - } + if (frame.track !== undefined && frame.track !== publish.track) frame.discarded = false; frame.track = publish.track; frame.published = publish.at; // Between overlapping publishes the pick is only as good as the clock, and the wrong one could @@ -288,23 +228,27 @@ export class CallLog { const down = frame.arrive - publish.at; const holders = this.publishes.filter((p) => frame.media >= p.start && frame.media < p.end).length; if (holders === 1 && (this.minDown === undefined || down < this.minDown)) this.minDown = down; - const retired = this.#retired.get(publish.track); - if (retired !== undefined) this.#discard(frame, retired); + if (this.#retired.has(publish.track)) this.#discard(frame); } - /** What a retired track still held never plays; draw it where it would have. */ - #discard(frame: ReplyFrame, retired: number) { - if (frame.play !== undefined) return; - // Before any playhead there is nowhere it would have played, so it ends where it was retired. - frame.play = this.playAt(frame, retired) ?? retired; - frame.discarded = true; + /** What a retired track still held never plays. */ + #discard(frame: ReplyFrame) { + if (!frame.played) frame.discarded = true; } /** A reply frame as `@moq/watch` received it. */ received(media: number, arrive: number, late: boolean) { const frame: ReplyFrame = { media, arrive, late }; - // Of the publishes holding this media time, the newest one sent before it arrived. Without a - // clock offset yet there has been no barge-in, so there is only one. + const publish = this.#holder(media, arrive); + if (publish) this.#published(frame, publish); + insert(this.reply, frame, (f) => f.media); + } + + /** + * Of the publishes holding this media time, the newest one sent before local time `arrive`. + * Without a clock offset yet there has been no barge-in, so there is only one. + */ + #holder(media: number, arrive: number): Publish | undefined { let best: Publish | undefined; for (let i = this.publishes.length - 1; i >= 0; i--) { const p = this.publishes[i]; @@ -314,84 +258,118 @@ export class CallLog { if (!this.#sentBefore(p.at, arrive)) continue; if (!best || p.at > best.at) best = p; } - if (best) this.#published(frame, best); - insert(this.reply, frame, (f) => f.media); + return best; } - /** Reply media that will never arrive. */ - gap(media: number, duration: number) { - this.gaps.push({ media, duration }); + /** Reply media that will never arrive, as found at local time `at`. */ + gap(media: number, duration: number, at: number) { + // A barge-in's track can reuse the media times of the one it replaced, which lost nothing. + this.gaps.push({ media, duration, track: this.#holder(media, at)?.track }); } - /** The reply's playhead: media `media` is playing at local time `now`, or nothing is. */ - playhead(now: number, media: number | undefined, playing: boolean) { - // The reply's media time only moves forward, across track changes too; an idle player - // reports zero. - if (media === undefined || (this.#anchor && media < this.#anchor.media)) { - this.#playing = false; - return; - } - this.#playing = playing; - this.#anchor = { local: now, media }; + /** The reply media the player holds, which runs ahead of the arrivals its decoder has reached. */ + buffer(ranges: readonly { start: number; end: number }[]) { + this.buffered = ranges.map(({ start, end }) => ({ start, end })); + } + + /** The reply's playhead is at `media`, or nothing is playing. */ + playhead(media: number | undefined) { + // An idle player reports zero. + if (media === undefined || (this.#head !== undefined && media < this.#head)) return; + this.#head = media; for (const frame of this.reply) { - if (frame.play === undefined && frame.media <= media) frame.play = now - (media - frame.media); + if (!frame.played && !frame.discarded && frame.media <= media) frame.played = true; } } - /** When a reply frame plays (or played) on the local clock, if it can be placed. */ - playAt(frame: ReplyFrame, now: number): number | undefined { - if (frame.play !== undefined) return frame.play; - const head = this.mediaNow(now); - return head === undefined ? undefined : now + Math.max(0, frame.media - head); + /** Your word's frames, one per FRAME_MS, as the agent received them. */ + sent(word: Word): Packet[] { + return slots(word).map((t) => { + // The frame whose 20 ms holds this slot's middle. + const i = above(this.mic, t - FRAME_MS, (f) => f.capture); + const frame = this.mic[i]; + if (!frame || frame.capture > t) return "flight"; + return frame.reordered ? "reordered" : "delivered"; + }); } - /** Media time of the reply's playhead now, extrapolated between updates. */ - mediaNow(now: number): number | undefined { - if (this.#head?.now === now) return this.#head.media; - const anchor = this.#anchor; - let media: number | undefined; - if (anchor && this.#playing) { - media = anchor.media + (now - anchor.local); - } else { - // A stalled player starts the next utterance from its own first frame, about now. - const next = this.reply.find((f) => f.play === undefined && !f.discarded); - media = next ? Math.max(next.media, anchor?.media ?? 0) : anchor?.media; - } - this.#head = { now, media }; - return media; + /** The reply started or stopped playing at local time `at`. */ + replying(playing: boolean, at: number) { + if (this.#replying && !playing) this.#replyStopped = at; + this.#replying = playing; } /** - * Where the latest exchange starts on the local clock: your latest statement or the agent's - * latest reply, whichever began first, as `voice.json` keeps the latest statement on each side. + * Your audio the STT has not made words of, up to `now`: its frames, as far as they have got. + * Nothing while the reply plays, and it starts over once it stops. */ - exchange(): number | undefined { - // The turn is decided a little into the statement, so it starts with the speech before it. - const heard = this.marks.findLast((m) => m.kind === "turn" && m.label === "your turn")?.media; - const said = heard === undefined ? undefined : (this.speech.findLast((s) => s.start <= heard)?.start ?? heard); - const thought = this.spans.findLast((s) => s.kind === "llm"); - const replied = thought && this.local(thought.start); - if (said === undefined) return replied; - return replied === undefined ? said : Math.min(said, replied); + pending(now: number): Packet[] | undefined { + const first = this.mic[0]; + if (this.#replying || !first) return undefined; + const none = Number.NEGATIVE_INFINITY; + let start = Math.max(first.capture, this.#replyStopped ?? none, this.userWords.at(-1)?.end ?? none); + for (const word of this.interims) start = Math.max(start, word.end); + if (now - start < FRAME_MS) return undefined; + return this.sent({ text: "", start, end: now }); } - /** Drop what has scrolled far out of view. */ - prune(now: number) { - const cut = now - KEEP_MS; - const offset = this.offset ?? 0; - this.mic = this.mic.filter((f) => f.capture >= cut); - this.userWords = this.userWords.filter((w) => w.end >= cut); - this.interims = this.interims.filter((w) => w.end >= cut); - this.speech = this.speech.filter((s) => s.end === undefined || s.end >= cut); - this.marks = this.marks.filter((m) => m.at - offset >= cut); - this.spans = this.spans.filter((s) => s.end === undefined || s.end - offset >= cut); - this.reply = this.reply.filter((f) => f.play === undefined || f.play >= cut); - const media = this.mediaNow(now); - if (media !== undefined) { - const mediaCut = media - KEEP_MS; - this.agentWords = this.agentWords.filter((w) => w.end >= mediaCut); - this.publishes = this.publishes.filter((p) => p.end >= mediaCut); - this.gaps = this.gaps.filter((g) => g.media + g.duration >= mediaCut); - } + /** The agent's word's frames, one per FRAME_MS, as they reached you. */ + heard(word: Word): Packet[] { + return slots(word).map((t) => { + for (let i = above(this.reply, t - FRAME_MS, (f) => f.media); i < this.reply.length; i++) { + const frame = this.reply[i]; + if (frame.media > t) break; + // A frame can beat the record of the track that carries it. + if (frame.track !== undefined && word.track !== undefined && frame.track !== word.track) continue; + if (frame.discarded) return "interrupted"; + if (frame.late) return "lost"; + return frame.played ? "delivered" : "buffered"; + } + if (word.track !== undefined && this.#retired.has(word.track)) return "interrupted"; + const lost = this.gaps.some( + (g) => + t >= g.media && + t < g.media + g.duration && + (g.track === undefined || word.track === undefined || g.track === word.track), + ); + if (lost) return "lost"; + // The decoder reports a frame only once playback nears it; the player holds it before that. + if (this.buffered.some((r) => t >= r.start && t < r.end)) return "buffered"; + return "flight"; + }); } } + +// Words as compared across the transcript and `voice.json`: case and punctuation aside. +function norm(text: string): string { + return text + .normalize("NFKD") + .toLowerCase() + .replace(/[^\p{L}\p{N}]/gu, ""); +} + +// How far either side may run ahead on one matching word: the transcript and `voice.json` each +// leave words out. +const SKIP = 4; + +/** + * Pair each transcript word with the timed word it is, in order. The transcript shows what was + * said; `voice.json` leaves out any word it could not time, so a word without a match is undefined. + */ +export function align(texts: string[], words: Word[]): (Word | undefined)[] { + const keys = texts.map(norm); + const timed = words.map((w) => norm(w.text)); + let next = 0; + return keys.map((key, k) => { + if (!key) return undefined; + for (let i = next; i < timed.length; i++) { + if (timed[i] !== key) continue; + // Further ahead, a common word alone could skip a stretch; the next word matching too means + // the two sides have met again. + if (i >= next + SKIP && (k + 1 >= keys.length || timed[i + 1] !== keys[k + 1])) continue; + next = i + 1; + return words[i]; + } + return undefined; + }); +} diff --git a/src/components/voice/timeline.tsx b/src/components/voice/timeline.tsx deleted file mode 100644 index 326a0da..0000000 --- a/src/components/voice/timeline.tsx +++ /dev/null @@ -1,383 +0,0 @@ -// The live call on one scrolling clock, drawn like the post's diagrams come alive: your mic frames -// crossing to the agent, what it heard and did, and its reply crossing back into your buffer. -// Canvas, not SVG: a call is thousands of 20 ms frames, redrawn every animation frame. -import { createSignal, onCleanup, onMount, Show } from "solid-js"; -import { type CallLog, FRAME_MS, WEBRTC_MS } from "./call"; -import { COLOR } from "./diagram"; - -// The axis spans the latest exchange, at least this many milliseconds, with room either side. -const MIN_SPAN = 8000; -const PAD = 500; -// How quickly the axis eases to a new exchange, in milliseconds. -const EASE = 250; - -const GUTTER = 64; -const LANE_H = 26; -const NET_H = 34; -const AGENT_H = 40; -const AXIS_H = 16; -const TOP = 4; - -// Lane tops, top to bottom. -const MIC_Y = TOP; -const UP_Y = MIC_Y + LANE_H; -const AGENT_Y = UP_Y + NET_H; -const DOWN_Y = AGENT_Y + AGENT_H; -const SPEAKER_Y = DOWN_Y + NET_H; -const AXIS_Y = SPEAKER_Y + LANE_H; -const H = AXIS_Y + AXIS_H; - -const PALETTE = { - ...COLOR, - flight: "rgba(245, 158, 11, 0.45)", - thinking: "rgba(245, 158, 11, 0.55)", - speech: "rgba(148, 163, 184, 0.18)", - discarded: "rgba(148, 163, 184, 0.35)", - wire: "rgba(34, 197, 94, 0.22)", - late: "#ef4444", -}; - -const FONT = "11px 'JetBrains Mono', ui-monospace, monospace"; - -export default function Timeline(props: { call: CallLog }) { - const [summary, setSummary] = createSignal(""); - const [stats, setStats] = createSignal<{ - lateUp: number; - up: number; - lateDown: number; - down: number; - ahead?: number; - error?: number; - }>(); - let canvas!: HTMLCanvasElement; - - onMount(() => { - const ctx = canvas.getContext("2d"); - if (!ctx) return; - - let width = 0; - let frame = 0; - let view: { left: number; right: number; at: number } | undefined; - const resize = () => { - const dpr = window.devicePixelRatio || 1; - width = canvas.clientWidth; - canvas.width = Math.round(width * dpr); - canvas.height = Math.round(H * dpr); - ctx.setTransform(dpr, 0, 0, dpr, 0, 0); - // Resizing clears the canvas, so a finished call whose loop has stopped draws again. - const call = props.call; - if (frame === 0 && view && call.ended !== undefined) draw(ctx, call, call.ended, width, view.left, view.right); - }; - const observer = new ResizeObserver(resize); - observer.observe(canvas); - resize(); - - let pruned = 0; - let summarized = 0; - const tick = () => { - const call = props.call; - const now = call.ended ?? performance.now(); - if (now - pruned > 1000) { - call.prune(now); - pruned = now; - } - const [left, right] = span(call, now); - const ease = view ? 1 - Math.exp(-(performance.now() - view.at) / EASE) : 1; - view = { - left: view ? view.left + (left - view.left) * ease : left, - right: view ? view.right + (right - view.right) * ease : right, - at: performance.now(), - }; - draw(ctx, call, now, width, view.left, view.right); - if (now - summarized > 500) { - summarized = now; - const s = measure(call, now); - setStats(s); - setSummary(describe(s)); - } - frame = call.ended === undefined || performance.now() - call.ended < 2000 ? requestAnimationFrame(tick) : 0; - }; - frame = requestAnimationFrame(tick); - onCleanup(() => { - cancelAnimationFrame(frame); - observer.disconnect(); - }); - }); - - return ( -
- -
- arrived - buffered - in flight - late for WebRTC (+{WEBRTC_MS} ms) - interrupted -
- - {(s) => ( -

- you → agent: {s().lateUp} of {s().up} frames late for WebRTC · agent → you: {s().lateDown} of {s().down} - · reply buffered {((s().ahead ?? 0) / 1000).toFixed(1)}s ahead - · clocks synced ±{Math.round(s().error ?? 0)} ms -

- )} - -
- ); -} - -function Swatch(props: { color: string; children: unknown }) { - return ( - - - {props.children as string} - - ); -} - -function measure(call: CallLog, now: number) { - let lateUp = 0; - for (const f of call.mic) if (call.upLateness(f) > WEBRTC_MS) lateUp++; - let lateDown = 0; - let down = 0; - for (const f of call.reply) { - const late = call.downLateness(f); - if (late === undefined) continue; - down++; - if (late > WEBRTC_MS) lateDown++; - } - const head = call.mediaNow(now); - // A retired track's frames can sit past the replacement in media time, but never play. - const last = call.reply.findLast((f) => f.play === undefined && !f.discarded); - // A frame the extrapolated head already passed is due, not owed. - const ahead = head !== undefined && last ? Math.max(0, last.media + FRAME_MS - head) : undefined; - return { lateUp, up: call.mic.length, lateDown, down, ahead, error: call.error }; -} - -function describe(s: ReturnType): string { - const parts = [ - `Live call timeline. Your audio: ${s.lateUp} of ${s.up} frames reached the agent too late for a WebRTC jitter buffer.`, - `The reply: ${s.lateDown} of ${s.down} frames late.`, - ]; - if (s.ahead !== undefined) parts.push(`${(s.ahead / 1000).toFixed(1)} seconds of reply buffered ahead.`); - return parts.join(" "); -} - -// The latest exchange, through the end of the reply buffered so far, with now always in view. -function span(call: CallLog, now: number): [number, number] { - let end = now; - const last = call.reply.findLast((f) => !f.discarded); - const played = last && call.playAt(last, now); - if (played !== undefined) end = Math.max(end, played + FRAME_MS); - const right = end + PAD; - const start = call.exchange(); - const left = start === undefined ? right - MIN_SPAN : Math.min(start - PAD, right - MIN_SPAN); - return [left, right]; -} - -function draw(ctx: CanvasRenderingContext2D, call: CallLog, now: number, width: number, left: number, right: number) { - ctx.clearRect(0, 0, width, H); - if (width <= GUTTER) return; - - const trackW = width - GUTTER; - const pxPerMs = trackW / (right - left); - const x = (t: number) => GUTTER + (t - left) * pxPerMs; - const visible = (start: number, end: number) => end >= left && start <= right; - - ctx.font = FONT; - ctx.textBaseline = "middle"; - - // Lane backgrounds and labels. - const lane = (y: number, h: number, label: string) => { - ctx.fillStyle = PALETTE.track; - ctx.fillRect(GUTTER, y, trackW, h); - ctx.fillStyle = PALETTE.neutral; - ctx.textAlign = "right"; - ctx.fillText(label, GUTTER - 6, y + h / 2); - }; - lane(MIC_Y, LANE_H, "you"); - lane(AGENT_Y, AGENT_H, "agent"); - lane(SPEAKER_Y, LANE_H, "you hear"); - - ctx.save(); - ctx.beginPath(); - ctx.rect(GUTTER, 0, trackW, H); - ctx.clip(); - - // Your speech as the agent's VAD heard it. - ctx.fillStyle = PALETTE.speech; - for (const s of call.speech) { - const end = s.end ?? now; - if (visible(s.start, end)) ctx.fillRect(x(s.start), MIC_Y, (end - s.start) * pxPerMs, LANE_H); - } - - // Mic frames: green once the agent has them, red if too late for WebRTC. - const frameW = Math.max(1, FRAME_MS * pxPerMs); - let lastCapture = left; - for (const f of call.mic) { - if (f.capture + FRAME_MS < left || f.capture > right) continue; - const late = call.upLateness(f) > WEBRTC_MS; - ctx.fillStyle = late ? PALETTE.late : PALETTE.done; - ctx.fillRect(x(f.capture), MIC_Y + (late ? 2 : 6), frameW, LANE_H - (late ? 4 : 12)); - lastCapture = Math.max(lastCapture, f.capture + FRAME_MS); - } - // Captured but not yet at the agent: the agent's arrivals stop short of now. - if (call.ended === undefined && call.mic.length > 0 && lastCapture < now) { - ctx.fillStyle = PALETTE.flight; - ctx.fillRect(x(lastCapture), MIC_Y + 6, (now - lastCapture) * pxPerMs, LANE_H - 12); - } - - // Your words over their audio; interim guesses fainter. - words(ctx, call.interims, x, MIC_Y + LANE_H / 2, PALETTE.faint, left, right, (w) => w.start); - words(ctx, call.userWords, x, MIC_Y + LANE_H / 2, PALETTE.text, left, right, (w) => w.start); - - // Request frames crossing to the agent, once its clock is placed on ours. - const offset = call.offset; - if (offset !== undefined) { - ctx.lineWidth = 1; - for (const f of call.mic) { - const arrive = f.arrive - offset; - if (!visible(f.capture, arrive)) continue; - const late = call.upLateness(f) > WEBRTC_MS; - // Every fifth frame, or the stream is a solid wall. - if (!late && Math.round(f.capture / FRAME_MS) % 5 !== 0) continue; - ctx.strokeStyle = late ? PALETTE.late : PALETTE.wire; - ctx.beginPath(); - ctx.moveTo(x(f.capture + FRAME_MS / 2), UP_Y); - ctx.lineTo(x(arrive), AGENT_Y); - ctx.stroke(); - } - } - - // The agent: hearing on top, thinking and speaking below. - if (offset !== undefined) { - const hearY = AGENT_Y + AGENT_H / 4; - const doY = AGENT_Y + (AGENT_H * 3) / 4; - // Where each row's last label ends, so a quick retry's label does not overprint the last one. - const labeled = { llm: Number.NEGATIVE_INFINITY, tts: Number.NEGATIVE_INFINITY }; - for (const s of call.spans) { - const start = s.start - offset; - const end = (s.end ?? now + offset) - offset; - if (!visible(start, end)) continue; - const first = s.first !== undefined ? s.first - offset : end; - const y = s.kind === "llm" ? doY - 8 : doY + 1; - ctx.fillStyle = PALETTE.thinking; - ctx.fillRect(x(start), y, (first - start) * pxPerMs, 7); - ctx.fillStyle = PALETTE.done; - ctx.fillRect(x(first), y, Math.max(0, (end - first) * pxPerMs), 7); - const label = Math.max(GUTTER + 2, x(start)) + 2; - if (label < labeled[s.kind]) continue; - ctx.fillStyle = PALETTE.text; - ctx.textAlign = "left"; - ctx.font = "9px 'JetBrains Mono', ui-monospace, monospace"; - ctx.fillText(s.kind.toUpperCase(), label, y + 4); - labeled[s.kind] = label + ctx.measureText(s.kind.toUpperCase()).width + 4; - ctx.font = FONT; - } - // A barge-in lands right on the turn that caused it, so its label goes first and a turn label - // that would overprint a placed one is left off. - const placed: [number, number][] = []; - ctx.textAlign = "left"; - for (const kind of ["retire", "turn", "vad"] as const) { - for (const m of call.marks) { - const at = m.at - offset; - if (m.kind !== kind || at < left || at > right) continue; - const px = x(at); - if (kind === "vad") { - ctx.fillStyle = PALETTE.neutral; - ctx.fillRect(px - 0.5, AGENT_Y + 2, 1, AGENT_H / 2 - 4); - continue; - } - ctx.fillStyle = kind === "retire" ? PALETTE.late : PALETTE.text; - if (kind === "retire") ctx.fillRect(px - 1, AGENT_Y, 2, AGENT_H); - else ctx.fillRect(px - 1, AGENT_Y + 2, 2, AGENT_H / 2 - 4); - const end = px + 4 + ctx.measureText(m.label).width; - if (placed.some(([a, b]) => px < b + 4 && end > a)) continue; - placed.push([px, end]); - ctx.fillText(m.label, px + 4, hearY); - } - } - // When STT settled on each of your words. - ctx.fillStyle = PALETTE.text; - for (const w of call.userWords) { - if (w.at === undefined) continue; - const at = w.at - offset; - if (at < left || at > right) continue; - ctx.beginPath(); - ctx.arc(x(at), hearY, 2, 0, Math.PI * 2); - ctx.fill(); - } - } - - // Reply frames: crossing from the agent, then waiting in your buffer until they play. - const head = call.mediaNow(now); - for (const f of call.reply) { - const play = call.playAt(f, now); - const downLate = call.downLateness(f); - const late = f.late || (downLate !== undefined && downLate > WEBRTC_MS); - if (offset !== undefined && f.published !== undefined) { - const sent = f.published - offset; - if (visible(sent, f.arrive) && (late || Math.round(f.media / FRAME_MS) % 5 === 0)) { - ctx.strokeStyle = late ? PALETTE.late : PALETTE.wire; - ctx.beginPath(); - ctx.moveTo(x(sent), AGENT_Y + AGENT_H); - ctx.lineTo(x(f.arrive), SPEAKER_Y); - ctx.stroke(); - } - } - if (play === undefined || play + FRAME_MS < left || play > right) continue; - const played = f.play !== undefined || (head !== undefined && f.media <= head); - ctx.fillStyle = f.discarded ? PALETTE.discarded : late ? PALETTE.late : played ? PALETTE.done : PALETTE.have; - ctx.fillRect(x(play), SPEAKER_Y + (late ? 2 : 6), frameW, LANE_H - (late ? 4 : 12)); - } - - // The agent's words over the audio that says them. - if (head !== undefined) { - words(ctx, call.agentWords, x, SPEAKER_Y + LANE_H / 2, PALETTE.ink, left, right, (w) => { - const frame = call.reply.find((f) => f.media >= w.start && (w.track === undefined || f.track === w.track)); - return frame ? call.playAt(frame, now) : undefined; - }); - } - - ctx.restore(); - - // The playhead: everything left of it has happened, the buffer sits right of it. - ctx.fillStyle = PALETTE.text; - ctx.fillRect(x(now) - 1, MIC_Y - 2, 2, AXIS_Y - MIC_Y + 2); - - // Seconds from now, as many as fit. - ctx.fillStyle = PALETTE.dim; - ctx.textAlign = "center"; - const step = [1, 2, 5, 10, 30].find((s) => s * 1000 * pxPerMs >= 40) ?? 60; - for (let s = Math.ceil((left - now) / 1000 / step) * step; now + s * 1000 <= right; s += step) { - const tx = x(now + s * 1000); - if (tx < GUTTER + 8 || tx > width - 8) continue; - ctx.fillRect(tx, AXIS_Y, 1, 3); - ctx.fillText(s === 0 ? "now" : `${s > 0 ? "+" : ""}${s}s`, tx, AXIS_Y + 10); - } -} - -function words( - ctx: CanvasRenderingContext2D, - list: T[], - x: (t: number) => number, - y: number, - color: string, - left: number, - right: number, - at: (w: T) => number | undefined, -) { - ctx.fillStyle = color; - ctx.textAlign = "left"; - let edge = Number.NEGATIVE_INFINITY; - for (const w of list) { - const t = at(w); - if (t === undefined || t > right || t + (w.end - w.start) < left) continue; - const px = x(t); - // Skip a word that would overprint the previous one. - if (px < edge) continue; - ctx.fillText(w.text, px + 1, y); - edge = px + ctx.measureText(w.text).width + 4; - } -} diff --git a/src/components/voice/transcript.tsx b/src/components/voice/transcript.tsx new file mode 100644 index 0000000..555c041 --- /dev/null +++ b/src/components/voice/transcript.tsx @@ -0,0 +1,186 @@ +// Both sides' words, each underlined by its own audio: one segment per 20 ms packet, colored by how +// far it got. Your words fill in as the agent receives them, behind a bar of the speech it has not +// recognized yet; its words land in your buffer well ahead of the playhead and turn solid as you +// hear them. +import { createEffect, createMemo, createSignal, For, Index, type JSX, onCleanup, Show } from "solid-js"; +import { align, type CallLog, FRAME_MS, type Packet } from "./call"; + +// A run of speech by one side. `interim` is your words the recognizer may still revise. +export type Turn = { who: "you" | "ai"; text: string; interim: string }; + +const PACKET: Record = { + delivered: "#22c55e", + buffered: "rgba(34, 197, 94, 0.35)", + reordered: "#f59e0b", + flight: "#334155", + interrupted: "#94a3b8", + lost: "#ef4444", +}; + +const LEGEND: [Packet, string][] = [ + ["delivered", "delivered"], + ["buffered", "buffered ahead"], + ["reordered", "arrived out of order"], + ["flight", "in flight"], + ["interrupted", "interrupted"], + ["lost", "lost"], +]; + +const REDRAW_MS = 100; + +// About the width the words take to say, so the bar gives way to them without a jump. +const PX_PER_MS = 0.12; + +function words(text: string): string[] { + return text.split(/\s+/).filter(Boolean); +} + +function underline(packets: Packet[]): JSX.CSSProperties { + // A run of one state is one gradient stop pair. + const stops: string[] = []; + let from = 0; + packets.forEach((packet, i) => { + if (packets[i + 1] === packet) return; + const to = ((i + 1) / packets.length) * 100; + stops.push(`${PACKET[packet]} ${from}% ${to}%`); + from = to; + }); + return { + "background-image": `linear-gradient(to right, ${stops.join(", ")})`, + "background-size": "100% 3px", + "background-position": "0 100%", + "background-repeat": "no-repeat", + "padding-bottom": "3px", + // A word a barge-in cut off entirely was never said. Only the agent's words are cut off. + color: packets.every((p) => p === "interrupted") ? "rgba(203, 213, 225, 0.4)" : undefined, + }; +} + +/** `live` while the call's audio still moves, so the underlines follow its packets. */ +export default function Transcript(props: { turns: Turn[]; log?: CallLog; live: boolean }) { + const [tick, setTick] = createSignal(0); + createEffect(() => { + if (!props.live) return; + const timer = setInterval(() => setTick((t) => t + 1), REDRAW_MS); + onCleanup(() => { + clearInterval(timer); + // One last look at where every packet ended up. + setTick((t) => t + 1); + }); + }); + + // Each word's underline by turn, from the `voice.json` word it matches. + const underlines = createMemo(() => { + tick(); + const log = props.log; + const turns = props.turns; + if (!log) return []; + const spoken = (who: Turn["who"]) => turns.filter((t) => t.who === who).flatMap((t) => words(t.text)); + const matched = { you: align(spoken("you"), log.userWords), ai: align(spoken("ai"), log.agentWords) }; + const next = { you: 0, ai: 0 }; + return turns.map((turn) => ({ + text: words(turn.text).map(() => { + const word = matched[turn.who][next[turn.who]++]; + if (!word) return undefined; + return underline(turn.who === "you" ? log.sent(word) : log.heard(word)); + }), + interim: align(words(turn.interim), log.interims).map((word) => word && underline(log.sent(word))), + })); + }); + + // Your speech the agent has heard but not made words of, while the call is up. + const pending = createMemo(() => { + tick(); + return props.live ? props.log?.pending(performance.now()) : undefined; + }); + const bar = (packets: () => Packet[]) => ( + + ); + // The bar trails your current turn, or starts one before its first word. + const opening = () => props.turns.at(-1)?.who !== "you"; + + // The transcript follows new words unless the reader has scrolled up to reread. + let list: HTMLOListElement | undefined; + let following = true; + createEffect(() => { + void props.turns; + void pending(); + if (list && following) list.scrollTop = list.scrollHeight; + }); + + return ( + 0 || pending()}> +
    { + list = el; + following = true; + }} + onScroll={(e) => { + const el = e.currentTarget; + following = el.scrollHeight - el.scrollTop - el.clientHeight < 24; + }} + class="mt-4 mb-0 max-h-80 list-none space-y-3 overflow-y-auto p-0" + aria-label="Transcript" + > + + {(turn, t) => ( +
  1. + + {turn.who === "ai" ? "AI" : "you"} + +

    + + {(word, i) => ( + + {word()} + + )} + + + {(word, i) => ( + + {word()} + + )} + + {(p) => bar(p)} +

    +
    2. + )} +     + + {(p) => ( +
  2. + you +

    {bar(p)}

    +
    3. + )} +     +
+ +
    + + {([packet, label]) => ( +
  • + + {label} +
    • + )} +     +
+ + + ); +}