@devries-cameron20 / t3code Mirror · read-only / commits / 40c3682543
Fork 0

The best way to control Claude Code, Codex and more from any computer or phone

Mirrored from https://github.com/Camo67/t3code.git

Synced Nov 21, 23:30 UTC

feat(web): screen readers can hear the agent's messages and steps during a turn (#33900)

Co-authored-by: Claude Fable 5.2 <noreply@anthropic.com>

shivam committed Nov 11, 2026 at 13:16 UTC 40c3682543fdcb9f8b7d4106ed2ab970c25b7dc5
11 files changed +935 -13
apps/web/src/components/chat/MessagesTimeline.logic.ts
+2 -2
@@ -79,7 +79,7 @@ import {
79 } from "@t3tools/shared/toolActivity";
80 import { isWindowsAbsolutePath } from "@t3tools/shared/path";
81
82 -function timelineEntryRunId(entry: TimelineEntry): RunId | null {
82 +export function timelineEntryRunId(entry: TimelineEntry): RunId | null {
83 if (entry.kind === "message") {
84 return entry.message.role === "assistant" ? (entry.message.runId ?? null) : null;
85 }
@@ -917,7 +917,7 @@ function timelineEntryIsSteer(entry: TimelineEntry): boolean {
917 );
918 }
919
920 -function timelineEntryStartsResponse(entry: TimelineEntry): boolean {
920 +export function timelineEntryStartsResponse(entry: TimelineEntry): boolean {
921 return (
922 (entry.kind === "message" && entry.message.role === "user" && !timelineEntryIsSteer(entry)) ||
923 (entry.kind === "work" && entry.entry.itemType === "notification")
apps/web/src/components/chat/MessagesTimeline.tsx
+7
@@ -200,6 +200,7 @@ const TIMELINE_MAINTAIN_SCROLL_AT_END = {
200 },
201 } as const satisfies MaintainScrollAtEndOptions;
202 const EMPTY_RUN_IDS: ReadonlySet<RunId> = new Set();
203 +const EMPTY_TIMELINE_ENTRIES: ReadonlyArray<TimelineEntry> = [];
204 const TIMELINE_MAINTAIN_SCROLL_AT_END_SMOOTH = {
205 ...TIMELINE_MAINTAIN_SCROLL_AT_END,
206 animated: true,
@@ -552,6 +553,9 @@ const ConversationTimeline = memo(function ConversationTimeline({
553 }
554
555 const preferredToolCallVisibility = useClientSettings((settings) => settings.toolCallVisibility);
556 + const screenReaderAnnouncements = useClientSettings(
557 + (settings) => settings.screenReaderAnnouncements,
558 + );
559 const toolCallVisibility =
560 findActive && preferredToolCallVisibility === "zen" ? "hideAll" : preferredToolCallVisibility;
561 const revealedMessageId = citationRequest?.citation.messageId ?? null;
@@ -1078,6 +1082,9 @@ const ConversationTimeline = memo(function ConversationTimeline({
1082 runId={announcedRun?.runId ?? null}
1083 runStatus={announcedRun?.status ?? null}
1084 detailBehind={detailBehindShell}
1085 + level={screenReaderAnnouncements}
1086 + entries={screenReaderAnnouncements === "status" ? EMPTY_TIMELINE_ENTRIES : timelineEntries}
1087 + workspaceRoot={workspaceRoot}
1088 />
1089 );
1090 const listHeaderLeadingContent =
apps/web/src/components/chat/ThreadActivityAnnouncer.browser.test.tsx new
+157
@@ -0,0 +1,157 @@
1 +import { MessageId, RunId } from "@t3tools/contracts";
2 +import { type ComponentProps } from "react";
3 +import { describe, expect, it, vi } from "vite-plus/test";
4 +
5 +import { act, fakeTimers, render } from "../../../test/render";
6 +import type { TimelineEntry } from "../../session-logic";
7 +import { ThreadActivityAnnouncer } from "./ThreadActivityAnnouncer";
8 +
9 +const run = RunId.make("run-1");
10 +const at = "2026-01-01T00:00:00.000Z";
11 +
12 +const message = (id: string, role: "user" | "assistant", streaming = false): TimelineEntry => ({
13 + kind: "message",
14 + id,
15 + createdAt: at,
16 + message: {
17 + id: MessageId.make(id),
18 + role,
19 + text: id,
20 + runId: role === "assistant" ? run : null,
21 + streaming,
22 + createdAt: at,
23 + updatedAt: at,
24 + },
25 +});
26 +const command = (program: string): TimelineEntry => ({
27 + kind: "work",
28 + id: program,
29 + createdAt: at,
30 + entry: {
31 + id: program,
32 + createdAt: at,
33 + runId: run,
34 + label: program,
35 + tone: "tool",
36 + itemType: "command_execution",
37 + command: program,
38 + toolLifecycleStatus: "inProgress",
39 + },
40 +});
41 +const prompt = message("prompt", "user");
42 +
43 +const announcer = (props: Partial<ComponentProps<typeof ThreadActivityAnnouncer>>) => (
44 + <ThreadActivityAnnouncer
45 + threadKey="thread-1"
46 + status="working"
47 + requestId={null}
48 + runId={run}
49 + runStatus="running"
50 + detailBehind={false}
51 + level="everything"
52 + entries={[prompt]}
53 + workspaceRoot={undefined}
54 + {...props}
55 + />
56 +);
57 +
58 +const regions = (container: HTMLElement) => {
59 + const [detail, status] = container.querySelectorAll<HTMLElement>('[aria-live="polite"]');
60 + return { detail: detail!, status: status! };
61 +};
62 +
63 +describe("ThreadActivityAnnouncer", () => {
64 + it("drops a waiting step once the agent has moved on", async () => {
65 + // The announcer reads no route.
66 + await using screen = await render(announcer({ status: "idle", runStatus: "completed" }), {
67 + router: false,
68 + });
69 + const { detail, status } = regions(screen.container);
70 + {
71 + // Restored before the assertions, which poll on real timers.
72 + using _timers = fakeTimers();
73 + await screen.rerender(announcer({}));
74 + await screen.rerender(announcer({ entries: [prompt, command("git")] }));
75 + await screen.rerender(
76 + announcer({ entries: [prompt, command("git"), message("reply", "assistant", true)] }),
77 + );
78 + await act(() => vi.advanceTimersByTime(5_000));
79 + }
80 +
81 + await expect.element(status).toHaveTextContent("Working");
82 + await expect.element(detail).toBeEmptyDOMElement();
83 + });
84 +
85 + it("holds a waiting step while the thread's detail is behind, then reads it", async () => {
86 + // The announcer reads no route.
87 + await using screen = await render(announcer({ status: "idle", runStatus: "completed" }), {
88 + router: false,
89 + });
90 + const { detail } = regions(screen.container);
91 + const entries = [prompt, command("git")];
92 + const heard: string[] = [];
93 + {
94 + // Restored before the assertion, which polls on real timers.
95 + using _timers = fakeTimers();
96 + await screen.rerender(announcer({}));
97 + await screen.rerender(announcer({ entries }));
98 + await screen.rerender(announcer({ entries, detailBehind: true }));
99 + await act(() => vi.advanceTimersByTime(5_000));
100 + heard.push(detail.textContent);
101 + await screen.rerender(announcer({ entries }));
102 + await act(() => vi.advanceTimersByTime(2_000));
103 + }
104 +
105 + expect(heard).toEqual([""]);
106 + await expect.element(detail).toHaveTextContent("Running git");
107 + });
108 +
109 + it("keeps a message beside the status that follows it, then reads the newest step", async () => {
110 + // The announcer reads no route.
111 + await using screen = await render(announcer({ status: "idle", runStatus: "completed" }), {
112 + router: false,
113 + });
114 + const { detail, status } = regions(screen.container);
115 + const entries = [prompt, message("Checking the repo", "assistant"), command("git")];
116 + const heard: Array<[status: string, detail: string]> = [];
117 + using _timers = fakeTimers();
118 + for (const props of [
119 + {},
120 + { entries },
121 + { entries, status: "approval", requestId: "request-1" },
122 + { entries: [...entries, command("rg")] },
123 + ] satisfies ReadonlyArray<Partial<ComponentProps<typeof ThreadActivityAnnouncer>>>) {
124 + await screen.rerender(announcer(props));
125 + heard.push([status.textContent, detail.textContent]);
126 + }
127 + await act(() => vi.advanceTimersByTime(2_000));
128 + heard.push([status.textContent, detail.textContent]);
129 +
130 + expect(heard).toEqual([
131 + ["Working", ""],
132 + ["Working", "T3 Code: Checking the repo"],
133 + ["Needs approval", "T3 Code: Checking the repo"],
134 + ["Needs approval", "T3 Code: Checking the repo"],
135 + ["Needs approval", "Running rg"],
136 + ]);
137 + });
138 +
139 + it("reads a status that lands with a message as the message's last sentence", async () => {
140 + // The announcer reads no route.
141 + await using screen = await render(announcer({ status: "idle", runStatus: "completed" }), {
142 + router: false,
143 + });
144 + const { detail, status } = regions(screen.container);
145 + await screen.rerender(announcer({}));
146 + await screen.rerender(
147 + announcer({
148 + entries: [prompt, message("Checking the repo", "assistant"), command("git")],
149 + status: "approval",
150 + requestId: "request-1",
151 + }),
152 + );
153 +
154 + await expect.element(detail).toHaveTextContent("T3 Code: Checking the repo. Needs approval");
155 + await expect.element(status).toHaveTextContent("Working");
156 + });
157 +});
apps/web/src/components/chat/ThreadActivityAnnouncer.tsx
+149 -11
@@ -1,14 +1,46 @@
1 +import type { ScreenReaderAnnouncements } from "@t3tools/contracts";
2 import { memo, useEffect, useRef } from "react";
3
4 import { useLiveRegion } from "../../hooks/useLiveRegion";
5 +import type { TimelineEntry } from "../../session-logic";
6
7 import {
8 nextThreadAnnouncement,
9 type ThreadActivitySnapshot,
10 type ThreadAnnouncementMemory,
11 } from "./threadActivityAnnouncement";
12 +import {
13 + nextTurnDetailAnnouncement,
14 + NO_TURN_DETAIL,
15 + turnDetail,
16 + type TurnDetailMemory,
17 +} from "./turnDetailAnnouncement";
18 +
19 +// Long enough for a screen reader to finish a short label before the next one.
20 +const ACTIVITY_GAP_MS = 2_000;
21 +// A polite region queues what it can't say yet, so steps written while a
22 +// message is still being read would all be heard late. They wait this long.
23 +const MESSAGE_HOLD_MS_PER_CHARACTER = 50;
24 +const MESSAGE_HOLD_MAX_MS = 15_000;
25 +
26 +function holdAfter(text: string) {
27 + return Math.min(
28 + MESSAGE_HOLD_MAX_MS,
29 + Math.max(ACTIVITY_GAP_MS, text.length * MESSAGE_HOLD_MS_PER_CHARACTER),
30 + );
31 +}
32 +
33 +/** One write, since a second write to a region replaces the first before it is heard. */
34 +function asOneAnnouncement(texts: ReadonlyArray<string>) {
35 + return texts
36 + .map((text, index) => (index === texts.length - 1 || /[.!?:…]$/.test(text) ? text : `${text}.`))
37 + .join(" ");
38 +}
39
11 -/** One polite live region for the open thread. */
40 +/**
41 + * The open thread's polite live regions. Status has its own, so "Needs
42 + * approval" landing a moment after a message doesn't replace it unheard.
43 + */
44 export const ThreadActivityAnnouncer = memo(function ThreadActivityAnnouncer({
45 threadKey,
46 status,
@@ -16,35 +48,141 @@ export const ThreadActivityAnnouncer = memo(function ThreadActivityAnnouncer({
48 runId,
49 runStatus,
50 detailBehind,
51 + level,
52 + entries,
53 + workspaceRoot,
54 }: ThreadActivitySnapshot & {
55 /** Null while the timeline shows another thread during a switch. */
56 readonly threadKey: string | null;
57 /** See isDetailBehindShell. */
58 readonly detailBehind: boolean;
59 + readonly level: ScreenReaderAnnouncements;
60 + /** The thread's entries before any are hidden; unread at the "status" level. */
61 + readonly entries: ReadonlyArray<TimelineEntry>;
62 + readonly workspaceRoot: string | undefined;
63 }) {
64 const { regionRef, announce, clear } = useLiveRegion();
26 - const memoryRef = useRef<{ threadKey: string; memory: ThreadAnnouncementMemory } | null>(null);
65 + const {
66 + regionRef: detailRegionRef,
67 + announce: announceDetail,
68 + clear: clearDetail,
69 + } = useLiveRegion();
70 + const memoryRef = useRef<{
71 + threadKey: string;
72 + level: ScreenReaderAnnouncements;
73 + memory: ThreadAnnouncementMemory;
74 + detail: TurnDetailMemory;
75 + } | null>(null);
76 + // Messages are written at once; a step waits its turn, and a newer one replaces it.
77 + const stepRef = useRef<{
78 + quietUntil: number;
79 + waiting: { label: string; timer: ReturnType<typeof setTimeout> } | null;
80 + }>({ quietUntil: 0, waiting: null });
81
28 - // oxlint-disable-next-line t3code/no-unreasoned-state-sync -- writes thread status changes to the aria-live region after commit
82 + // oxlint-disable-next-line t3code/no-unreasoned-state-sync -- writes the turn's status, messages and steps to the aria-live region after commit
83 useEffect(() => {
84 + const step = stepRef.current;
85 + const write = (text: string, quietMs: number) => {
86 + announceDetail(text);
87 + step.quietUntil = Date.now() + quietMs;
88 + };
89 + const takeWaiting = () => {
90 + const label = step.waiting?.label ?? null;
91 + if (step.waiting !== null) clearTimeout(step.waiting.timer);
92 + step.waiting = null;
93 + return label;
94 + };
95 +
96 if (threadKey === null) {
97 + takeWaiting();
98 memoryRef.current = null;
99 clear();
100 + clearDetail();
101 + return;
102 + }
103 + const waitingLabel = takeWaiting();
104 + if (detailBehind) {
105 + // The dropped step counts as unsaid, so it is read once the detail catches up.
106 + if (waitingLabel !== null && memoryRef.current !== null) {
107 + const { detail } = memoryRef.current;
108 + memoryRef.current = { ...memoryRef.current, detail: { ...detail, activity: null } };
109 + }
110 return;
111 }
35 - if (detailBehind) return;
36 - const previous = memoryRef.current?.threadKey === threadKey ? memoryRef.current.memory : null;
37 - const { memory, announcement } = nextThreadAnnouncement(previous, {
112 + const previous = memoryRef.current?.threadKey === threadKey ? memoryRef.current : null;
113 + const turn = nextThreadAnnouncement(previous?.memory ?? null, {
114 status,
115 requestId,
116 runId,
117 runStatus,
118 });
43 - memoryRef.current = { threadKey, memory };
44 - if (previous === null) clear();
45 - else if (announcement !== null) announce(announcement);
46 - }, [threadKey, status, requestId, runId, runStatus, detailBehind, announce, clear]);
119 + // A new level starts from what the turn already holds, so raising it mid-turn replays nothing.
120 + const previousDetail = previous?.level === level ? previous.detail : null;
121 + const detail = nextTurnDetailAnnouncement(previousDetail, {
122 + ...(level === "status"
123 + ? NO_TURN_DETAIL
124 + : turnDetail(entries, workspaceRoot, previousDetail?.tailId ?? null)),
125 + level,
126 + status,
127 + });
128 + memoryRef.current = { threadKey, level, memory: turn.memory, detail: detail.memory };
129 + if (previous === null) {
130 + step.quietUntil = 0;
131 + clear();
132 + clearDetail();
133 + return;
134 + }
135 + if (detail.messages.length > 0) {
136 + // NVDA reads two regions written in the same frame in either order, so a
137 + // status that lands with a message is read as its last sentence.
138 + const text = asOneAnnouncement(
139 + turn.announcement === null ? detail.messages : [...detail.messages, turn.announcement],
140 + );
141 + write(text, holdAfter(text));
142 + } else if (turn.announcement !== null) {
143 + announce(turn.announcement);
144 + step.quietUntil = Math.max(step.quietUntil, Date.now() + ACTIVITY_GAP_MS);
145 + }
146 + // A waiting label is stale once its step is over or the level no longer reads steps.
147 + const label =
148 + level !== "everything" || detail.memory.activity === null
149 + ? null
150 + : (detail.activity ?? waitingLabel);
151 + if (label === null) return;
152 + const speakStep = () => {
153 + step.waiting = null;
154 + write(label, ACTIVITY_GAP_MS);
155 + };
156 + const wait = step.quietUntil - Date.now();
157 + if (wait <= 0) {
158 + speakStep();
159 + return;
160 + }
161 + const timer = setTimeout(speakStep, wait);
162 + step.waiting = { label, timer };
163 + // The next run takes the label over; after an unmount nothing is left to speak it.
164 + return () => clearTimeout(timer);
165 + }, [
166 + threadKey,
167 + status,
168 + requestId,
169 + runId,
170 + runStatus,
171 + detailBehind,
172 + level,
173 + entries,
174 + workspaceRoot,
175 + announce,
176 + clear,
177 + announceDetail,
178 + clearDetail,
179 + ]);
180
181 // No role="status": Orca 50 maps it to a status bar in Chromium and drops its updates.
49 - return <div ref={regionRef} aria-live="polite" className="sr-only" />;
182 + return (
183 + <>
184 + <div ref={detailRegionRef} aria-live="polite" className="sr-only" />
185 + <div ref={regionRef} aria-live="polite" className="sr-only" />
186 + </>
187 + );
188 });
apps/web/src/components/chat/timelineMinimapItems.ts
+5
@@ -89,6 +89,11 @@ function markdownNodeText(node: Nodes): string {
89 return node.type === "link" && parseThreadLinkHref(text) !== null ? "a thread" : text;
90 }
91
92 +/** Markdown as one line of the text it renders, without markup or link targets. */
93 +export function markdownPlainText(markdown: string): string {
94 + return markdownNodeText(markdownPreviewParser.parse(markdown)).replace(/\s+/g, " ").trim();
95 +}
96 +
97 function compactMinimapPreview(text: string | null | undefined) {
98 const compact = text?.replace(/\s+/g, " ").trim() ?? "";
99 return compact.length > 0 ? compact : null;
apps/web/src/components/chat/turnDetailAnnouncement.test.ts new
+320
@@ -0,0 +1,320 @@
1 +import { MessageId, RunAttemptId, RunId, type ScreenReaderAnnouncements } from "@t3tools/contracts";
2 +import * as DateTime from "effect/DateTime";
3 +import { describe, expect, it } from "vite-plus/test";
4 +
5 +import type { TimelineAttempt, TimelineEntry, WorkLogEntry } from "../../session-logic";
6 +import {
7 + nextTurnDetailAnnouncement,
8 + turnDetail,
9 + type TurnDetailMemory,
10 + type TurnDetailSnapshot,
11 + type TurnMessage,
12 +} from "./turnDetailAnnouncement";
13 +
14 +const run = RunId.make("run-1");
15 +const at = "2026-01-01T00:00:00.000Z";
16 +
17 +function message(
18 + id: string,
19 + role: "user" | "assistant",
20 + fields: Partial<Extract<TimelineEntry, { kind: "message" }>["message"]> = {},
21 +): TimelineEntry {
22 + return {
23 + kind: "message",
24 + id,
25 + createdAt: at,
26 + message: {
27 + id: MessageId.make(id),
28 + role,
29 + text: id,
30 + runId: role === "assistant" ? run : null,
31 + streaming: false,
32 + createdAt: at,
33 + updatedAt: at,
34 + ...fields,
35 + },
36 + };
37 +}
38 +
39 +function work(id: string, fields: Partial<WorkLogEntry> = {}): TimelineEntry {
40 + return {
41 + kind: "work",
42 + id,
43 + createdAt: at,
44 + entry: { id, createdAt: at, runId: run, label: id, tone: "tool", ...fields },
45 + };
46 +}
47 +
48 +const prompt = message("prompt", "user");
49 +const command = (id: string, fields: Partial<WorkLogEntry> = {}) =>
50 + work(id, {
51 + itemType: "command_execution",
52 + command: "git status",
53 + toolLifecycleStatus: "inProgress",
54 + ...fields,
55 + });
56 +const notice = work("notice", { itemType: "system_notice", tone: "info", label: "Heads up" });
57 +const steer = message("steer", "user", { inputIntent: "steer" });
58 +const time = (clock: string) => DateTime.makeUnsafe(`2026-01-01T${clock}.000Z`);
59 +const inAttempt = (
60 + entry: TimelineEntry,
61 + id: string,
62 + status: TimelineAttempt["status"],
63 + completed?: string,
64 +): TimelineEntry => ({
65 + ...entry,
66 + attempt: {
67 + id: RunAttemptId.make(id),
68 + runId: run,
69 + attemptOrdinal: 1,
70 + rootNodeId: `${id}-root` as TimelineAttempt["rootNodeId"],
71 + status,
72 + completedAt: completed === undefined ? null : time(completed),
73 + },
74 +});
75 +const completedAt = (entry: TimelineEntry, clock: string) =>
76 + ({
77 + ...entry,
78 + projectedItem: {
79 + item: { type: "assistant_message", status: "completed", completedAt: time(clock) },
80 + },
81 + }) as TimelineEntry;
82 +
83 +describe("turnDetail", () => {
84 + it.each<{
85 + name: string;
86 + entries: ReadonlyArray<TimelineEntry>;
87 + activity: string | null;
88 + }>([
89 + { name: "a running command", entries: [prompt, command("git")], activity: "Running git" },
90 + {
91 + name: "reasoning, without its text",
92 + entries: [
93 + prompt,
94 + work("thought", { itemType: "reasoning", tone: "thinking", detail: "Private idea." }),
95 + ],
96 + activity: "Thinking",
97 + },
98 + {
99 + name: "a file change, with the verb its icon stands for",
100 + entries: [prompt, work("edit", { itemType: "file_change", changedFiles: ["search.ts"] })],
101 + activity: "Editing search.ts",
102 + },
103 + {
104 + name: "the step under a request and a notice",
105 + entries: [prompt, command("git"), work("ask", { itemType: "approval_request" }), notice],
106 + activity: "Running git",
107 + },
108 + {
109 + name: "nothing for a failed step",
110 + entries: [prompt, command("git", { toolLifecycleStatus: "failed" })],
111 + activity: null,
112 + },
113 + {
114 + name: "nothing once the agent is writing again",
115 + entries: [prompt, command("git"), message("reply", "assistant", { streaming: true })],
116 + activity: null,
117 + },
118 + {
119 + name: "nothing while the context is compacted",
120 + entries: [
121 + prompt,
122 + command("git"),
123 + work("compact", { tone: "info", sourceActivityKind: "context-compaction" }),
124 + ],
125 + activity: null,
126 + },
127 + {
128 + name: "nothing from the turn before",
129 + entries: [message("earlier", "user"), command("git"), prompt],
130 + activity: null,
131 + },
132 + ])("names $name", ({ entries, activity }) => {
133 + expect(turnDetail(entries, undefined, null).activity).toBe(activity);
134 + });
135 +
136 + it.each<{
137 + name: string;
138 + entries: ReadonlyArray<TimelineEntry>;
139 + finished: ReadonlyArray<string>;
140 + }>([
141 + {
142 + name: "a message the agent kept working after, but not its last",
143 + entries: [
144 + prompt,
145 + message("first", "assistant"),
146 + command("git"),
147 + message("last", "assistant"),
148 + ],
149 + finished: ["first"],
150 + },
151 + {
152 + name: "a message once the next one starts",
153 + entries: [
154 + prompt,
155 + message("first", "assistant"),
156 + message("second", "assistant", { streaming: true }),
157 + ],
158 + finished: ["first"],
159 + },
160 + {
161 + name: "no reply a steer cut off",
162 + entries: [
163 + prompt,
164 + inAttempt(message("cut off", "assistant"), "attempt-1", "superseded"),
165 + steer,
166 + inAttempt(command("git"), "attempt-2", "running"),
167 + ],
168 + finished: [],
169 + },
170 + {
171 + name: "a reply that completed before a steer replaced its attempt",
172 + entries: [
173 + prompt,
174 + completedAt(
175 + inAttempt(message("complete", "assistant"), "attempt-1", "superseded", "00:00:10"),
176 + "00:00:04",
177 + ),
178 + steer,
179 + inAttempt(command("git"), "attempt-2", "running"),
180 + ],
181 + finished: ["complete"],
182 + },
183 + {
184 + name: "no last message that only a notice follows",
185 + entries: [prompt, command("git"), message("last", "assistant"), notice],
186 + finished: [],
187 + },
188 + {
189 + name: "no last message when another run picks the thread up",
190 + entries: [
191 + prompt,
192 + message("last", "assistant"),
193 + command("git", { runId: RunId.make("run-2") }),
194 + ],
195 + finished: [],
196 + },
197 + ])("counts $name as finished", ({ entries, finished }) => {
198 + const { messages } = turnDetail(entries, undefined, null);
199 + expect(messages.filter((spoken) => spoken.finished).map((spoken) => spoken.text)).toEqual(
200 + finished,
201 + );
202 + });
203 +
204 + it.each<{ name: string; knownTailId: string | null; fresh: ReadonlyArray<boolean> }>([
205 + { name: "after the entry last seen", knownTailId: "seen", fresh: [false, true] },
206 + { name: "after a prompt that was last seen", knownTailId: "prompt", fresh: [true, true] },
207 + { name: "not when the entry last seen is gone", knownTailId: "gone", fresh: [false, false] },
208 + ])("marks messages as fresh $name", ({ knownTailId, fresh }) => {
209 + const entries = [
210 + prompt,
211 + message("loaded", "assistant"),
212 + command("seen"),
213 + message("new", "assistant"),
214 + command("git"),
215 + ];
216 + expect(
217 + turnDetail(entries, undefined, knownTailId).messages.map((spoken) => spoken.fresh),
218 + ).toEqual(fresh);
219 + });
220 +});
221 +
222 +type Frame = Pick<TurnDetailSnapshot, "status" | "activity" | "messages">;
223 +const said = (id: string, finished = true, fresh = false): TurnMessage => ({
224 + id,
225 + text: id,
226 + finished,
227 + fresh,
228 +});
229 +const frame = (
230 + activity: string | null,
231 + messages: ReadonlyArray<TurnMessage> = [],
232 + status: TurnDetailSnapshot["status"] = "working",
233 +): Frame => ({ status, activity, messages });
234 +
235 +function speak(level: ScreenReaderAnnouncements, frames: ReadonlyArray<Frame>) {
236 + let memory: TurnDetailMemory | null = null;
237 + const spoken: string[] = [];
238 + for (const next of frames) {
239 + const result = nextTurnDetailAnnouncement(memory, { ...next, level, tailId: null });
240 + memory = result.memory;
241 + spoken.push(...result.messages);
242 + if (result.activity !== null) spoken.push(result.activity);
243 + }
244 + return spoken;
245 +}
246 +
247 +describe("nextTurnDetailAnnouncement", () => {
248 + const turn = [
249 + frame(null),
250 + frame("Running git", [said("first", false)]),
251 + frame("Running git", [said("first", false)]),
252 + frame("Reading a.ts", [said("first"), said("second", false)]),
253 + frame(null, [said("first"), said("second")]),
254 + ];
255 +
256 + it.each<{ level: ScreenReaderAnnouncements; spoken: ReadonlyArray<string> }>([
257 + { level: "status", spoken: [] },
258 + { level: "messages", spoken: ["T3 Code: first", "T3 Code: second"] },
259 + {
260 + level: "everything",
261 + spoken: ["Running git", "T3 Code: first", "Reading a.ts", "T3 Code: second"],
262 + },
263 + ])("says each message and step once at the $level level", ({ level, spoken }) => {
264 + expect(speak(level, turn)).toEqual(spoken);
265 + });
266 +
267 + it("stays silent about what was there when the thread opened", () => {
268 + expect(
269 + speak("everything", [
270 + frame("Running git", [said("first")]),
271 + frame("Running git", [said("first")]),
272 + frame("Reading a.ts", [said("first"), said("second", true, true)]),
273 + ]),
274 + ).toEqual(["T3 Code: second", "Reading a.ts"]);
275 + });
276 +
277 + it("stays silent about history loaded above the messages it knows", () => {
278 + expect(
279 + speak("messages", [
280 + frame(null, [said("recent")]),
281 + frame(null, [said("older"), said("recent")]),
282 + frame(null, [said("older"), said("recent")]),
283 + ]),
284 + ).toEqual([]);
285 + });
286 +
287 + it("reads Markdown as the text it renders and skips a message with none", () => {
288 + const first = {
289 + ...said("first", true, true),
290 + text: "**Tests** pass in `web`.\n\n- See [the docs](https://x.test/a(b)c?token=secret)<br>",
291 + };
292 + expect(
293 + speak("messages", [
294 + frame(null),
295 + frame(null, [first, { ...said("empty", true, true), text: "<!-- note -->" }]),
296 + ]),
297 + ).toEqual(["T3 Code: Tests pass in web. See the docs"]);
298 + });
299 +
300 + it("says a step again after waiting on the user", () => {
301 + expect(
302 + speak("everything", [
303 + frame(null),
304 + frame("Running git"),
305 + frame("Running git", [], "approval"),
306 + frame("Running git"),
307 + ]),
308 + ).toEqual(["Running git", "Running git"]);
309 + });
310 +
311 + it("stays silent about a turn that already ended", () => {
312 + expect(
313 + speak("everything", [
314 + frame(null, [], "idle"),
315 + frame("Running git", [said("first", true, true)], "idle"),
316 + frame("Running git", [said("first")]),
317 + ]),
318 + ).toEqual(["Running git"]);
319 + });
320 +});
apps/web/src/components/chat/turnDetailAnnouncement.ts new
+202
@@ -0,0 +1,202 @@
1 +import type { RunId, ScreenReaderAnnouncements } from "@t3tools/contracts";
2 +import * as DateTime from "effect/DateTime";
3 +
4 +import type { TimelineEntry, WorkLogEntry } from "../../session-logic";
5 +import {
6 + liveWorkEntryLabel,
7 + timelineEntryRunId,
8 + timelineEntryStartsResponse,
9 +} from "./MessagesTimeline.logic";
10 +import { markdownPlainText } from "./timelineMinimapItems";
11 +
12 +export interface TurnMessage {
13 + readonly id: string;
14 + /** Markdown. */
15 + readonly text: string;
16 + /** The agent stopped writing it and went on working. A turn's closing reply never is. */
17 + readonly finished: boolean;
18 + /** It arrived after the entry the caller last saw, unlike history loaded above it. */
19 + readonly fresh: boolean;
20 +}
21 +
22 +export interface TurnDetail {
23 + /** The step the agent is on, or null when its newest output is not a step. */
24 + readonly activity: string | null;
25 + /** The agent's messages in the latest turn, oldest first. */
26 + readonly messages: ReadonlyArray<TurnMessage>;
27 + /** The thread's newest entry. */
28 + readonly tailId: string | null;
29 +}
30 +
31 +export const NO_TURN_DETAIL: TurnDetail = { activity: null, messages: [], tailId: null };
32 +
33 +const ACTIVITY_MAX_LENGTH = 120;
34 +// Parsing is synchronous, and nobody listens to a longer message through a live region.
35 +const MESSAGE_MAX_LENGTH = 2_000;
36 +
37 +// On screen these labels are a path or a query beside an icon.
38 +function withVerb(work: WorkLogEntry, label: string) {
39 + if (work.itemType === "file_change") return `Editing ${label}`;
40 + if (work.itemType !== "file_search" && work.itemType !== "web_search") return label;
41 + if (/^search(?:ed|ing)?\s/i.test(label)) return label;
42 + return `${work.itemType === "web_search" ? "Searching the web for" : "Searching for"} ${label}`;
43 +}
44 +
45 +type MessageEntry = Extract<TimelineEntry, { kind: "message" }>;
46 +
47 +// A steer starts a new attempt in the same run. The reply it cut off also stops streaming,
48 +// so only one that completed before the steer finished; see completeSupersededReplyEntryIds.
49 +function completedBeforeSuperseded(entry: MessageEntry) {
50 + const item = entry.projectedItem?.item;
51 + const supersededAt = entry.attempt?.completedAt ?? null;
52 + return (
53 + item?.type === "assistant_message" &&
54 + item.status === "completed" &&
55 + item.completedAt !== null &&
56 + supersededAt !== null &&
57 + DateTime.isLessThan(item.completedAt, supersededAt)
58 + );
59 +}
60 +
61 +function stepLabel(work: WorkLogEntry, workspaceRoot: string | undefined): string | null {
62 + // Reasoning text is never read out.
63 + if (work.itemType === "reasoning") return "Thinking";
64 + const status = work.toolLifecycleStatus;
65 + if (status === "failed" || status === "declined" || status === "stopped") return null;
66 + const label = liveWorkEntryLabel(work, workspaceRoot, true).replace(/\s+/g, " ").trim();
67 + if (label === "") return null;
68 + const spoken = withVerb(work, label);
69 + return spoken.length > ACTIVITY_MAX_LENGTH ? `${spoken.slice(0, ACTIVITY_MAX_LENGTH)}…` : spoken;
70 +}
71 +
72 +/**
73 + * What a screen reader can be told about the latest turn beyond its status.
74 + * It reads the thread's entries before "Tool calls in chat" hides any, so what
75 + * is spoken follows the announcement setting alone.
76 + */
77 +export function turnDetail(
78 + entries: ReadonlyArray<TimelineEntry>,
79 + workspaceRoot: string | undefined,
80 + knownTailId: string | null,
81 +): TurnDetail {
82 + const messages: TurnMessage[] = [];
83 + // Runs and attempts the agent kept working in after the entry being looked at.
84 + const continuedRunIds = new Set<RunId | null>();
85 + const continuedAttemptIds = new Set<string>();
86 + const markContinued = (entry: TimelineEntry, runId: RunId | null) => {
87 + continuedRunIds.add(runId);
88 + if (entry.attempt !== undefined) continuedAttemptIds.add(entry.attempt.id);
89 + };
90 + let activity: string | null | undefined;
91 + let fresh = true;
92 + let index = entries.length - 1;
93 + for (; index >= 0; index -= 1) {
94 + const entry = entries[index]!;
95 + if (timelineEntryStartsResponse(entry)) break;
96 + if (entry.id === knownTailId) fresh = false;
97 + if (entry.kind === "event") continue;
98 + if (entry.kind === "work") {
99 + const work = entry.entry;
100 + if (work.sourceActivityKind === "context-compaction") {
101 + activity ??= null;
102 + continue;
103 + }
104 + // Notices, todo lists and checkpoints sit beside the turn's steps without being one.
105 + if (work.tone !== "tool" && work.itemType !== "reasoning") continue;
106 + markContinued(entry, work.runId ?? null);
107 + // A request is spoken as the thread's status; once answered, the step it was for goes on.
108 + if (work.itemType === "approval_request" || work.itemType === "user_input_request") continue;
109 + if (activity === undefined) activity = stepLabel(work, workspaceRoot);
110 + continue;
111 + }
112 + if (entry.kind === "message" && entry.message.role !== "assistant") continue;
113 + // A reply, plan or rendered page is newer than any step before it.
114 + activity ??= null;
115 + const runId = timelineEntryRunId(entry);
116 + if (entry.kind === "message") {
117 + messages.push({
118 + id: entry.message.id,
119 + text: entry.message.text,
120 + finished:
121 + !entry.message.streaming &&
122 + continuedRunIds.has(runId) &&
123 + (entry.attempt?.status !== "superseded" ||
124 + continuedAttemptIds.has(entry.attempt.id) ||
125 + completedBeforeSuperseded(entry)),
126 + fresh,
127 + });
128 + }
129 + markContinued(entry, runId);
130 + }
131 + // Nothing is known to be new when the entry last seen is gone, or there was none.
132 + let tailKnown = !fresh;
133 + if (knownTailId !== null) {
134 + for (; !tailKnown && index >= 0; index -= 1) tailKnown = entries[index]!.id === knownTailId;
135 + }
136 + return {
137 + activity: activity ?? null,
138 + messages: (tailKnown
139 + ? messages
140 + : messages.map((message) => ({ ...message, fresh: false }))
141 + ).reverse(),
142 + tailId: entries.at(-1)?.id ?? null,
143 + };
144 +}
145 +
146 +export interface TurnDetailSnapshot extends TurnDetail {
147 + readonly level: ScreenReaderAnnouncements;
148 + readonly status: "working" | "approval" | "input" | "idle";
149 +}
150 +
151 +export interface TurnDetailMemory {
152 + readonly activity: string | null;
153 + /** Each message seen in the turn, and whether it was already finished then. */
154 + readonly messages: ReadonlyMap<string, boolean>;
155 + readonly tailId: string | null;
156 +}
157 +
158 +/** The heading every agent message has in the timeline. */
159 +const MESSAGE_AUTHOR = "T3 Code";
160 +
161 +/**
162 + * Decides which of the turn's messages and steps a screen reader hears next.
163 + * The first snapshot of a thread is a silent baseline, like its status.
164 + */
165 +export function nextTurnDetailAnnouncement(
166 + previous: TurnDetailMemory | null,
167 + next: TurnDetailSnapshot,
168 +): {
169 + readonly memory: TurnDetailMemory;
170 + readonly messages: ReadonlyArray<string>;
171 + readonly activity: string | null;
172 +} {
173 + const memory: TurnDetailMemory = {
174 + // A step is spoken again after a pause for an approval or a question.
175 + activity: next.status === "working" ? next.activity : null,
176 + messages: new Map(next.messages.map((message) => [message.id, message.finished])),
177 + tailId: next.tailId,
178 + };
179 + // A turn that already ended is history, however late its entries arrive.
180 + if (previous === null || next.level === "status" || next.status === "idle") {
181 + return { memory, messages: [], activity: null };
182 + }
183 + return {
184 + memory,
185 + messages: next.messages
186 + .filter(
187 + (message) =>
188 + message.finished &&
189 + previous.messages.get(message.id) !== true &&
190 + (message.fresh || previous.messages.has(message.id)),
191 + )
192 + .map((message) =>
193 + message.text.length > MESSAGE_MAX_LENGTH
194 + ? `${markdownPlainText(message.text.slice(0, MESSAGE_MAX_LENGTH))} More in the thread.`
195 + : markdownPlainText(message.text),
196 + )
197 + .filter((text) => text !== "")
198 + .map((text) => `${MESSAGE_AUTHOR}: ${text}`),
199 + activity:
200 + next.level === "everything" && memory.activity !== previous.activity ? memory.activity : null,
201 + };
202 +}
apps/web/src/components/settings/SettingsPanels.tsx
+65
@@ -22,6 +22,7 @@ import {
22 type ResponseStreamingMode,
23 type QuitConfirmationMode,
24 SidebarProjectSortOrder,
25 + ScreenReaderAnnouncements,
26 ToolCallVisibility,
27 } from "@t3tools/contracts/settings";
28 import { desktopUpdateChannelOf } from "@t3tools/shared/cliRelease";
@@ -539,6 +540,9 @@ export function useSettingsRestore(onRestored?: () => void) {
540 ...(settings.toolCallVisibility !== DEFAULT_UNIFIED_SETTINGS.toolCallVisibility
541 ? ["Tool calls in chat"]
542 : []),
543 + ...(settings.screenReaderAnnouncements !== DEFAULT_UNIFIED_SETTINGS.screenReaderAnnouncements
544 + ? ["Screen reader announcements"]
545 + : []),
546 ...(settings.responseStreamingMode !== DEFAULT_UNIFIED_SETTINGS.responseStreamingMode
547 ? ["Response streaming"]
548 : []),
@@ -654,6 +658,7 @@ export function useSettingsRestore(onRestored?: () => void) {
658 sendShortcut: DEFAULT_UNIFIED_SETTINGS.sendShortcut,
659 followUpBehavior: DEFAULT_UNIFIED_SETTINGS.followUpBehavior,
660 toolCallVisibility: DEFAULT_UNIFIED_SETTINGS.toolCallVisibility,
661 + screenReaderAnnouncements: DEFAULT_UNIFIED_SETTINGS.screenReaderAnnouncements,
662 environmentIdentificationMode: DEFAULT_UNIFIED_SETTINGS.environmentIdentificationMode,
663 glassOpacity: DEFAULT_UNIFIED_SETTINGS.glassOpacity,
664 panelAnimationDurationMs: DEFAULT_UNIFIED_SETTINGS.panelAnimationDurationMs,
@@ -1208,6 +1213,7 @@ export function ChatSettingsPanel() {
1213
1214 <SettingsSection id="activity" title="Activity">
1215 <ToolCallVisibilityRow />
1216 + <ScreenReaderAnnouncementsRow />
1217 </SettingsSection>
1218 </SettingsPageContainer>
1219 );
@@ -1262,6 +1268,65 @@ function ToolCallVisibilityRow() {
1268 );
1269 }
1270
1271 +const SCREEN_READER_ANNOUNCEMENTS_LABELS: Record<ScreenReaderAnnouncements, string> = {
1272 + status: "Status only",
1273 + messages: "Status and messages",
1274 + everything: "Everything",
1275 +};
1276 +
1277 +const isScreenReaderAnnouncements = Schema.is(ScreenReaderAnnouncements);
1278 +
1279 +function ScreenReaderAnnouncementsRow() {
1280 + const settings = useScopedSettings();
1281 + const updateSettings = useUpdateScopedSettings();
1282 + return (
1283 + <SettingsRow
1284 + {...searchableSetting("screen-reader-announcements")}
1285 + description="What a screen reader hears from the open thread: a turn starting, needing you, and ending. Status and messages adds what the agent writes before its last message. Everything adds each step."
1286 + resetAction={
1287 + settings.screenReaderAnnouncements !==
1288 + DEFAULT_UNIFIED_SETTINGS.screenReaderAnnouncements ? (
1289 + <SettingResetButton
1290 + label="screen reader announcements"
1291 + onClick={() =>
1292 + updateSettings({
1293 + screenReaderAnnouncements: DEFAULT_UNIFIED_SETTINGS.screenReaderAnnouncements,
1294 + })
1295 + }
1296 + />
1297 + ) : null
1298 + }
1299 + control={
1300 + <Select
1301 + value={settings.screenReaderAnnouncements}
1302 + onValueChange={(value) => {
1303 + if (isScreenReaderAnnouncements(value)) {
1304 + updateSettings({ screenReaderAnnouncements: value });
1305 + }
1306 + }}
1307 + >
1308 + <SelectTrigger
1309 + size="sm"
1310 + className="w-full sm:w-48"
1311 + aria-label="Screen reader announcements"
1312 + >
1313 + <SelectValue>
1314 + {SCREEN_READER_ANNOUNCEMENTS_LABELS[settings.screenReaderAnnouncements]}
1315 + </SelectValue>
1316 + </SelectTrigger>
1317 + <SelectPopup align="end" alignItemWithTrigger={false}>
1318 + {ScreenReaderAnnouncements.literals.map((value) => (
1319 + <SelectItem key={value} value={value}>
1320 + {SCREEN_READER_ANNOUNCEMENTS_LABELS[value]}
1321 + </SelectItem>
1322 + ))}
1323 + </SelectPopup>
1324 + </Select>
1325 + }
1326 + />
1327 + );
1328 +}
1329 +
1330 function DiffColorsSettingsRow() {
1331 const settings = useScopedSettings();
1332 const updateSettings = useUpdateScopedSettings();
apps/web/src/components/settings/settingsSearch.ts
+8
@@ -483,6 +483,14 @@ export const SETTINGS_SEARCH_ITEMS = [
483 "hide tool calls commands edits searches work log activity quiet chat zen final answers only",
484 ],
485 },
486 + {
487 + id: "screen-reader-announcements",
488 + title: "Screen reader announcements",
489 + to: "/settings/chat",
490 + searchTerms: [
491 + "accessibility a11y screen reader nvda jaws voiceover orca narrator speech announce read aloud messages activity steps",
492 + ],
493 + },
494 {
495 id: "guest-links",
496 title: "Guest links",
docs/user/activity-log.md
+12
@@ -29,3 +29,15 @@ and the agent's last reply, which appears when the turn ends. If you steer while
29 answering, the answer it finished before reading your steer stays too. On web and desktop, Find
30 shows the in-between messages while it is open. Approvals, questions, and errors always show. The
31 setting applies to this browser or device only.
32 +
33 +## Screen reader announcements
34 +
35 +On web and desktop, a screen reader hears the open thread's turn as it runs. **Screen reader
36 +announcements** in **Settings → Chat** sets how much. **Status only** says when a turn starts,
37 +needs your approval or input, and ends. **Status and messages** also reads each message the agent
38 +writes and then keeps working after, once that message is complete. A very long message is cut
39 +short. The reply that closes the turn is not read, so you can read it yourself when you hear
40 +**Done**. **Everything** adds the step the agent is on, such as the command it is running. Steps
41 +are spaced out, so a quick step can be skipped for a newer one.
42 +The agent's reasoning text is never read. The setting follows the agent's real activity, whatever
43 +**Tool calls in chat** hides, and applies to this browser or device only.
packages/contracts/src/settings.ts
+8
@@ -346,6 +346,10 @@ export type EffortDisplay = typeof EffortDisplay.Type;
346 export const ToolCallVisibility = Schema.Literals(["showAll", "hideSuccessful", "hideAll", "zen"]);
347 export type ToolCallVisibility = typeof ToolCallVisibility.Type;
348
349 +/** What the open thread reads to a screen reader: turn status, the agent's in-between messages too, or every step. */
350 +export const ScreenReaderAnnouncements = Schema.Literals(["status", "messages", "everything"]);
351 +export type ScreenReaderAnnouncements = typeof ScreenReaderAnnouncements.Type;
352 +
353 export const ModelFavorite = Schema.Struct({
354 provider: ProviderInstanceId,
355 model: TrimmedNonEmptyString,
@@ -517,6 +521,9 @@ export const ClientSettingsSchema = Schema.Struct({
521 toolCallVisibility: ToolCallVisibility.pipe(
522 Schema.withDecodingDefault(Effect.succeed("showAll" as const)),
523 ),
524 + screenReaderAnnouncements: ScreenReaderAnnouncements.pipe(
525 + Schema.withDecodingDefault(Effect.succeed("status" as const)),
526 + ),
527 // Retired Labs toggle. Still read as the default for the Usage item in threadDetailsItems.
528 threadDetailsUsageEnabled: Schema.Boolean.pipe(Schema.withDecodingDefault(Effect.succeed(false))),
529 // Sparse show/hide overrides for thread details card items. Missing ids use the client's
@@ -2029,6 +2036,7 @@ export const ClientSettingsPatch = Schema.Struct({
2036 boringSidebarEnabled: Schema.optionalKey(Schema.Boolean),
2037 boringSidebarMultilineRows: Schema.optionalKey(Schema.Boolean),
2038 toolCallVisibility: Schema.optionalKey(ToolCallVisibility),
2039 + screenReaderAnnouncements: Schema.optionalKey(ScreenReaderAnnouncements),
2040 threadDetailsUsageEnabled: Schema.optionalKey(Schema.Boolean),
2041 threadDetailsItems: Schema.optionalKey(Schema.Record(Schema.String, Schema.Boolean)),
2042 sidebarWorkingShelfEnabled: Schema.optionalKey(Schema.Boolean),