@devries-cameron20 / t3code-1 / commits / bc647ed535
Fork 0

The best way to control Claude Code, Codex and more from any computer or phone

fix(server): title requests about referenced threads accurately (#34723)

Yash Singh committed Nov 11, 2026 at 22:11 UTC bc647ed53595888efacdd8ca435967d48888a952
4 files changed +38
evals/thread-titles/README.md
+1
@@ -11,6 +11,7 @@ pnpm thread-titles --model <codex model> --out runs/titles-new --baseline runs/t
11 ```
12
13 - It uses your local Codex login, like the server does.
14 +- `--case <id>` limits the run to one case. Repeat the flag for multiple cases.
15 - `--initial` titles from the first message only, as a new thread does.
16 - A run writes `results.json` (titles and latency), `review.json` (titles A and B in random order, with a rubric), and `answer-key.json`. Score `review.json` before you open `answer-key.json`.
17
evals/thread-titles/cases.ts
+28
@@ -1,6 +1,34 @@
1 import type { ThreadTitleMessage } from "@t3tools/provider-core/server/threadTitleContext";
2
3 export const threadTitleEvaluationCases = [
4 + {
5 + id: "referenced-thread-startup",
6 + source: "Maintainer report of chart thread references overriding a startup question",
7 + request: "Explain why one example thread failed to start and another needed a retry.",
8 + previousTitle: "Fix Chart Formatting",
9 + messages: [
10 + {
11 + role: "user",
12 + text: "[xlsx: chart data labels use the cells' number format](t3-context://v1/thread/thread_labels) why could this thread not start up? [Run chart format tests on external target](t3-context://v1/thread/thread_tests) why did i have to say it like this over here to get it working?",
13 + },
14 + ],
15 + rubric:
16 + "Name thread startup or prompt/retry behavior. Chart formatting and tests are titles of example threads, not the requested work. Replace the inaccurate previous title.",
17 + },
18 + {
19 + id: "referenced-thread-subject",
20 + source: "Control for maintainer report of referenced thread titles overriding the request",
21 + request: "Continue the actual chart formatting work in a referenced thread.",
22 + previousTitle: "Chart Label Number Formats",
23 + messages: [
24 + {
25 + role: "user",
26 + text: "Continue [xlsx: chart data labels use the cells' number format](t3-context://v1/thread/thread_labels). Make the chart labels inherit the cells' number formats.",
27 + },
28 + ],
29 + rubric:
30 + "Name chart label number formatting. Referenced titles can supply the subject when the user explicitly continues that work.",
31 + },
32 {
33 id: "linked-reset-credits",
34 source: "https://github.com/pingdotgg/t3code/pull/10462",
evals/thread-titles/run.ts
+7
@@ -32,6 +32,7 @@ import * as ServerConfig from "../../apps/server/src/config.ts";
32 const { values } = NodeUtil.parseArgs({
33 options: {
34 model: { type: "string" },
35 + case: { type: "string", multiple: true },
36 out: { type: "string" },
37 baseline: { type: "string" },
38 initial: { type: "boolean", default: false },
@@ -39,6 +40,11 @@ const { values } = NodeUtil.parseArgs({
40 });
41 if (!values.model || !values.out)
42 throw new Error("Use --model <configured-model> --out <directory>.");
43 +for (const id of values.case ?? []) {
44 + if (!threadTitleEvaluationCases.some((fixture) => fixture.id === id)) {
45 + throw new Error(`Unknown case id: ${id}.`);
46 + }
47 +}
48 const model = values.model;
49 const outputDirectory = values.out;
50 const Results = Schema.fromJsonString(
@@ -70,6 +76,7 @@ await Effect.runPromise(
76 const review = [];
77 const answerKey = [];
78 for (const fixture of threadTitleEvaluationCases) {
79 + if (values.case?.length && !values.case.includes(fixture.id)) continue;
80 const previous = baseline.find((entry) => entry.id === fixture.id);
81 if (values.baseline && !previous) throw new Error(`Baseline is missing ${fixture.id}.`);
82 const firstMessage: ThreadTitleMessage | undefined = fixture.messages.find(
packages/provider-core/src/server/textGenerationPrompts.ts
+2
@@ -391,6 +391,7 @@ Editorial rules:
391 - 3-8 words, fewer than 40 characters.
392 - Use a compact noun phrase or clear action phrase.
393 - Capture the umbrella goal when the request lists several symptoms or steps.
394 +- When asked about thread behavior, never include the referenced titles' topics. "[Fix login] why didn't this thread start?" means "Thread Startup Failure". Use linked topics only for requests about that work.
395 - Name the product change, not the mock, plan, report, branch, or PR used to produce it.
396 - Models, subagents, tools, output formats, and monitoring instructions do not belong in the title unless they are themselves the topic.
397 - For reviews, name what is being reviewed and the relevant concern. Avoid generic titles such as "Review PR 123" when linked or attached context reveals the subject.
@@ -422,6 +423,7 @@ Editorial rules:
423 - 3-8 words, fewer than 40 characters.
424 - Use a compact noun phrase or clear action phrase.
425 - Preserve the umbrella subject when later messages focus on one finding, provider, platform, or implementation detail.
426 +- When asked about thread behavior, never include the referenced titles' topics. "[Fix login] why didn't this thread start?" means "Thread Startup Failure". Use linked topics only for requests about that work.
427 - A thread progressing through research, planning, implementation, review, CI, merge, and monitoring has usually not changed subjects.
428 - Ignore deliverables and operations such as mocks, plans, HTML, branches, PRs, tests, CI, commits, merging, and monitoring unless they are the actual topic.
429 - Models, subagents, tools, output formats, and monitoring instructions do not belong in the title unless they are themselves the topic.