The best way to control Claude Code, Codex and more from any computer or phone

main
ts 105 lines 4.11 KB
Raw
1 import { assert } from "@effect/vitest";
2 import type { ProviderReplayTranscript } from "@t3tools/contracts";
3
4 import type { OrchestratorV2ScenarioResult } from "../../OrchestratorScenario.ts";
5 import { projectionFor } from "../shared.ts";
6 import { assertSimpleOutput } from "./codex_output.ts";
7
8 function isRecord(value: unknown): value is Record<string, unknown> {
9 return typeof value === "object" && value !== null && !Array.isArray(value);
10 }
11
12 function field(value: unknown, key: string): unknown {
13 return isRecord(value) ? value[key] : undefined;
14 }
15
16 function lastPiResponseData(transcript: ProviderReplayTranscript, command: string): unknown {
17 const frame = transcript.entries
18 .flatMap((entry) => (entry.type === "emit_inbound" ? [entry.frame] : []))
19 .findLast(
20 (candidate) =>
21 field(candidate, "type") === "response" && field(candidate, "command") === command,
22 );
23 assert.isDefined(frame, `transcript must record a ${command} response`);
24 return field(frame, "data");
25 }
26
27 /** Settled Pi usage uses each recorded get_session_stats response in order, not the last streaming total. */
28 export function assertPiSettledTokenUsage(
29 result: OrchestratorV2ScenarioResult,
30 transcript: ProviderReplayTranscript,
31 ) {
32 const stats = transcript.entries.flatMap((entry) =>
33 entry.type === "emit_inbound" &&
34 field(entry.frame, "type") === "response" &&
35 field(entry.frame, "command") === "get_session_stats"
36 ? [field(entry.frame, "data")]
37 : [],
38 );
39 const turns = projectionFor(result, transcript.scenario).providerTurns;
40 assert.lengthOf(turns, stats.length);
41 for (const [index, turn] of turns.entries()) {
42 const contextUsage = field(stats[index], "contextUsage");
43 const tokens = field(stats[index], "tokens");
44 const { updatedAt: _updatedAt, ...tokenUsage } = turn.tokenUsage ?? { updatedAt: "" };
45 assert.deepEqual(tokenUsage, {
46 usedTokens: field(contextUsage, "tokens"),
47 maxTokens: field(contextUsage, "contextWindow"),
48 inputTokens: field(tokens, "input"),
49 cachedInputTokens: field(tokens, "cacheRead"),
50 outputTokens: field(tokens, "output"),
51 });
52 }
53 }
54
55 export function assertPiSimpleOutput(
56 result: OrchestratorV2ScenarioResult,
57 transcript: ProviderReplayTranscript,
58 ) {
59 assertSimpleOutput(result, transcript);
60 const projection = projectionFor(result, transcript.scenario);
61
62 const recordedThinking = transcript.entries.flatMap((entry) => {
63 const update = field(
64 entry.type === "emit_inbound" ? entry.frame : undefined,
65 "assistantMessageEvent",
66 );
67 return field(update, "type") === "thinking_end" ? [field(update, "content")] : [];
68 });
69 assert.lengthOf(recordedThinking, 1, "the recording must stream one thinking block");
70 assert.deepEqual(
71 projection.turnItems.flatMap((item) => (item.type === "reasoning" ? [item.text] : [])),
72 recordedThinking,
73 "Pi thinking deltas must project as one reasoning item with the recorded text",
74 );
75
76 assertPiSettledTokenUsage(result, transcript);
77 const [turn] = projection.providerTurns;
78
79 // Pi streams cumulative usage, initially zeros until the provider reports.
80 const streamedTotals = transcript.entries.flatMap((entry) =>
81 entry.type === "emit_inbound" && field(entry.frame, "type") === "message_update"
82 ? [field(field(entry.frame, "usage"), "totalTokens")]
83 : [],
84 );
85 const liveUsage = result.domainEvents.flatMap((event) =>
86 event.type === "provider-turn.updated" &&
87 event.payload.status === "running" &&
88 event.payload.tokenUsage !== undefined
89 ? [event.payload.tokenUsage.usedTokens]
90 : [],
91 );
92 assert.deepEqual(liveUsage, [...new Set(streamedTotals.filter((total) => total !== 0))]);
93
94 const entries = field(lastPiResponseData(transcript, "get_entries"), "entries");
95 const userEntryId = (Array.isArray(entries) ? entries : [])
96 .filter(
97 (entry) =>
98 field(entry, "type") === "message" && field(field(entry, "message"), "role") === "user",
99 )
100 .map((entry) => field(entry, "id"))
101 .at(0);
102 assert.isString(userEntryId);
103 assert.equal(turn?.nativeTurnRef?.nativeId, userEntryId);
104 assert.equal(turn?.nativeTurnRef?.strength, "strong");
105 }