@devries-cameron20 / t3code-1 / commits / 3121c19a1b
Fork 0

The best way to control Claude Code, Codex and more from any computer or phone

fix(usage): count compressed Codex rollouts (#25453)

Co-authored-by: Michael <208665133+Trench-Monster@users.noreply.github.com>

maria committed Oct 30, 2026 at 21:14 UTC 3121c19a1b68e34fac880cf9eeb575e89e3679c6
3 files changed +187 -7
apps/server/src/usage/UsageService.test.ts
+68
@@ -5,6 +5,7 @@ import * as NodeFSP from "node:fs/promises";
5 import * as NodeOS from "node:os";
6 import * as NodePath from "node:path";
7 import * as NodeSqlite from "node:sqlite";
8 +import * as NodeZlib from "node:zlib";
9
10 import { assert, describe, it } from "@effect/vitest";
11 import { vi } from "vite-plus/test";
@@ -467,6 +468,73 @@ describe("Codex usage history", () => {
468 });
469
470 describe("UsageService", () => {
471 + it.live("counts compressed Codex rollouts across refresh, rewrites, and cache restarts", () =>
472 + Effect.gen(function* () {
473 + const { home, settings } = yield* setup;
474 + const sessions = NodePath.join(home, "codex", "sessions");
475 + const compressedPath = NodePath.join(sessions, "compressed.jsonl.zst");
476 + const usage = (output_tokens: number, timestamp = "2026-08-01T10:00:00Z") =>
477 + encodeUnknownJsonString({
478 + type: "event_msg",
479 + timestamp,
480 + payload: {
481 + type: "token_count",
482 + info: { last_token_usage: { input_tokens: 10, output_tokens } },
483 + },
484 + }) + "\n";
485 + const rollout = (id: string, output: number) =>
486 + [
487 + { type: "session_meta", payload: { id } },
488 + { type: "turn_context", payload: { model: "gpt-6-astra" } },
489 + ]
490 + .map((line) => encodeUnknownJsonString(line))
491 + .join("\n") +
492 + "\n" +
493 + usage(output);
494 + const compressed = rollout("compressed-session", 23);
495 + yield* Effect.promise(async () => {
496 + await NodeFSP.mkdir(sessions, { recursive: true });
497 + await NodeFSP.writeFile(
498 + NodePath.join(sessions, "plain.jsonl"),
499 + rollout("plain-session", 17),
500 + );
501 + await NodeFSP.writeFile(compressedPath, NodeZlib.zstdCompressSync(compressed));
502 + });
503 + yield* Effect.gen(function* () {
504 + const service = yield* UsageService.make;
505 + const first = yield* service.readSummary(WINDOW);
506 + assert.strictEqual(totalOutputTokens(first), 40);
507 + assert.strictEqual(
508 + first.sources.find((source) => source.fingerprint.provider === "codex")?.distinctSessions,
509 + 2,
510 + );
511 + assert.deepStrictEqual((yield* service.readSummary(WINDOW)).buckets, first.buckets);
512 + yield* service.awaitPersisted;
513 + // The same rollout may remain in a plain file while compression finishes.
514 + yield* Effect.promise(() =>
515 + NodeFSP.writeFile(NodePath.join(sessions, "copy.jsonl"), compressed),
516 + );
517 + assert.strictEqual(totalOutputTokens(yield* service.readSummary(WINDOW)), 40);
518 + yield* Effect.promise(() =>
519 + NodeFSP.writeFile(
520 + compressedPath,
521 + NodeZlib.zstdCompressSync(compressed + usage(31, "2026-08-01T10:00:05Z")),
522 + ),
523 + );
524 + const rewritten = yield* service.readSummary(WINDOW);
525 + assert.strictEqual(totalOutputTokens(rewritten), 71);
526 + assert.deepStrictEqual((yield* service.readSummary(WINDOW)).buckets, rewritten.buckets);
527 + yield* service.awaitPersisted;
528 + const restarted = yield* UsageService.make;
529 + assert.deepStrictEqual((yield* restarted.readSummary(WINDOW)).buckets, rewritten.buckets);
530 + yield* restarted.awaitPersisted;
531 + }).pipe(
532 + Effect.scoped,
533 + Effect.provide(layerService({ prefix: "usage-compressed-codex", home, settings })),
534 + );
535 + }).pipe(Effect.scoped),
536 + );
537 +
538 it.live("keeps the first host identity across recreation and explicit override removal", () =>
539 Effect.gen(function* () {
540 const { transcript, settings, home } = yield* setup;
apps/server/src/usage/usageTranscriptReader.test.ts
+103 -1
@@ -4,13 +4,18 @@
4 import * as NodeFSP from "node:fs/promises";
5 import * as NodeOS from "node:os";
6 import * as NodePath from "node:path";
7 +import * as NodeZlib from "node:zlib";
8
9 import { afterEach, assert, beforeEach, describe, it } from "@effect/vitest";
10 import type { TranscriptUsageFormat } from "@t3tools/provider-core/server/usage";
11 import * as Schema from "effect/Schema";
12
13 import { TEST_FORMATS } from "./usageTestFormats.ts";
13 -import { readTranscriptRecords, transcriptLineGate } from "./usageTranscriptReader.ts";
14 +import {
15 + listTranscriptFiles,
16 + readTranscriptRecords,
17 + transcriptLineGate,
18 +} from "./usageTranscriptReader.ts";
19
20 let dir: string;
21
@@ -71,6 +76,103 @@ function codexUsageLine(outputTokens: number, secondsOffset: number): string {
76 })}\n`;
77 }
78
79 +describe("compressed transcripts", () => {
80 + it("discovers nested plain and zstd JSONL files within the requested window", async () => {
81 + const nested = NodePath.join(dir, "nested");
82 + await NodeFSP.mkdir(nested);
83 + const plain = NodePath.join(dir, "active.jsonl");
84 + const compressed = NodePath.join(nested, "archived.jsonl.zst");
85 + const old = NodePath.join(nested, "old.jsonl.zst");
86 + await NodeFSP.writeFile(plain, claudeLine(1, 5));
87 + await NodeFSP.writeFile(compressed, NodeZlib.zstdCompressSync(claudeLine(2, 7)));
88 + await NodeFSP.writeFile(old, NodeZlib.zstdCompressSync(claudeLine(3, 11)));
89 + await NodeFSP.writeFile(NodePath.join(nested, "unrelated.zst"), "ignored");
90 + await NodeFSP.writeFile(NodePath.join(nested, "partial.jsonl.zst.tmp"), "ignored");
91 + await NodeFSP.utimes(old, 1, 1);
92 +
93 + const listed = await listTranscriptFiles(dir, Date.parse("2026-01-01T00:00:00Z"));
94 + assert.deepStrictEqual(
95 + listed.files.map((file) => file.path).sort(),
96 + [plain, compressed].sort(),
97 + );
98 + assert.strictEqual(listed.failedPaths, 0);
99 + assert.isAbove(listed.files.find((file) => file.path === compressed)?.size ?? 0, 0);
100 + });
101 +
102 + it.each([
103 + { parser: "native", streamingThresholdBytes: undefined },
104 + { parser: "projected", streamingThresholdBytes: 1024 },
105 + ])(
106 + "matches plain Codex records and unfinished tails through long lines ($parser)",
107 + async ({ streamingThresholdBytes }) => {
108 + const plain = NodePath.join(dir, "rollout.jsonl");
109 + const compressed = plain + ".zst";
110 + const large = codexUsageLine(9, 5).replace(
111 + '"payload":',
112 + '"padding":' + JSON.stringify("日本語 é".repeat(128 * 1024)) + ',"payload":',
113 + );
114 + const content =
115 + codexMetaLine() +
116 + codexModelLine("gpt-6-astra") +
117 + codexTierLine("ultrafast") +
118 + large +
119 + codexUsageLine(13, 6).replace('"output_tokens":13', '"output_tokens":"invalid"') +
120 + codexUsageLine(21, 8).trimEnd();
121 + await NodeFSP.writeFile(plain, content);
122 + await NodeFSP.writeFile(compressed, NodeZlib.zstdCompressSync(content));
123 +
124 + const options =
125 + streamingThresholdBytes === undefined ? undefined : { streamingThresholdBytes };
126 + const expected = await readTranscriptRecords(plain, TEST_FORMATS.codex, undefined, options);
127 + const actual = await readTranscriptRecords(
128 + compressed,
129 + TEST_FORMATS.codex,
130 + undefined,
131 + options,
132 + );
133 + assert.isNotNull(expected);
134 + assert.isNotNull(actual);
135 + assert.strictEqual(expected.records.length, 1);
136 + assert.strictEqual(expected.tailRecords.length, 1);
137 + assert.deepStrictEqual(actual.records, expected.records);
138 + assert.deepStrictEqual(actual.tailRecords, expected.tailRecords);
139 + assert.strictEqual(expected.malformedRecords, 1);
140 + assert.strictEqual(actual.malformedRecords, expected.malformedRecords);
141 + assert.strictEqual(actual.records[0]?.speed, "ultrafast");
142 + assert.isFalse(actual.resumed);
143 + assert.strictEqual(actual.position.resumeOffset, 0);
144 + },
145 + );
146 +
147 + it("ignores a prior resume position and rereads compressed rollouts from their start", async () => {
148 + const plain = NodePath.join(dir, "rollout.jsonl");
149 + const content = codexMetaLine() + codexModelLine("gpt-6-astra") + codexUsageLine(9, 5);
150 + await NodeFSP.writeFile(plain, content);
151 + const first = await readTranscriptRecords(plain, TEST_FORMATS.codex);
152 + assert.isNotNull(first);
153 +
154 + const compressed = plain + ".zst";
155 + await NodeFSP.writeFile(compressed, NodeZlib.zstdCompressSync(content + codexUsageLine(21, 8)));
156 + const actual = await readTranscriptRecords(compressed, TEST_FORMATS.codex, first.position);
157 + assert.isNotNull(actual);
158 + assert.isFalse(actual.resumed);
159 + assert.deepStrictEqual(
160 + actual.records.map((record) => record.totals.outputTokens),
161 + [9, 21],
162 + );
163 + const repeated = await readTranscriptRecords(compressed, TEST_FORMATS.codex, actual.position);
164 + assert.isNotNull(repeated);
165 + assert.isFalse(repeated.resumed);
166 + assert.deepStrictEqual(repeated.records, actual.records);
167 + });
168 +
169 + it("returns null for zstd decoder errors rather than caching an empty parse", async () => {
170 + const path = NodePath.join(dir, "corrupt.jsonl.zst");
171 + await NodeFSP.writeFile(path, "not a zstd frame");
172 + assert.isNull(await readTranscriptRecords(path, TEST_FORMATS.claude));
173 + });
174 +});
175 +
176 describe("readTranscriptRecords resume", () => {
177 it.each([
178 { mode: "buffered", streamingThresholdBytes: Infinity },
apps/server/src/usage/usageTranscriptReader.ts
+16 -6
@@ -3,6 +3,7 @@
3 import * as NodeFSP from "node:fs/promises";
4 import * as NodePath from "node:path";
5 import * as NodeStringDecoder from "node:string_decoder";
6 +import * as NodeZlib from "node:zlib";
7
8 import type {
9 SelectedFields,
@@ -85,7 +86,11 @@ export async function listTranscriptFiles(
86 for (const entry of entries) {
87 const child = NodePath.join(dir, entry.name);
88 if (entry.isDirectory()) await walk(child);
88 - else if (fileName !== undefined ? entry.name === fileName : entry.name.endsWith(".jsonl")) {
89 + else if (
90 + fileName !== undefined
91 + ? entry.name === fileName
92 + : entry.name.endsWith(".jsonl") || entry.name.endsWith(".jsonl.zst")
93 + ) {
94 candidates.push(child);
95 }
96 }
@@ -162,6 +167,7 @@ export async function readTranscriptRecords<State>(
167 options?: { readonly streamingThresholdBytes?: number },
168 ): Promise<TranscriptParseResult | null> {
169 const streamingThresholdBytes = options?.streamingThresholdBytes ?? STREAMING_THRESHOLD_BYTES;
170 + const compressed = filePath.endsWith(".jsonl.zst");
171 let handle: NodeFSP.FileHandle;
172 try {
173 handle = await NodeFSP.open(filePath, "r");
@@ -181,6 +187,7 @@ export async function readTranscriptRecords<State>(
187 ? Option.none()
188 : decodeState(resumeFrom.state);
189 if (
190 + !compressed &&
191 resumeFrom !== undefined &&
192 resumeFrom.resumeOffset > 0 &&
193 (codec === undefined || Option.isSome(savedState)) &&
@@ -247,11 +254,14 @@ export async function readTranscriptRecords<State>(
254 streaming = undefined;
255 decoder = undefined;
256 };
250 - const stream = handle.createReadStream({
257 + const input = handle.createReadStream({
258 start,
252 - autoClose: false,
259 + autoClose: compressed,
260 highWaterMark: 256 * 1024,
254 - }) as AsyncIterable<Buffer>;
261 + });
262 + // Compressed snapshots cache a full parse; decoded byte offsets cannot
263 + // resume the compressed input.
264 + const stream = compressed ? input.compose(NodeZlib.createZstdDecompress()) : input;
265 for await (const chunk of stream) {
266 let lineStart = 0;
267 while (lineStart < chunk.length) {
@@ -279,7 +289,7 @@ export async function readTranscriptRecords<State>(
289 decodeState === undefined ? state : Option.getOrThrow(decodeState(encodedState));
290 finish(tailState, tailRecords);
291
282 - const guardLength = Math.min(GUARD_LENGTH, resumeOffset);
292 + const guardLength = compressed ? 0 : Math.min(GUARD_LENGTH, resumeOffset);
293 let guardHash = 0;
294 if (guardLength > 0) {
295 const window = Buffer.alloc(guardLength);
@@ -292,7 +302,7 @@ export async function readTranscriptRecords<State>(
302 tailRecords,
303 malformedRecords: format.readMalformedRecords?.(tailState) ?? 0,
304 position: {
295 - resumeOffset,
305 + resumeOffset: compressed ? 0 : resumeOffset,
306 guardLength,
307 guardHash,
308 state: encodedState,