@devries-cameron20 / t3code Mirror · read-only / commits / 9b6dde9a23
Fork 0

The best way to control Claude Code, Codex and more from any computer or phone

Mirrored from https://github.com/Camo67/t3code.git

Synced Nov 21, 17:15 UTC

fix(server): session import finds compressed Codex rollouts (#34947)

Julius Marminge committed Nov 13, 2026 at 08:49 UTC 9b6dde9a2384b883dda71fd31cab21616caf436c
2 files changed +166 -9
apps/server/src/project/AgentSessionScanner.test.ts
+102
@@ -5,6 +5,7 @@ import * as NodeChildProcess from "node:child_process";
5 import * as NodeFS from "node:fs";
6 import * as NodeOS from "node:os";
7 import * as NodeSqlite from "node:sqlite";
8 +import * as NodeZlib from "node:zlib";
9 import { describe, expect, it } from "@effect/vitest";
10 import {
11 AgentSessionImportSource,
@@ -174,6 +175,21 @@ const writeTranscript = Effect.fn("AgentSessionScanner.test.writeTranscript")(fu
175 yield* fileSystem.utimes(input.filePath, seconds, seconds);
176 });
177
178 +const writeCompressedTranscript = Effect.fn("AgentSessionScanner.test.writeCompressedTranscript")(
179 + function* (input: {
180 + readonly filePath: string;
181 + readonly contents: string;
182 + readonly mtimeMs: number;
183 + }) {
184 + const fileSystem = yield* FileSystem.FileSystem;
185 + const path = yield* Path.Path;
186 + yield* fileSystem.makeDirectory(path.dirname(input.filePath), { recursive: true });
187 + yield* fileSystem.writeFile(input.filePath, NodeZlib.zstdCompressSync(input.contents));
188 + const seconds = input.mtimeMs / 1000;
189 + yield* fileSystem.utimes(input.filePath, seconds, seconds);
190 + },
191 +);
192 +
193 const claudeSessionLine = (cwd: string) =>
194 `${JSON.stringify({ type: "user", cwd, sessionId: "s1" })}\n${JSON.stringify({ type: "assistant" })}\n`;
195
@@ -1244,6 +1260,32 @@ it.layer(NodeServices.layer)("AgentSessionScanner", (it) => {
1260
1261 const result = yield* runScan({ claudeHomePath, codexHomePath });
1262
1263 + expect(result.candidates.map(({ path, threadCount }) => ({ path, threadCount }))).toEqual([
1264 + { path: workspace, threadCount: 1 },
1265 + ]);
1266 + }),
1267 + );
1268 + it.effect("discovers projects whose Codex rollouts are zstd-compressed", () =>
1269 + Effect.gen(function* () {
1270 + const path = yield* Path.Path;
1271 + const claudeHomePath = yield* makeTempDir("t3code-claude-home-");
1272 + const codexHomePath = yield* makeTempDir("t3code-codex-home-");
1273 + const workspace = yield* makeTempDir("t3code-workspace-");
1274 + yield* writeCompressedTranscript({
1275 + filePath: path.join(
1276 + codexHomePath,
1277 + "sessions",
1278 + "2026",
1279 + "05",
1280 + "03",
1281 + "rollout-2026-05-03T00-00-00-019a5f00-0000-7000-8000-00000000000b.jsonl.zst",
1282 + ),
1283 + contents: codexRolloutLine(workspace),
1284 + mtimeMs: Date.parse("2026-05-03T00:00:00.000Z"),
1285 + });
1286 +
1287 + const result = yield* runScan({ claudeHomePath, codexHomePath });
1288 +
1289 expect(result.candidates.map(({ path, threadCount }) => ({ path, threadCount }))).toEqual([
1290 { path: workspace, threadCount: 1 },
1291 ]);
@@ -1252,6 +1294,66 @@ it.layer(NodeServices.layer)("AgentSessionScanner", (it) => {
1294 });
1295
1296 describe("recentThreads", () => {
1297 + it.effect("imports Codex conversations stored only as zstd-compressed rollouts", () =>
1298 + Effect.gen(function* () {
1299 + const path = yield* Path.Path;
1300 + const nowMs = Date.parse("2026-08-24T12:00:00.000Z");
1301 + yield* TestClock.setTime(nowMs);
1302 + const claudeHomePath = yield* makeTempDir("t3code-claude-home-");
1303 + const codexHomePath = yield* makeTempDir("t3code-codex-home-");
1304 + const workspace = yield* makeTempDir("t3code-workspace-");
1305 + yield* writeCompressedTranscript({
1306 + filePath: path.join(
1307 + codexHomePath,
1308 + "sessions",
1309 + "2026",
1310 + "08",
1311 + "20",
1312 + "rollout-2026-08-20T10-00-00-019a5f00-0000-7000-8000-00000000000c.jsonl.zst",
1313 + ),
1314 + contents: [
1315 + encodeTranscriptRecord({
1316 + type: "session_meta",
1317 + payload: { id: "019a5f00-0000-7000-8000-00000000000c", cwd: workspace },
1318 + }),
1319 + encodeTranscriptRecord({
1320 + type: "event_msg",
1321 + timestamp: "2026-08-20T10:00:00.000Z",
1322 + payload: { type: "user_message", message: "Compressed prompt" },
1323 + }),
1324 + encodeTranscriptRecord({
1325 + type: "response_item",
1326 + timestamp: "2026-08-20T10:01:00.000Z",
1327 + payload: {
1328 + type: "message",
1329 + role: "assistant",
1330 + content: [{ type: "output_text", text: "Compressed answer" }],
1331 + },
1332 + }),
1333 + ].join("\n"),
1334 + mtimeMs: nowMs - 4 * 24 * 60 * 60 * 1000,
1335 + });
1336 +
1337 + const threads = yield* runRecentThreads({
1338 + claudeHomePath,
1339 + codexHomePath,
1340 + workspaceRoot: workspace,
1341 + });
1342 +
1343 + expect(
1344 + threads.map((thread) => ({
1345 + id: thread.providerSessionId,
1346 + messages: thread.messages.map((message) => message.text),
1347 + })),
1348 + ).toEqual([
1349 + {
1350 + id: "019a5f00-0000-7000-8000-00000000000c",
1351 + messages: ["Compressed prompt", "Compressed answer"],
1352 + },
1353 + ]);
1354 + }),
1355 + );
1356 +
1357 it.effect("imports recent Claude, Codex, and Pi sessions for the selected project only", () =>
1358 Effect.gen(function* () {
1359 const path = yield* Path.Path;
apps/server/src/project/AgentSessionScanner.ts
+64 -9
@@ -1,5 +1,6 @@
1 import * as NodeOS from "node:os";
2 import * as NodeSqlite from "node:sqlite";
3 +import * as NodeZlib from "node:zlib";
4
5 import {
6 AgentSessionImportSource,
@@ -17,11 +18,13 @@ import {
18 import * as Clock from "effect/Clock";
19 import * as Context from "effect/Context";
20 import * as DateTime from "effect/DateTime";
21 +import * as NodeStream from "@effect/platform-node/NodeStream";
22 import * as Effect from "effect/Effect";
23 import * as FileSystem from "effect/FileSystem";
24 import * as Layer from "effect/Layer";
25 import * as Option from "effect/Option";
26 import * as Path from "effect/Path";
27 +import * as Pull from "effect/Pull";
28 import * as Schema from "effect/Schema";
29 import * as Semaphore from "effect/Semaphore";
30 import * as Stream from "effect/Stream";
@@ -177,7 +180,11 @@ const decodeCodexHistoryBase = Schema.decodeUnknownOption(
180 );
181 // Original rollouts end in their thread ID, rewound ones in `<thread ID>_<rollout ID>`.
182 const CODEX_ROLLOUT_ID_PATTERN =
180 - /-([0-9a-f]{8}-[0-9a-f]{4}-[0-9a-f]{4}-[0-9a-f]{4}-[0-9a-f]{12})(?:_([0-9a-f]{8}-[0-9a-f]{4}-[0-9a-f]{4}-[0-9a-f]{4}-[0-9a-f]{12}))?\.jsonl$/i;
183 + /-([0-9a-f]{8}-[0-9a-f]{4}-[0-9a-f]{4}-[0-9a-f]{4}-[0-9a-f]{12})(?:_([0-9a-f]{8}-[0-9a-f]{4}-[0-9a-f]{4}-[0-9a-f]{4}-[0-9a-f]{12}))?\.jsonl(?:\.zst)?$/i;
184 +// Codex zstd-compresses older rollouts; their byte limits and offsets count decompressed bytes.
185 +const COMPRESSED_TRANSCRIPT_SUFFIX = ".jsonl.zst";
186 +const isCompressedTranscript = (filePath: string) =>
187 + filePath.endsWith(COMPRESSED_TRANSCRIPT_SUFFIX);
188 // Codex records a chat's project only here; moving a chat leaves its rollout cwd unchanged.
189 const CODEX_STATE_DATABASE = "state_5.sqlite";
190 const decodeCodexSessionIndexEntry = Schema.decodeUnknownOption(
@@ -861,6 +868,33 @@ export const make = Effect.gen(function* () {
868 const statOption = (target: string) =>
869 fileSystem.stat(target).pipe(Effect.asSome, Effect.orElseSucceed(Option.none));
870
871 + const transcriptChunkReader = Effect.fn("AgentSessionScanner.transcriptChunkReader")(function* (
872 + file: FileSystem.File,
873 + filePath: string,
874 + ) {
875 + if (!isCompressedTranscript(filePath)) return (size: number) => file.readAlloc(size);
876 + const pull = yield* Stream.toPull(
877 + fileSystem
878 + .stream(filePath)
879 + .pipe(NodeStream.pipeThroughSimple(() => NodeZlib.createZstdDecompress())),
880 + );
881 + let pending: Array<Uint8Array> = [];
882 + let done = false;
883 + // Untraced: called once per chunk inside the caller's traced read.
884 + return Effect.fnUntraced(function* (size: number) {
885 + while (pending.length === 0) {
886 + if (done) return Option.none<Uint8Array>();
887 + const next = yield* pull.pipe(Pull.catchDone(() => Effect.undefined));
888 + if (next === undefined) done = true;
889 + else pending = [...next];
890 + }
891 + const chunk = pending.shift();
892 + if (chunk === undefined) return Option.none<Uint8Array>();
893 + if (chunk.byteLength > size) pending.unshift(chunk.subarray(size));
894 + return Option.some(chunk.subarray(0, size));
895 + });
896 + });
897 +
898 const directoryIdentity = Effect.fn("AgentSessionScanner.directoryIdentity")(function* (
899 target: string,
900 knownStats?: FileSystem.File.Info,
@@ -1037,11 +1071,15 @@ export const make = Effect.gen(function* () {
1071 fileSystem.open(transcript.filePath, { flag: "r" }).pipe(
1072 Effect.flatMap((file) =>
1073 Effect.gen(function* () {
1074 + const readChunk = yield* transcriptChunkReader(file, transcript.filePath);
1075 + const size = isCompressedTranscript(transcript.filePath)
1076 + ? Number.POSITIVE_INFINITY
1077 + : transcript.identity.size;
1078 const decoder = new TextDecoder();
1079 let remaining = "";
1080 let bytesRead = 0;
1081 let recordsRead = 0;
1044 - const maxBytes = Math.min(MAX_TRANSCRIPT_SCAN_BYTES, transcript.identity.size);
1082 + const maxBytes = Math.min(MAX_TRANSCRIPT_SCAN_BYTES, size);
1083 const reserveRecord = () => {
1084 if (
1085 recordsRead === MAX_METADATA_RECORDS_PER_TRANSCRIPT ||
@@ -1073,7 +1111,7 @@ export const make = Effect.gen(function* () {
1111 );
1112 budget.operationsRemaining -= 1;
1113 budget.bytesRemaining -= readSize;
1076 - const next = yield* file.readAlloc(readSize);
1114 + const next = yield* readChunk(readSize);
1115 if (Option.isNone(next)) {
1116 return readLastRecord();
1117 }
@@ -1090,7 +1128,7 @@ export const make = Effect.gen(function* () {
1128 }
1129 }
1130
1093 - if (bytesRead < transcript.identity.size) {
1131 + if (bytesRead < size) {
1132 budget.truncated = true;
1133 return null;
1134 }
@@ -1107,8 +1145,12 @@ export const make = Effect.gen(function* () {
1145 source: AgentSessionSource,
1146 prefix?: { readonly endByteOffset: number; readonly historyBytes: number },
1147 ) {
1110 - const readBytes = prefix?.endByteOffset ?? expected.size;
1111 - if (readBytes > expected.size || readBytes > MAX_IMPORTED_TRANSCRIPT_BYTES) return null;
1148 + const compressed = isCompressedTranscript(filePath);
1149 + const readBytes =
1150 + prefix?.endByteOffset ?? (compressed ? MAX_IMPORTED_TRANSCRIPT_BYTES : expected.size);
1151 + if ((!compressed && readBytes > expected.size) || readBytes > MAX_IMPORTED_TRANSCRIPT_BYTES) {
1152 + return null;
1153 + }
1154
1155 return yield* Effect.scoped(
1156 fileSystem.open(filePath, { flag: "r" }).pipe(
@@ -1117,6 +1159,7 @@ export const make = Effect.gen(function* () {
1159 if (!sameTranscriptIdentity(expected, transcriptIdentity(filePath, yield* file.stat))) {
1160 return null;
1161 }
1162 + const readChunk = yield* transcriptChunkReader(file, filePath);
1163 const records: Array<DecodedTranscriptRecord> = [];
1164 const claudeHistory = createClaudeHistoryWindow();
1165 let cwd = Option.none<string>();
@@ -1177,10 +1220,12 @@ export const make = Effect.gen(function* () {
1220 };
1221
1222 while (bytesRead < readBytes) {
1180 - const next = yield* file.readAlloc(
1223 + const next = yield* readChunk(
1224 Math.min(TRANSCRIPT_PREFIX_BYTES, readBytes - bytesRead),
1225 );
1226 if (Option.isNone(next)) {
1227 + // A compressed rollout's decompressed length is only known at its end.
1228 + if (compressed && prefix === undefined) break;
1229 return null;
1230 }
1231
@@ -1398,7 +1443,12 @@ export const make = Effect.gen(function* () {
1443 }
1444 const directory = path.join(sessionsDir, year, month, day);
1445 for (const entry of (yield* readDirectory(directory)).toSorted().toReversed()) {
1401 - if (!entry.startsWith("rollout-") || !entry.endsWith(".jsonl")) continue;
1446 + if (
1447 + !entry.startsWith("rollout-") ||
1448 + !(entry.endsWith(".jsonl") || isCompressedTranscript(entry))
1449 + ) {
1450 + continue;
1451 + }
1452 if (operationsRemaining <= 0) {
1453 truncated = true;
1454 break;
@@ -2024,7 +2074,12 @@ export const make = Effect.gen(function* () {
2074 {
2075 source: candidate.source,
2076 providerInstanceId: candidate.providerInstanceId,
2027 - fallbackSessionId: path.basename(transcript.filePath, ".jsonl"),
2077 + fallbackSessionId: path.basename(
2078 + transcript.filePath,
2079 + isCompressedTranscript(transcript.filePath)
2080 + ? COMPRESSED_TRANSCRIPT_SUFFIX
2081 + : ".jsonl",
2082 + ),
2083 lastActiveAtMs: transcript.mtimeMs,
2084 },
2085 inheritedRecords === null