The best way to control Claude Code, Codex and more from any computer or phone
Mirrored from https://github.com/Camo67/t3code.git
Synced Nov 21, 19:45 UTC
perf(client): threads load while the socket connects, with fewer round trips (#35071)
The prepare-time shell load follows the idea and structure of #27413. Co-authored-by: ahalekelly <7078138+ahalekelly@users.noreply.github.com> Co-Authored-By: Claude Fable 5.2 <noreply@anthropic.com>
Theo Browne committed
Nov 12, 2026 at 06:31 UTC
2482b03a01cf998a26cddbff65c5e5b0b0f39b38
29 files changed
+1363
-256
apps/server/integration/TransferBudgetReport.integration.ts
+16
-3
@@ -74,7 +74,11 @@ export interface TransferBudgetRun {
74
readonly fleetShellLegacy: TimelineHttpMeasurement;
75
/** The shell subscription a cold connect opens from that snapshot's cursor. */
76
readonly fleetShellSubscribe: SocketOpenMeasurement;
77
- /** A shell resumed from a cursor too old to replay, by a client that reloads over HTTP. */
77
+ /** A client that holds that snapshot's rows asks the HTTP route whether they still resume. */
78
+ readonly fleetShellResume: TimelineHttpMeasurement;
79
+ /** The same question from a cursor too old to replay: the route answers with the snapshot. */
80
+ readonly staleShellReload: TimelineHttpMeasurement;
81
+ /** A shell resumed over the socket from that old cursor, by a client that reloads over HTTP. */
82
readonly staleShellResume: SocketOpenMeasurement;
83
/**
84
* The compact bounded snapshot of the long thread whose finished runs left todo lists open,
@@ -141,6 +145,7 @@ const SOCKET_BUDGETS = {
145
longTurnColdOpen: { decodedBytes: LONG_TURN_FINISHED_BUDGET.decodedBytes, messages: 2 },
146
// These three send no rows. A snapshot in their place is 393 KB for the thread, 42 KB for the shell.
147
idleThreadResume: { decodedBytes: 200, messages: 1 },
148
+ // A subscribe that does not name the snapshot's project identities gets the project again: 851 B.
149
fleetShellSubscribe: { decodedBytes: 200, messages: 1 },
150
staleShellResume: { decodedBytes: 400, messages: 2 },
151
};
@@ -151,6 +156,12 @@ const FLEET_SHELL_BUDGET = {
156
rows: 27,
157
};
158
159
+// The answer is `{"resumable":true}`. A snapshot in its place is 42 KB.
160
+const FLEET_SHELL_RESUME_BUDGET = {
161
+ decodedBytes: 100,
162
+ rows: 0,
163
+};
164
+
165
// About 14% above the observed 75 KB. The window holds the working run only: 2 nodes, 2 subagents,
166
// 1 plan, 1 provider turn and the last finished turn's checkpoint. When finished runs ride along
167
// it holds 82 nodes, 82 subagents, 41 plans, 41 provider turns, 40 checkpoints and 740 KB.
@@ -367,6 +378,8 @@ function scenarioRows(run: TransferBudgetRun): ScenarioRow[] {
378
run.fleetShellSubscribe,
379
SOCKET_BUDGETS.fleetShellSubscribe,
380
),
381
+ ...httpRows("fleet shell resume check", run.fleetShellResume, FLEET_SHELL_RESUME_BUDGET),
382
+ ...httpRows("stale shell reload", run.staleShellReload, FLEET_SHELL_BUDGET),
383
...socketRows("stale shell resume", run.staleShellResume, SOCKET_BUDGETS.staleShellResume),
384
...httpRows("stale todo list cold open", run.staleTodoColdOpen, STALE_TODO_BUDGET),
385
...STALE_TODO_RECORD_KINDS.map((kind): ScenarioRow => ({
@@ -439,8 +452,8 @@ export function formatTransferBudgetReport(runs: ReadonlyArray<TransferBudgetRun
452
"The measured turn is observed on three sockets at once: one with only the thread subscription (the capped rows), one with only the shell subscription, and a second client holding both. Server egress is the sum of the three. After the turn the second client disconnects and new sockets resubscribe from the cursor it held before the turn, which is the cursor a backgrounded phone would hold. SQL statements are the SQL client's per-statement spans, counted across V2 persistence and the HTTP/WebSocket handlers.",
453
`Scenario: ${TRANSFER_HISTORY_TURN_COUNT} historical turns with ${TRANSFER_HISTORY_TOOLS_PER_TURN} command tools and one retained ${formatBytes(TRANSFER_HISTORY_MCP_RESULT_BYTES)} MCP result each, followed by one measured turn with ${TRANSFER_MEASURED_TOOLS} command tools and a retained ${formatBytes(TRANSFER_MEASURED_MCP_RESULT_BYTES)} MCP result. Synthetic V2 domain events exercise persistence, wire projection, and production HTTP/subscription handlers; this does not measure provider adapter ingestion. Payloads contain no user data.`,
454
`The long turn rows read a second thread whose last user turn holds ${LONG_TURN_WORK_ROWS} rows of reasoning, commands, file changes, MCP calls, progress messages, checkpoints, and subagents. While that turn runs, web and mobile load the compact bounded snapshot and then one older page. Once it finishes, its work is deferred: the cold open is read over HTTP as web does and over the socket alone as mobile does, then the first page of the deferred work is loaded as an expanded work group does.`,
442
- "The resume rows open a new socket from a cursor the client already holds: the first thread after one more ordinary turn, the long-turn thread with nothing new, and the shell from a cursor too old to replay.",
443
- `The fleet shell rows read the cold-connect shell after ${SHELL_SETTLED_THREADS} settled threads and ${SHELL_IDLE_SUBAGENTS} idle subagent children are added, then subscribe from that snapshot's cursor. Budgeted reads send the opt-ins current clients send; "older client" rows show the same read without the opt-ins added since row budgets.`,
455
+ "The resume rows open a new socket from a cursor the client already holds: the first thread after one more ordinary turn, the long-turn thread with nothing new, and the shell from a cursor too old to replay. A client that holds shell rows first names their cursor to the HTTP shell route: the resume check row is the answer that keeps the rows, and the stale shell reload row is the snapshot that replaces them.",
456
+ `The fleet shell rows read the cold-connect shell after ${SHELL_SETTLED_THREADS} settled threads and ${SHELL_IDLE_SUBAGENTS} idle subagent children are added, then subscribe from that snapshot's cursor and name its project identities, so the project's repository identity is not sent again. Budgeted reads send the opt-ins current clients send; "older client" rows show the same read without the opt-ins added since row budgets.`,
457
`The stale todo list rows open a third thread the same way: ${STALE_TODO_SETTLED_RUNS} finished runs, then one run still at work with ${STALE_TODO_LIVE_ROWS} rows. Every run left its todo list open and has ${STALE_TODO_SUBAGENTS_PER_RUN} finished subagents, a provider turn and, once finished, a checkpoint. The record counts must not grow with the number of finished runs.`,
458
"",
459
"| Provider | Total thread wire | Budget | Result |",
apps/server/integration/TransferBudgetV2Fixture.integration.ts
+40
@@ -14,8 +14,10 @@ import {
14
RunId,
15
ThreadId,
16
TurnItemId,
17
+ type ApplicationProjectEvent,
18
type ProviderDriverKind,
19
type OrchestrationV2DomainEvent,
20
+ type RepositoryIdentity,
21
type OrchestrationV2TurnItem,
22
} from "@t3tools/contracts";
23
import * as DateTime from "effect/DateTime";
@@ -837,6 +839,44 @@ export function staleTodoListThreadBatches(
839
];
840
}
841
842
+const FLEET_PROJECT_ID = ProjectId.make("transfer-project");
843
+export const FLEET_PROJECT_IDENTITY: RepositoryIdentity = {
844
+ canonicalKey: "github.com/t3/transfer-budget",
845
+ locator: {
846
+ source: "git-remote",
847
+ remoteName: "origin",
848
+ remoteUrl: "git@github.com:t3/transfer-budget.git",
849
+ },
850
+ owner: "t3",
851
+ name: "transfer-budget",
852
+};
853
+
854
+/**
855
+ * The project every fixture thread belongs to. The test resolves it to FLEET_PROJECT_IDENTITY,
856
+ * which a shell resume sends again unless the client names the identities it already holds.
857
+ */
858
+export const fleetProjectCreated: ApplicationProjectEvent = {
859
+ sequence: 0,
860
+ eventId: EventId.make("project-created:transfer-project"),
861
+ aggregateKind: "project",
862
+ aggregateId: FLEET_PROJECT_ID,
863
+ occurredAt: "2026-06-01T00:00:00.000Z",
864
+ commandId: null,
865
+ causationEventId: null,
866
+ correlationId: null,
867
+ metadata: {},
868
+ type: "project.created",
869
+ payload: {
870
+ projectId: FLEET_PROJECT_ID,
871
+ title: "Transfer budget",
872
+ workspaceRoot: "/tmp/transfer-budget",
873
+ defaultModelSelection: null,
874
+ scripts: [],
875
+ createdAt: "2026-06-01T00:00:00.000Z",
876
+ updatedAt: "2026-06-01T00:00:00.000Z",
877
+ },
878
+};
879
+
880
export const SHELL_SETTLED_THREADS = 100;
881
export const SHELL_IDLE_SUBAGENTS = 40;
882
apps/server/integration/transferBudgetV2.integration.test.ts
+45
-16
@@ -19,7 +19,8 @@ import {
19
OrchestrationV2ThreadBoundedSnapshot,
20
OrchestrationV2ThreadDetailSnapshot,
21
OrchestrationV2ThreadHistoryPage,
22
- OrchestrationV2ShellSnapshot,
22
+ OrchestrationV2ShellHttpSnapshot,
23
+ OrchestrationV2ShellResumable,
24
OrchestrationV2GetThreadProjectionError,
25
OrchestrationV2GetShellSnapshotError,
26
ProviderDriverKind,
@@ -75,6 +76,8 @@ import {
76
} from "./TransferBudgetReport.integration.ts";
77
import { TRANSFER_HISTORY_TURN_COUNT } from "./fixtures/transferBudget.ts";
78
import {
79
+ FLEET_PROJECT_IDENTITY,
80
+ fleetProjectCreated,
81
LONG_TURN_THREAD_ID,
82
longTurnThreadBatches,
83
SHELL_IDLE_SUBAGENTS,
@@ -92,7 +95,8 @@ import {
95
96
// The opt-ins current web and mobile clients send, as client-runtime's boundedThreadSnapshotHttp.ts,
97
// threadHistoryHttp.ts, shellSnapshotHttp.ts, threads.ts and shell.ts send them. Each is `Required`,
95
-// so an opt-in added to a contract fails typecheck here until this test sends it.
98
+// so an opt-in added to a contract fails typecheck here until this test sends it. An omitted field
99
+// differs per request (a cursor, or what the client holds), so each scenario sends its own.
100
const CLIENT_BOUNDED_OPT_INS = {
101
compactTurnItems: "1",
102
dedupeBodies: "1",
@@ -105,7 +109,7 @@ const CLIENT_HISTORY_OPT_INS = { compactIds: "1", deferWork: "1" } satisfies Req
109
const CLIENT_SHELL_OPT_INS = {
110
omitIdleSubagents: "1",
111
omitOlderSettled: "1",
108
-} satisfies Required<typeof EnvironmentOrchestrationShellSnapshotQuery.Type>;
112
+} satisfies Required<Omit<typeof EnvironmentOrchestrationShellSnapshotQuery.Type, "resumeFrom">>;
113
const CLIENT_THREAD_SUBSCRIPTION = {
114
requestCompletionMarker: true,
115
acceptBoundedSnapshot: true,
@@ -114,17 +118,22 @@ const CLIENT_THREAD_SUBSCRIPTION = {
118
acceptCompactIds: true,
119
acceptDedupedBodies: true,
120
} satisfies Required<Omit<OrchestrationV2SubscribeThreadInput, "threadId" | "afterSequence">>;
121
+type ShellResumeField = "afterSequence" | "httpSnapshotFallback" | "knownProjectIdentities";
122
const CLIENT_SHELL_SUBSCRIPTION = {
123
requestCompletionMarker: true,
124
omitIdleSubagents: true,
125
omitOlderSettled: true,
121
-} satisfies Required<
122
- Omit<OrchestrationV2SubscribeShellInput, "afterSequence" | "httpSnapshotFallback">
123
->;
126
+} satisfies Required<Omit<OrchestrationV2SubscribeShellInput, ShellResumeField>>;
127
const query = (optIns: Readonly<Record<string, string>>) => new URLSearchParams(optIns).toString();
128
const CLIENT_BOUNDED_QUERY = query(CLIENT_BOUNDED_OPT_INS);
129
const CLIENT_HISTORY_QUERY = query(CLIENT_HISTORY_OPT_INS);
130
const CLIENT_SHELL_QUERY = query(CLIENT_SHELL_OPT_INS);
131
+// A client that holds shell rows names their cursor in the same request.
132
+const clientShellResumeQuery = (resumeFrom: number) =>
133
+ query({
134
+ ...CLIENT_SHELL_OPT_INS,
135
+ resumeFrom: String(resumeFrom),
136
+ } satisfies Required<typeof EnvironmentOrchestrationShellSnapshotQuery.Type>);
137
138
const decodeThreadSnapshot = Schema.decodeUnknownEffect(
139
Schema.fromJsonString(Schema.toCodecJson(OrchestrationV2ThreadDetailSnapshot)),
@@ -136,7 +145,10 @@ const decodeHistoryPage = Schema.decodeUnknownEffect(
145
Schema.fromJsonString(Schema.toCodecJson(OrchestrationV2ThreadHistoryPage)),
146
);
147
const decodeShellSnapshot = Schema.decodeUnknownEffect(
139
- Schema.fromJsonString(Schema.toCodecJson(OrchestrationV2ShellSnapshot)),
148
+ Schema.fromJsonString(Schema.toCodecJson(OrchestrationV2ShellHttpSnapshot)),
149
+);
150
+const decodeShellResumable = Schema.decodeUnknownEffect(
151
+ Schema.fromJsonString(Schema.toCodecJson(OrchestrationV2ShellResumable)),
152
);
153
154
const layerPersistence = EventSink.layer.pipe(
@@ -176,7 +188,7 @@ const layerEnrichment = Layer.unwrap(
188
return Layer.mock(ProjectEnrichmentService.ProjectEnrichmentService)({
189
getAvailable: () =>
190
Effect.succeed({
179
- repositoryIdentity: null,
191
+ repositoryIdentity: FLEET_PROJECT_IDENTITY,
192
faviconPath: null,
193
repositoryIdentityResolved: true,
194
}),
@@ -263,10 +275,7 @@ const collectUntil = Effect.fn("transfer.collectUntil")(function* <T extends Str
275
});
276
type Subscription =
277
| ({ readonly thread: ThreadId } & Pick<OrchestrationV2SubscribeThreadInput, "afterSequence">)
266
- | ({ readonly shell: true } & Pick<
267
- OrchestrationV2SubscribeShellInput,
268
- "afterSequence" | "httpSnapshotFallback"
269
- >);
278
+ | ({ readonly shell: true } & Pick<OrchestrationV2SubscribeShellInput, ShellResumeField>);
279
function subscriptionStream(
280
client: MeasuredWsClient,
281
subscription: Subscription,
@@ -457,8 +466,10 @@ it.live("reports V2 thread HTTP and WebSocket transfer budgets", () =>
466
const readShell = (query: string) =>
467
Effect.gen(function* () {
468
const http = yield* get(`/api/orchestration/shell?${query}`);
460
- const { threads, snapshotSequence } = yield* decodeShellSnapshot(decoded(http));
461
- return { http, rows: threads.length, snapshotSequence };
469
+ const { threads, snapshotSequence, projectIdentities } = yield* decodeShellSnapshot(
470
+ decoded(http),
471
+ );
472
+ return { http, rows: threads.length, snapshotSequence, projectIdentities };
473
});
474
// A new socket, read until the subscription caught up or the server ended it.
475
const open = (subscription: Subscription) =>
@@ -529,6 +540,8 @@ it.live("reports V2 thread HTTP and WebSocket transfer budgets", () =>
540
541
// The transfer thread plus the long-turn thread.
542
const earlierThreads = decodedShell.threads.length + 1;
543
+ // The fleet's project is in a repository. Earlier phases have no project, as on `main`.
544
+ yield* (yield* ProjectStore.ProjectStoreV2).apply(fleetProjectCreated);
545
yield* sink.write({ events: shellFleetEvents(provider) });
546
const fleetShellLegacy = yield* readShell("");
547
const fleetShell = yield* readShell(CLIENT_SHELL_QUERY);
@@ -536,13 +549,27 @@ it.live("reports V2 thread HTTP and WebSocket transfer budgets", () =>
549
fleetShellLegacy.rows,
550
earlierThreads + SHELL_SETTLED_THREADS + SHELL_IDLE_SUBAGENTS,
551
);
539
- // A cold connect subscribes from the snapshot it just loaded.
552
+ const { projectIdentities } = fleetShell;
553
+ if (projectIdentities === undefined)
554
+ return yield* Effect.die("Expected the shell snapshot to name its project identities");
555
+ // A cold connect subscribes from the snapshot it just loaded and names its identities.
556
+ // Without them the resume sends the project again, in a marked snapshot.
557
const fleetShellSubscribe = yield* open({
558
shell: true,
559
afterSequence: fleetShell.snapshotSequence,
560
+ knownProjectIdentities: projectIdentities,
561
});
562
assert.notInclude(fleetShellSubscribe.kinds, "snapshot");
545
- // A cursor too old to replay is told to reload over HTTP, not sent a socket snapshot.
563
+ // A client that holds those rows asks the route whether they still resume, and keeps them.
564
+ const fleetShellResumeHttp = yield* get(
565
+ `/api/orchestration/shell?${clientShellResumeQuery(fleetShell.snapshotSequence)}`,
566
+ );
567
+ yield* decodeShellResumable(decoded(fleetShellResumeHttp));
568
+ const fleetShellResume = { http: fleetShellResumeHttp, rows: 0 };
569
+ // A cursor too old to replay gets the snapshot in the same answer.
570
+ const staleShellReload = yield* readShell(clientShellResumeQuery(sequence));
571
+ assert.equal(staleShellReload.rows, fleetShell.rows);
572
+ // When that request fails, the socket tells the client to reload, with no snapshot.
573
const staleShellResume = yield* open({
574
shell: true,
575
afterSequence: sequence,
@@ -601,6 +628,8 @@ it.live("reports V2 thread HTTP and WebSocket transfer budgets", () =>
628
fleetShellLegacy,
629
fleetShell,
630
fleetShellSubscribe,
631
+ fleetShellResume,
632
+ staleShellReload,
633
staleShellResume,
634
staleTodoColdOpen,
635
} satisfies TransferBudgetRun;
apps/server/src/environment/ServerEnvironment.ts
+2
@@ -291,6 +291,8 @@ export const make = Effect.gen(function* () {
291
linkFence: true,
292
mcpApprovalCodes: true,
293
mcpHostedApproval: true,
294
+ shellThreadFamilies: true,
295
+ shellOlderSettled: true,
296
},
297
};
298
apps/server/src/httpResponseErrorGuard.test.ts
+77
@@ -125,6 +125,83 @@ describe("guardHttpResponseWriteErrors", () => {
125
});
126
});
127
128
+describe("keepIdleConnections", () => {
129
+ it("closes a connection whose response finishes after the server stopped listening", async () => {
130
+ const server = HttpResponseErrorGuard.keepIdleConnections(NodeHttp.createServer());
131
+ const body = "x".repeat(1_000_000);
132
+ const closed = Promise.withResolvers<void>();
133
+ server.on("request", (_request, response) => {
134
+ // The server shuts down while this request is in flight.
135
+ server.close(() => closed.resolve());
136
+ response.end(body);
137
+ });
138
+ const port = await listen(server);
139
+ const agent = new NodeHttp.Agent({ keepAlive: true });
140
+
141
+ const received = await new Promise<number>((resolve, reject) => {
142
+ const request = NodeHttp.get({ host: "127.0.0.1", port, agent }, (response) => {
143
+ let bytes = 0;
144
+ response.on("data", (chunk: Buffer) => {
145
+ bytes += chunk.length;
146
+ });
147
+ response.on("end", () => resolve(bytes));
148
+ response.on("error", reject);
149
+ });
150
+ requests.push(request);
151
+ request.on("error", reject);
152
+ });
153
+
154
+ expect(received).toBe(body.length);
155
+ // Left alone, the kept-alive connection holds the server open for the keep-alive time.
156
+ await closed.promise;
157
+ agent.destroy();
158
+ });
159
+
160
+ it("keeps a connection until every response on it has finished", async () => {
161
+ const server = HttpResponseErrorGuard.keepIdleConnections(NodeHttp.createServer());
162
+ const closed = Promise.withResolvers<void>();
163
+ // Larger than a socket buffer, so it still has output to send after `end`.
164
+ const thirdBody = "x".repeat(8_000_000);
165
+ const held: NodeHttp.ServerResponse[] = [];
166
+ server.on("request", (request, response) => {
167
+ if (request.url !== "/third") {
168
+ held.push(response);
169
+ return;
170
+ }
171
+ const [first, second] = held;
172
+ // All three pipelined requests are in flight when the server shuts down. The third
173
+ // response is ended before the first two, and the second ends after the first.
174
+ response.end(thirdBody);
175
+ server.close(() => closed.resolve());
176
+ first?.once("finish", () => setImmediate(() => second?.end("second-body")));
177
+ first?.end("first-body");
178
+ });
179
+ const port = await listen(server);
180
+
181
+ const received = await new Promise<string>((resolve, reject) => {
182
+ let data = "";
183
+ const client = NodeNet.connect(port, "127.0.0.1", () => {
184
+ client.write(
185
+ ["/first", "/second", "/third"]
186
+ .map((path) => `GET ${path} HTTP/1.1\r\nHost: 127.0.0.1\r\n\r\n`)
187
+ .join(""),
188
+ );
189
+ });
190
+ sockets.push(client);
191
+ client.on("data", (chunk: Buffer) => {
192
+ data += chunk.toString();
193
+ });
194
+ client.on("end", () => resolve(data));
195
+ client.on("error", reject);
196
+ });
197
+
198
+ expect(received.slice(0, 1_000)).toContain("first-body");
199
+ expect(received.slice(0, 1_000)).toContain("second-body");
200
+ expect(received.endsWith(thirdBody)).toBe(true);
201
+ await closed.promise;
202
+ });
203
+});
204
+
205
describe("rejectRequestsUntilServing", () => {
206
it("rejects early HTTP and upgrades, then hands each off to its serving handler", async () => {
207
const writeErrors: unknown[] = [];
apps/server/src/httpResponseErrorGuard.ts
+26
@@ -20,6 +20,32 @@ export function guardHttpResponseWriteErrors<T extends NodeHttp.Server>(
20
return server;
21
}
22
23
+// cloudflared and Tailscale serve keep an idle connection to this server for 90
24
+// seconds. Node's 5 second default closes it first, so they can send a request
25
+// into a closed connection, and a slow client reconnects in the middle of a connect.
26
+const HTTP_KEEP_ALIVE_TIMEOUT_MS = 95_000;
27
+
28
+/** Install before `rejectRequestsUntilServing`, whose handoff starts at the next request listener. */
29
+export function keepIdleConnections<T extends NodeHttp.Server>(server: T): T {
30
+ server.keepAliveTimeout = HTTP_KEEP_ALIVE_TIMEOUT_MS;
31
+ // `close` drops only the connections idle at that moment. One that finishes
32
+ // its responses later would hold shutdown for the whole keep-alive time.
33
+ // `closeIdleConnections` cannot do this: it destroys a socket whose next
34
+ // pipelined response is ended but still has output to send.
35
+ const responsesInFlight = new WeakMap<object, number>();
36
+ server.on("request", (request, response) => {
37
+ const socket = request.socket;
38
+ responsesInFlight.set(socket, (responsesInFlight.get(socket) ?? 0) + 1);
39
+ response.once("close", () => {
40
+ const left = (responsesInFlight.get(socket) ?? 1) - 1;
41
+ responsesInFlight.set(socket, left);
42
+ // Pipelined requests share the socket, so the last response ends it.
43
+ if (left === 0 && !server.listening) socket.end();
44
+ });
45
+ });
46
+ return server;
47
+}
48
+
49
/**
50
* NodeHttpServer listens before serve() attaches its handlers, so requests in
51
* between would hang. Install this after the write-error guards so only the
apps/server/src/orchestration-v2/ShellStream.ts
+104
@@ -19,6 +19,8 @@ import {
19
OrchestrationV2ThreadShell as ThreadShellSchema,
20
} from "@t3tools/contracts";
21
import * as NodeSqliteClient from "@t3tools/shared/nodeSqliteClient";
22
+// @effect-diagnostics-next-line nodeBuiltinImport:off - a synchronous digest of a few project rows.
23
+import * as NodeCrypto from "node:crypto";
24
import type * as Arr from "effect/Array";
25
import type * as Cause from "effect/Cause";
26
import * as Clock from "effect/Clock";
@@ -31,11 +33,113 @@ import * as Schema from "effect/Schema";
33
import type * as SqlClient from "effect/sql/SqlClient";
34
import * as Stream from "effect/Stream";
35
36
+import type * as OrchestrationEventStore from "../persistence/OrchestrationEventStore.ts";
37
import { bufferLatestLiveStream } from "./LiveStreamBudget.ts";
38
import type { ActiveShellIndexEntry } from "./ProjectionStore.ts";
39
40
type ShellDelta = Extract<OrchestrationV2ShellStreamItem, { readonly sequence: number }>;
41
42
+// Replaying each intervening event costs a shell refetch; past this gap a
43
+// single O(active-threads) snapshot is cheaper and bounded.
44
+const SHELL_RESUME_MAX_GAP = 1_000;
45
+// Row counts do not bound large tool payloads; sum stored JSON bytes in SQL and snapshot above budget.
46
+const SHELL_REPLAY_PAYLOAD_BUDGET_BYTES = 8 * 1024 * 1024;
47
+
48
+/**
49
+ * Whether the events after a shell cursor can be replayed. The socket resume and the HTTP
50
+ * snapshot's `resumeFrom` ask the same question, so a client the route tells to resume is not
51
+ * refused by the socket.
52
+ */
53
+export const canResumeShellFrom = Effect.fnUntraced(function* (
54
+ events: Pick<
55
+ OrchestrationEventStore.OrchestrationEventStore["Service"],
56
+ "prunedThroughSequence" | "getReplayStats"
57
+ >,
58
+ afterSequence: number,
59
+ headSequence: number,
60
+) {
61
+ // A cursor at the head missed nothing, even where pruning has passed the head.
62
+ if (afterSequence === headSequence) return true;
63
+ const replayGap = headSequence - afterSequence;
64
+ if (replayGap < 0 || replayGap > SHELL_RESUME_MAX_GAP) return false;
65
+ if (afterSequence < (yield* events.prunedThroughSequence)) return false;
66
+ const stats = yield* events.getReplayStats({ afterSequence, throughSequence: headSequence });
67
+ if (stats.rawPayloadBytes > SHELL_REPLAY_PAYLOAD_BUDGET_BYTES) {
68
+ yield* Effect.logDebug("orchestration replay replaced by snapshot", {
69
+ afterSequence,
70
+ headSequence,
71
+ replayGap,
72
+ eventCount: stats.eventCount,
73
+ payloadBytes: stats.rawPayloadBytes,
74
+ payloadBudgetBytes: SHELL_REPLAY_PAYLOAD_BUDGET_BYTES,
75
+ });
76
+ return false;
77
+ }
78
+ return true;
79
+});
80
+
81
+/**
82
+ * Answers a client that asks for the shell with the cursor of the rows it holds: `resumable`
83
+ * when the socket can replay from that cursor, so the client keeps its rows, else the snapshot.
84
+ */
85
+export const loadShellUnlessResumable = Effect.fnUntraced(function* <A, E, R>(input: {
86
+ readonly events: Pick<
87
+ OrchestrationEventStore.OrchestrationEventStore["Service"],
88
+ "prunedThroughSequence" | "getReplayStats" | "latestApplicationSequence"
89
+ >;
90
+ readonly resumeFrom: number | undefined;
91
+ readonly loadSnapshot: Effect.Effect<A, E, R>;
92
+}) {
93
+ if (
94
+ input.resumeFrom !== undefined &&
95
+ (yield* canResumeShellFrom(
96
+ input.events,
97
+ input.resumeFrom,
98
+ yield* input.events.latestApplicationSequence,
99
+ ))
100
+ ) {
101
+ return { resumable: true as const };
102
+ }
103
+ return yield* input.loadSnapshot;
104
+});
105
+
106
+/**
107
+ * Names the repository identities a list of project shells carries. A client sends back the one
108
+ * its HTTP snapshot had, and a resume that computes the same one skips its marked snapshot.
109
+ */
110
+export function projectIdentitiesDigest(
111
+ projects: ReadonlyArray<Pick<OrchestrationProjectShell, "id" | "repositoryIdentity">>,
112
+): string {
113
+ const hash = NodeCrypto.createHash("sha256");
114
+ for (const project of projects) {
115
+ hash.update(JSON.stringify([project.id, project.repositoryIdentity ?? null]));
116
+ }
117
+ return hash.digest("base64url");
118
+}
119
+
120
+/**
121
+ * The resolved roots a resume sends. A client that names the current identities holds them
122
+ * already, but it keeps a cached identity over an absent one, so a root resolved to no identity
123
+ * still goes out.
124
+ */
125
+export function resumeIdentityRoots(input: {
126
+ readonly projects: ReadonlyArray<
127
+ Pick<OrchestrationProjectShell, "id" | "workspaceRoot" | "repositoryIdentity">
128
+ >;
129
+ readonly resolvedRepositoryIdentityRoots: ReadonlyArray<string>;
130
+ readonly knownProjectIdentities: string | undefined;
131
+}): ReadonlyArray<string> {
132
+ if (input.knownProjectIdentities !== projectIdentitiesDigest(input.projects)) {
133
+ return input.resolvedRepositoryIdentityRoots;
134
+ }
135
+ const withoutIdentity = new Set(
136
+ input.projects.flatMap((project) =>
137
+ project.repositoryIdentity == null ? [project.workspaceRoot] : [],
138
+ ),
139
+ );
140
+ return input.resolvedRepositoryIdentityRoots.filter((root) => withoutIdentity.has(root));
141
+}
142
+
143
/** Snapshots, synchronization, and enrichment markers must bypass full-state coalescing. */
144
export const bufferShellLiveStream = <E, R>(
145
source: Stream.Stream<ShellDelta, E, R>,
apps/server/src/orchestration-v2/http.test.ts
+10
@@ -369,6 +369,8 @@ it.effect("leaves older settled threads out of the HTTP shell only when asked",
369
}),
370
);
371
return (await response.json()) as {
372
+ readonly snapshotSequence: number;
373
+ readonly projectIdentities?: string;
374
readonly threads: ReadonlyArray<{ readonly id: string }>;
375
readonly olderSettled?: { readonly counts: ReadonlyArray<unknown> };
376
};
@@ -382,5 +384,13 @@ it.effect("leaves older settled threads out of the HTTP shell only when asked",
384
[thread.id, ...Array.from({ length: 25 }, (_, index) => `settled-${index + 1}`)].toSorted(),
385
);
386
expect(cut.olderSettled?.counts).toEqual([{ projectId: thread.projectId, count: 1 }]);
387
+
388
+ // A client that holds rows the socket can replay from keeps them; any other cursor reloads.
389
+ expect(full.projectIdentities).toBeTypeOf("string");
390
+ expect(yield* shell(`?resumeFrom=${full.snapshotSequence}`)).toEqual({ resumable: true });
391
+ expect(yield* shell(`?resumeFrom=${full.snapshotSequence - 1}`)).toEqual({ resumable: true });
392
+ for (const cursor of [full.snapshotSequence + 1, "x"]) {
393
+ expect((yield* shell(`?omitOlderSettled=1&resumeFrom=${cursor}`)).threads).toHaveLength(26);
394
+ }
395
}),
396
);
apps/server/src/orchestration-v2/http.ts
+24
-6
@@ -31,7 +31,11 @@ import {
31
} from "./threadHistoryPaging.ts";
32
import * as ThreadManagementService from "./ThreadManagementService.ts";
33
import * as ProjectStore from "./ProjectStore.ts";
34
-import { loadSubscriberShellSnapshot } from "./ShellStream.ts";
34
+import {
35
+ loadShellUnlessResumable,
36
+ loadSubscriberShellSnapshot,
37
+ projectIdentitiesDigest,
38
+} from "./ShellStream.ts";
39
import { boundedSnapshotResponseFields } from "./ThreadStream.ts";
40
import { projectThreadProjectionForWire } from "./WireProjection.ts";
41
@@ -94,10 +98,27 @@ export const layer = HttpApiBuilder.group(
98
options,
99
});
100
const projects = yield* enrichProjectShells(snapshot.projects);
97
- return { ...snapshot, projects };
101
+ return { ...snapshot, projects, projectIdentities: projectIdentitiesDigest(projects) };
102
},
103
);
104
105
+ // A client that holds rows names their cursor; a small answer tells it to keep them.
106
+ const answerShellSnapshot = Effect.fn("http.orchestration.answerShellSnapshot")((query: {
107
+ readonly omitIdleSubagents?: string;
108
+ readonly omitOlderSettled?: string;
109
+ readonly resumeFrom?: string;
110
+ }) => {
111
+ const resumeFrom = query.resumeFrom === undefined ? Number.NaN : Number(query.resumeFrom);
112
+ return loadShellUnlessResumable({
113
+ events: applicationEvents,
114
+ resumeFrom: Number.isSafeInteger(resumeFrom) && resumeFrom >= 0 ? resumeFrom : undefined,
115
+ loadSnapshot: loadShellSnapshot({
116
+ omitIdleSubagents: query.omitIdleSubagents === "1",
117
+ omitOlderSettled: query.omitOlderSettled === "1",
118
+ }),
119
+ });
120
+ });
121
+
122
const loadThreadSnapshot = Effect.fn("http.orchestration.loadThreadSnapshot")(function* (
123
threadId: Parameters<typeof threadManagement.getThreadSnapshot>[0],
124
failureReason:
@@ -166,10 +187,7 @@ export const layer = HttpApiBuilder.group(
187
Effect.fn("environment.orchestration.shellSnapshot")(function* (args) {
188
yield* annotateEnvironmentRequest(args.endpoint.name);
189
yield* requireEnvironmentScope(AuthOrchestrationReadScope);
169
- return yield* loadShellSnapshot({
170
- omitIdleSubagents: args.query.omitIdleSubagents === "1",
171
- omitOlderSettled: args.query.omitOlderSettled === "1",
172
- }).pipe(
190
+ return yield* answerShellSnapshot(args.query).pipe(
191
traceLocalHandlerWork,
192
Effect.catch((cause) =>
193
failEnvironmentInternal("orchestration_snapshot_failed", cause),
apps/server/src/server.ts
+2
-1
@@ -33,6 +33,7 @@ import { withUntracedRequests } from "./http.ts";
33
import * as ServerHttp from "./http.ts";
34
import {
35
guardHttpResponseWriteErrors,
36
+ keepIdleConnections,
37
rejectRequestsUntilServing,
38
} from "./httpResponseErrorGuard.ts";
39
import { fixPath } from "./os-jank.ts";
@@ -354,7 +355,7 @@ const layerHttpServer = Layer.unwrap(
355
return NodeHttpServer.layer(
356
() => {
357
const server = rejectRequestsUntilServing(
357
- guardHttpResponseWriteErrors(NodeHttp.createServer()),
358
+ keepIdleConnections(guardHttpResponseWriteErrors(NodeHttp.createServer())),
359
);
360
if (config.privateNetworkOnly) {
361
server.on("connection", (socket) => {
apps/server/src/ws.test.ts
+118
-35
@@ -149,6 +149,7 @@ import * as EventSink from "./orchestration-v2/EventSink.ts";
149
import * as LegacyV1ThreadImporter from "./orchestration-v2/legacy/LegacyV1ThreadImporter.ts";
150
import * as ThreadCommandExecutor from "./orchestration-v2/ThreadCommandExecutor.ts";
151
import * as IdAllocator from "@t3tools/provider-core/server/IdAllocator";
152
+import { projectIdentitiesDigest } from "./orchestration-v2/ShellStream.ts";
153
import * as OrchestrationEventStore from "./persistence/OrchestrationEventStore.ts";
154
import * as SqlitePersistence from "./persistence/Sqlite.ts";
155
import * as ExternalLauncher from "./process/externalLauncher.ts";
@@ -1980,41 +1981,123 @@ it.effect.each(["thread", "shell"] as const)(
1981
}).pipe(Effect.scoped, Effect.provide(ReplayTestLayer)),
1982
);
1983
1983
-// A thread idle since before the boundary missed nothing; resuming must not resend its body.
1984
-it.effect("thread resume at its own high-water below the pruned boundary skips the snapshot", () =>
1985
- Effect.gen(function* () {
1986
- const sql = yield* SqlClient.SqlClient;
1987
- const now = "2026-10-13T00:00:00.000Z";
1988
- const threadId = ThreadId.make("thread:idle");
1989
- const [last] = yield* sql<{ sequence: number }>`
1990
- INSERT INTO orchestration_events (
1991
- event_id, aggregate_kind, stream_id, stream_version, event_type,
1992
- occurred_at, actor_kind, payload_json, metadata_json, application_event_version
1993
- ) VALUES (
1994
- 'idle:0', 'thread', ${threadId}, 0, 'provider-session.detached',
1995
- ${now}, 'server', ${JSON.stringify({ providerSessionId: "session:idle", detachedAt: now })},
1996
- '{}', 2
1997
- ) RETURNING sequence
1998
- `;
1999
- yield* sql`
2000
- INSERT INTO orchestration_v2_projection_metadata (
2001
- projection_name, schema_version, last_sequence, updated_at
2002
- ) VALUES (${OrchestrationEventStore.EVENT_LOG_PRUNED_THROUGH}, 2, ${last!.sequence + 10}, ${now})
2003
- `;
2004
- const exit = yield* subscribeOrchestrationV2Thread({
2005
- threadId,
2006
- afterSequence: last!.sequence,
2007
- }).pipe(
2008
- Effect.provideService(ThreadManagement.ThreadManagementService, {
2009
- ensureLegacyTranscript: () => Effect.void,
2010
- getThreadSnapshot: () => Effect.die("snapshot"),
2011
- streamStoredEventsFrom: () => Stream.empty,
2012
- } as unknown as ThreadManagement.ThreadManagementService["Service"]),
2013
- Effect.exit,
2014
- );
2015
- // A snapshot load dies with "snapshot", so success means the resume replayed nothing.
2016
- assert.isTrue(Exit.isSuccess(exit));
2017
- }).pipe(Effect.scoped, Effect.provide(ReplayTestLayer)),
1984
+// A cursor at the high-water missed nothing, even below the boundary; resuming must not resend
1985
+// a snapshot. The shell cases are a client that just loaded the HTTP snapshot of a pruned
1986
+// database: it gets the marker, and the project identities only when it does not name them.
1987
+it.effect.each([
1988
+ { scope: "thread", knowsIdentities: false, isRepository: true, kinds: [] },
1989
+ {
1990
+ scope: "shell",
1991
+ knowsIdentities: false,
1992
+ isRepository: true,
1993
+ kinds: ["snapshot", "synchronized"],
1994
+ },
1995
+ { scope: "shell", knowsIdentities: true, isRepository: true, kinds: ["synchronized"] },
1996
+ // The client keeps a cached identity over an absent one, so "resolved to none" still goes out.
1997
+ {
1998
+ scope: "shell",
1999
+ knowsIdentities: true,
2000
+ isRepository: false,
2001
+ kinds: ["snapshot", "synchronized"],
2002
+ },
2003
+] as const)(
2004
+ "$scope resume at its own high-water below the pruned boundary skips the snapshot (knows identities: $knowsIdentities, repository: $isRepository)",
2005
+ ({ scope, knowsIdentities, isRepository, kinds }) =>
2006
+ Effect.gen(function* () {
2007
+ const sql = yield* SqlClient.SqlClient;
2008
+ const now = "2026-10-13T00:00:00.000Z";
2009
+ const threadId = ThreadId.make("thread:idle");
2010
+ const [last] = yield* sql<{ sequence: number }>`
2011
+ INSERT INTO orchestration_events (
2012
+ event_id, aggregate_kind, stream_id, stream_version, event_type,
2013
+ occurred_at, actor_kind, payload_json, metadata_json, application_event_version
2014
+ ) VALUES (
2015
+ 'idle:0', 'thread', ${threadId}, 0, 'provider-session.detached',
2016
+ ${now}, 'server', ${JSON.stringify({ providerSessionId: "session:idle", detachedAt: now })},
2017
+ '{}', 2
2018
+ ) RETURNING sequence
2019
+ `;
2020
+ yield* sql`
2021
+ INSERT INTO orchestration_v2_projection_metadata (
2022
+ projection_name, schema_version, last_sequence, updated_at
2023
+ ) VALUES (${OrchestrationEventStore.EVENT_LOG_PRUNED_THROUGH}, 2, ${last!.sequence + 10}, ${now})
2024
+ `;
2025
+ const identity = isRepository
2026
+ ? ({
2027
+ canonicalKey: "github.com/t3/idle",
2028
+ locator: {
2029
+ source: "git-remote",
2030
+ remoteName: "origin",
2031
+ remoteUrl: "git@github.com:t3/idle",
2032
+ },
2033
+ } as const)
2034
+ : null;
2035
+ const project = {
2036
+ id: ProjectId.make("project:idle"),
2037
+ title: "Idle",
2038
+ workspaceRoot: "/tmp/idle",
2039
+ defaultModelSelection: null,
2040
+ scripts: [],
2041
+ createdAt: now,
2042
+ updatedAt: now,
2043
+ } satisfies OrchestrationProjectShell;
2044
+ const subscription = Effect.gen(function* () {
2045
+ if (scope === "thread") {
2046
+ return yield* subscribeOrchestrationV2Thread({
2047
+ threadId,
2048
+ afterSequence: last!.sequence,
2049
+ }).pipe(Effect.as([]));
2050
+ }
2051
+ const stream = yield* subscribeOrchestrationV2Shell({
2052
+ afterSequence: last!.sequence,
2053
+ requestCompletionMarker: true,
2054
+ ...(knowsIdentities
2055
+ ? {
2056
+ knownProjectIdentities: projectIdentitiesDigest([
2057
+ { ...project, repositoryIdentity: identity },
2058
+ ]),
2059
+ }
2060
+ : {}),
2061
+ });
2062
+ return yield* stream.pipe(
2063
+ Stream.takeUntil((item) => item.kind === "synchronized"),
2064
+ Stream.map((item) => item.kind),
2065
+ Stream.runCollect,
2066
+ );
2067
+ });
2068
+ const exit = yield* subscription.pipe(
2069
+ Effect.provideService(ThreadManagement.ThreadManagementService, {
2070
+ ensureLegacyTranscript: () => Effect.void,
2071
+ getThreadSnapshot: () => Effect.die("snapshot"),
2072
+ listActiveShellIndex: () => Effect.die("snapshot"),
2073
+ readShellSnapshot: () => Effect.die("snapshot"),
2074
+ listActiveSubagentThreadIds: () => Effect.succeed([]),
2075
+ streamStoredEventsFrom: () => Stream.empty,
2076
+ } as unknown as ThreadManagement.ThreadManagementService["Service"]),
2077
+ Effect.provideService(ProjectStore.ProjectStoreV2, {
2078
+ listShells: () => Effect.succeed([project]),
2079
+ } as unknown as ProjectStore.ProjectStoreV2["Service"]),
2080
+ Effect.provideService(
2081
+ ProjectService.ProjectService,
2082
+ {} as ProjectService.ProjectService["Service"],
2083
+ ),
2084
+ Effect.provideService(ProjectEnrichment.ProjectEnrichmentService, {
2085
+ subscribeChanges: PubSub.subscribe(yield* PubSub.unbounded()),
2086
+ getAvailable: () =>
2087
+ Effect.succeed({
2088
+ repositoryIdentity: identity,
2089
+ faviconPath: null,
2090
+ repositoryIdentityResolved: true,
2091
+ }),
2092
+ } as unknown as ProjectEnrichment.ProjectEnrichmentService["Service"]),
2093
+ Effect.exit,
2094
+ );
2095
+ // A snapshot load dies with "snapshot", so success means the resume replayed nothing.
2096
+ assert.deepEqual(
2097
+ Exit.isFailure(exit) ? Cause.squash(exit.cause) : Array.from(exit.value),
2098
+ kinds,
2099
+ );
2100
+ }).pipe(Effect.scoped, Effect.provide(ReplayTestLayer)),
2101
);
2102
2103
const acpDriver = ProviderDriverKind.make("acpRegistry");
apps/server/src/ws.ts
+16
-39
@@ -136,6 +136,8 @@ import {
136
shellStreamItemFromThreadShell,
137
shellStreamItemsFromInitialSnapshot,
138
loadSubscriberShellSnapshot,
139
+ canResumeShellFrom,
140
+ resumeIdentityRoots,
141
shellStreamItemsFromResumeSnapshot,
142
shouldPublishShellApplicationEvent,
143
skipUnchangedThreadShells,
@@ -722,12 +724,6 @@ const isOtherWsRpc = (rpc: ServerWsRpc): rpc is Exclude<ServerWsRpc, SplitWsRpc>
724
const serverWsRpcs = Array.from(ServerWsRpcGroup.requests.values());
725
const SplitWsRpcGroup = RpcGroup.make(...serverWsRpcs.filter(isSplitWsRpc));
726
const OtherWsRpcGroup = RpcGroup.make(...serverWsRpcs.filter(isOtherWsRpc));
725
-// Replaying each intervening event costs a shell refetch; past this gap a
726
-// single O(active-threads) snapshot is cheaper and bounded.
727
-const SHELL_RESUME_MAX_GAP = 1_000;
728
-// Row counts do not bound large tool payloads; sum stored JSON bytes in SQL and snapshot above budget.
729
-const ORCHESTRATION_REPLAY_PAYLOAD_BUDGET_BYTES = 8 * 1024 * 1024;
730
-
727
function toAuthAccessStreamEvent(
728
change: PairingGrantStore.BootstrapCredentialChange | SessionStore.SessionCredentialChange,
729
revision: number,
@@ -864,36 +860,6 @@ function readClientOwners(
860
);
861
}
862
867
-const canReplayPersistedRange = Effect.fnUntraced(function* (
868
- afterSequence: number,
869
- headSequence: number,
870
- maxGap: number,
871
-) {
872
- const applicationEvents = yield* OrchestrationEventStore.OrchestrationEventStore;
873
-
874
- const replayGap = headSequence - afterSequence;
875
- if (replayGap < 0 || replayGap > maxGap) {
876
- return false;
877
- }
878
- if (afterSequence < (yield* applicationEvents.prunedThroughSequence)) return false;
879
- const stats = yield* applicationEvents.getReplayStats({
880
- afterSequence,
881
- throughSequence: headSequence,
882
- });
883
- if (stats.rawPayloadBytes > ORCHESTRATION_REPLAY_PAYLOAD_BUDGET_BYTES) {
884
- yield* Effect.logDebug("orchestration replay replaced by snapshot", {
885
- afterSequence,
886
- headSequence,
887
- replayGap,
888
- eventCount: stats.eventCount,
889
- payloadBytes: stats.rawPayloadBytes,
890
- payloadBudgetBytes: ORCHESTRATION_REPLAY_PAYLOAD_BUDGET_BYTES,
891
- });
892
- return false;
893
- }
894
- return true;
895
-});
896
-
863
const enrichProjectShells = Effect.fn("ws.orchestrationV2.enrichProjectShells")(
864
(projects: ReadonlyArray<OrchestrationProjectShell>) =>
865
Effect.flatMap(ProjectEnrichmentService.ProjectEnrichmentService, (projectEnrichment) =>
@@ -1158,6 +1124,7 @@ export const subscribeOrchestrationV2Shell = Effect.fn("ws.orchestrationV2.subsc
1124
function* (input: {
1125
readonly afterSequence?: number;
1126
readonly httpSnapshotFallback?: true;
1127
+ readonly knownProjectIdentities?: string;
1128
readonly requestCompletionMarker?: boolean;
1129
readonly omitIdleSubagents?: boolean;
1130
readonly omitOlderSettled?: boolean;
@@ -1383,7 +1350,7 @@ export const subscribeOrchestrationV2Shell = Effect.fn("ws.orchestrationV2.subsc
1350
});
1351
});
1352
const highWater = yield* applicationEvents.latestApplicationSequence;
1386
- if (!(yield* canReplayPersistedRange(input.afterSequence, highWater, SHELL_RESUME_MAX_GAP))) {
1353
+ if (!(yield* canResumeShellFrom(applicationEvents, input.afterSequence, highWater))) {
1354
return yield* fromSnapshot;
1355
}
1356
@@ -1402,12 +1369,22 @@ export const subscribeOrchestrationV2Shell = Effect.fn("ws.orchestrationV2.subsc
1369
skipUnknownEventTypes: true,
1370
})
1371
.pipe(Stream.filter(filterSubagentTranscript), Stream.runCollect);
1405
- if (input.afterSequence < (yield* applicationEvents.prunedThroughSequence)) {
1372
+ if (
1373
+ input.afterSequence < highWater &&
1374
+ input.afterSequence < (yield* applicationEvents.prunedThroughSequence)
1375
+ ) {
1376
return yield* fromSnapshot;
1377
}
1378
const replay = toShellStream(Stream.fromIterable(replayed));
1379
return composeShellStreamWithEnrichment({
1410
- initial: initialEnrichmentItems(loaded),
1380
+ initial: initialEnrichmentItems({
1381
+ snapshot: loaded.snapshot,
1382
+ resolvedRepositoryIdentityRoots: resumeIdentityRoots({
1383
+ projects: loaded.snapshot.projects,
1384
+ resolvedRepositoryIdentityRoots: loaded.resolvedRepositoryIdentityRoots,
1385
+ knownProjectIdentities: input.knownProjectIdentities,
1386
+ }),
1387
+ }),
1388
tail: Stream.concat(Stream.concat(replay, completionMarker), liveFrom(highWater)),
1389
enrichment: enrichmentRefreshes,
1390
});
docs/internals/connection-runtime.md
+10
@@ -75,6 +75,16 @@ connection nor overwrite newer live data during a reconnect. Loading and
75
resuming snapshots belongs to the shared state services, so every view agrees
76
on which data is current.
77
78
+The shell snapshot loads over HTTP, which needs only the prepared connection, so
79
+the [shell state](../../packages/client-runtime/src/state/shell.ts) can ask for
80
+it before the socket and the config are ready. Two constraints follow. The live
81
+config stays the authority: rows loaded early for a build or with omissions that
82
+it does not confirm are replaced, never resumed. And an HTTP answer must never
83
+land on rows the socket has moved on: the subscription waits for the answer, and
84
+when held rows can resume, it ends the request 3 seconds after the socket is
85
+ready. A proxy that drops `/api` without an answer then does not hold the
86
+connect for the HTTP timeout.
87
+
88
Each environment synchronizes at its own time, so a thread list that follows every
89
snapshot moves once per environment on each cold start and resume. Mobile lists read
90
[steady shells](../../packages/client-runtime/src/state/steadyShells.ts): an
docs/internals/performance-regressions.md
+8
-2
@@ -54,7 +54,10 @@ The v2 checks pin these invariants:
54
- The initial shell contains active navigation rows only. Archived rows use the dedicated archive
55
query, and transcript message bodies stay in thread detail regardless of message size.
56
- Shell resume sends deltas plus compact repository-enrichment metadata, not another full project
57
- and thread snapshot.
57
+ and thread snapshot. A cursor at the head sequence always resumes, even below the pruned
58
+ boundary, so a client that just loaded the HTTP snapshot never downloads it again over the
59
+ socket. That client also names the snapshot's project identities, and a server whose
60
+ identities did not change since resends only the roots that resolved to no identity.
61
- Auto, steer, and restart sends resolve delivery from authoritative state inside the
62
server's per-thread dispatch lock. Model selection and identified-checkpoint rollback also
63
dispatch without first fetching a full thread projection when the server advertises support.
@@ -76,7 +79,10 @@ bounded snapshot and one older page) and after it finishes (the deferred cold op
79
over the socket alone, and the first page of its deferred work). It caps the socket bytes of three
80
resumes that must not send a snapshot: a thread after one ordinary turn, an idle thread, and a
81
shell cursor too old to replay. It caps the cold-connect shell for a sidebar of settled threads and
79
-idle subagents, and the subscription a client opens from that snapshot. It also caps the bytes and
82
+idle subagents, and the subscription a client opens from that snapshot with its project identities.
83
+It caps both answers of the HTTP shell route to a client that holds rows and names their cursor: the
84
+few bytes that keep them, and the snapshot that replaces rows too old to replay. It also caps the
85
+bytes and
86
the node, subagent, plan, provider turn and checkpoint counts of the bounded snapshot for a thread
87
of 40 finished runs behind one working run, so finished runs cannot ride along with a window. The
88
working run has a ready context handoff, so its history cannot ride along either.
packages/client-runtime/src/authorization/layer.test.ts
+85
-15
@@ -51,13 +51,16 @@ const BOOTSTRAP: RelayEnvironmentConnectResponse = {
51
expiresAt: "2026-06-06T01:00:00.000Z",
52
};
53
54
-type RecordedResponse = Response | ((init: RequestInit) => Response);
54
+type RecordedResponse = Response | ((init: RequestInit) => Response | Promise<Response>);
55
56
-function recordedFetch(responses: ReadonlyArray<RecordedResponse>) {
56
+function recordedFetch(responses: ReadonlyArray<RecordedResponse>, descriptor?: object) {
57
const calls: Array<readonly [RequestInfo | URL, RequestInit]> = [];
58
let responseIndex = 0;
59
const fetchFn = ((input, init) => {
60
calls.push([input, init ?? {}]);
61
+ if (descriptor !== undefined && String(input).endsWith("/.well-known/t3/environment")) {
62
+ return Promise.resolve(Response.json(descriptor));
63
+ }
64
const response = responses[responseIndex++];
65
return response === undefined
66
? Promise.reject(new Error(`Unexpected fetch call to ${String(input)}`))
@@ -113,6 +116,8 @@ const persistedToken = (
116
const makeHarness = Effect.fn("TestRemoteAuthorization.makeHarness")(function* (input: {
117
readonly initialToken?: TokenStore.RemoteDpopAccessToken;
118
readonly responses: ReadonlyArray<RecordedResponse>;
119
+ /** Answers every descriptor read, so one that runs beside a ticket does not shift `responses`. */
120
+ readonly descriptor?: object;
121
readonly bootstrap?: RelayEnvironmentConnectResponse;
122
readonly beforeBootstrap?: Effect.Effect<void, ManagedRelay.ManagedRelayClientError>;
123
readonly beforePut?: Effect.Effect<void>;
@@ -141,7 +146,7 @@ const makeHarness = Effect.fn("TestRemoteAuthorization.makeHarness")(function* (
146
readonly accessToken?: string;
147
}>
148
>([]);
144
- const fetch = recordedFetch(input.responses);
149
+ const fetch = recordedFetch(input.responses, input.descriptor);
150
151
const tokenStore = TokenStore.RemoteDpopAccessTokenStore.of({
152
get: (environmentId) =>
@@ -238,6 +243,9 @@ const makeHarness = Effect.fn("TestRemoteAuthorization.makeHarness")(function* (
243
};
244
});
245
246
+const requestsTo = (harness: Effect.Success<ReturnType<typeof makeHarness>>, path: string) =>
247
+ harness.fetch.calls.filter(([url]) => String(url).endsWith(path)).length;
248
+
249
const prepareDirectBearer = Effect.fn("TestRemoteAuthorization.prepareDirectBearer")(function* (
250
harness: Effect.Success<ReturnType<typeof makeHarness>>,
251
times: number,
@@ -318,7 +326,7 @@ describe("RemoteEnvironmentAuthorization", () => {
326
}),
327
);
328
321
- it.effect("reuses a matching descriptor a check just read, once, while fresh, over HTTPS", () =>
329
+ it.effect("reuses a matching descriptor a check just read, once, while fresh", () =>
330
Effect.gen(function* () {
331
const descriptor = {
332
...DESCRIPTOR,
@@ -327,10 +335,13 @@ describe("RemoteEnvironmentAuthorization", () => {
335
const harness = yield* makeHarness({
336
responses: [
337
websocketTicket("reused"),
330
- ...Array.from({ length: 5 }, () => [
338
+ ...Array.from({ length: 4 }, () => [
339
Response.json(descriptor),
340
websocketTicket("fresh"),
341
]).flat(),
342
+ websocketTicket("reused over plain HTTP"),
343
+ Response.json(descriptor),
344
+ websocketTicket("fresh"),
345
],
346
});
347
@@ -346,7 +357,7 @@ describe("RemoteEnvironmentAuthorization", () => {
357
});
358
const authorize = authorizeAt(ENDPOINT.httpBaseUrl);
359
const remember = (value: Option.Option<ExecutionEnvironmentDescriptor>) =>
349
- remote.rememberCheckedDescriptor(ENDPOINT.httpBaseUrl, value);
360
+ remote.rememberCheckedDescriptor(ENDPOINT.httpBaseUrl, value, true);
361
362
yield* remember(Option.some(descriptor));
363
yield* authorize;
@@ -363,17 +374,31 @@ describe("RemoteEnvironmentAuthorization", () => {
374
yield* remember(Option.some(descriptor));
375
yield* remember(Option.none());
376
yield* authorize;
377
+ // Plain HTTP is no reason to read again where the address keeps its host.
378
+ yield* remote.rememberCheckedDescriptor(
379
+ "http://127.0.0.1:3773",
380
+ Option.some(descriptor),
381
+ true,
382
+ );
383
+ yield* authorizeAt("http://127.0.0.1:3773");
384
// After a network change, a plain LAN address can belong to another machine.
367
- yield* remote.rememberCheckedDescriptor("http://192.168.1.2:3773", Option.some(descriptor));
385
+ yield* remote.rememberCheckedDescriptor(
386
+ "http://192.168.1.2:3773",
387
+ Option.some(descriptor),
388
+ false,
389
+ );
390
yield* authorizeAt("http://192.168.1.2:3773");
391
}).pipe(Effect.provide(harness.layer));
392
393
expect(harness.fetch.calls.map(([url]) => new URL(String(url)).pathname)).toEqual([
394
"/api/auth/websocket-ticket/rpc",
373
- ...Array.from({ length: 5 }, () => [
395
+ ...Array.from({ length: 4 }, () => [
396
"/.well-known/t3/environment",
397
"/api/auth/websocket-ticket/rpc",
398
]).flat(),
399
+ "/api/auth/websocket-ticket/rpc",
400
+ "/.well-known/t3/environment",
401
+ "/api/auth/websocket-ticket/rpc",
402
]);
403
}),
404
);
@@ -446,6 +471,44 @@ describe("RemoteEnvironmentAuthorization", () => {
471
}),
472
);
473
474
+ it.effect("gives up on a silent loopback or LAN address sooner than on a remote one", () =>
475
+ Effect.gen(function* () {
476
+ const silent = () => new Promise<Response>(() => {});
477
+ const harness = yield* makeHarness({ responses: [silent, silent, silent] });
478
+
479
+ yield* Effect.gen(function* () {
480
+ const remote = yield* RemoteEnvironmentAuthorization.RemoteEnvironmentAuthorization;
481
+ const authorizeAt = (httpBaseUrl: string, connectionMethod: "direct" | "ssh" = "direct") =>
482
+ remote
483
+ .authorizeBearer({
484
+ expectedEnvironmentId: ENVIRONMENT_ID,
485
+ httpBaseUrl,
486
+ wsBaseUrl: httpBaseUrl.replace(/^http/, "ws"),
487
+ bearerToken: "bearer-token",
488
+ connectionMethod,
489
+ })
490
+ .pipe(Effect.flip, Effect.forkChild);
491
+ const lan = yield* authorizeAt("http://192.168.1.2:3773");
492
+ const tailnet = yield* authorizeAt("http://100.101.102.103:3773");
493
+ // An SSH forward listens on loopback but reaches a remote host.
494
+ const forwarded = yield* authorizeAt("http://127.0.0.1:4010", "ssh");
495
+
496
+ yield* TestClock.adjust("3 seconds");
497
+ expect(lan.pollUnsafe()).toBeDefined();
498
+ expect(tailnet.pollUnsafe()).toBeUndefined();
499
+ expect(forwarded.pollUnsafe()).toBeUndefined();
500
+ expect(yield* Fiber.join(lan)).toMatchObject({
501
+ _tag: "ConnectionTransientError",
502
+ reason: "timeout",
503
+ });
504
+
505
+ yield* TestClock.adjust("7 seconds");
506
+ expect(tailnet.pollUnsafe()).toBeDefined();
507
+ expect(forwarded.pollUnsafe()).toBeDefined();
508
+ }).pipe(Effect.provide(harness.layer));
509
+ }),
510
+ );
511
+
512
it.effect("reuses a same-account persisted environment token without contacting the relay", () =>
513
Effect.gen(function* () {
514
const cached = new TokenStore.RemoteDpopAccessToken({
@@ -459,6 +522,7 @@ describe("RemoteEnvironmentAuthorization", () => {
522
});
523
const harness = yield* makeHarness({
524
initialToken: cached,
525
+ descriptor: DESCRIPTOR,
526
responses: [websocketTicket("cached-ticket")],
527
});
528
@@ -472,10 +536,12 @@ describe("RemoteEnvironmentAuthorization", () => {
536
expect(authorized.socketUrl).toContain("wsTicket=cached-ticket");
537
expect(authorized.socketUrl).toContain("connectionMethod=relay");
538
expect(yield* Ref.get(harness.bootstrapCalls)).toBe(0);
475
- expect(harness.fetch.calls).toHaveLength(1);
476
- expect(String(harness.fetch.calls[0]?.[0])).toBe(
539
+ // The descriptor is read beside the ticket, so the connect needs no later read.
540
+ expect(authorized.descriptor).toMatchObject({ environmentId: ENVIRONMENT_ID });
541
+ expect(harness.fetch.calls.map(([url]) => String(url)).toSorted()).toEqual([
542
+ "https://environment.example.test/.well-known/t3/environment",
543
"https://environment.example.test/api/auth/websocket-ticket/rpc",
478
- );
544
+ ]);
545
}),
546
);
547
@@ -576,6 +642,8 @@ describe("RemoteEnvironmentAuthorization", () => {
642
}).pipe(Effect.provide(harness.layer));
643
644
expect(authorized.socketUrl).toContain("wsTicket=fresh-ticket");
645
+ // The mint already read the descriptor.
646
+ expect(authorized.descriptor).toMatchObject({ environmentId: ENVIRONMENT_ID });
647
expect(yield* Ref.get(harness.bootstrapCalls)).toBe(1);
648
expect((yield* Ref.get(harness.tokens)).get(ENVIRONMENT_ID)).toEqual(
649
expect.objectContaining({
@@ -729,9 +797,9 @@ describe("RemoteEnvironmentAuthorization", () => {
797
});
798
const harness = yield* makeHarness({
799
initialToken: cached,
800
+ descriptor: DESCRIPTOR,
801
responses: [
802
authInvalid(),
734
- Response.json(DESCRIPTOR),
803
accessToken("replacement-access-token"),
804
websocketTicket("replacement-ticket"),
805
],
@@ -751,7 +819,8 @@ describe("RemoteEnvironmentAuthorization", () => {
819
accessToken: "replacement-access-token",
820
}),
821
);
754
- expect(harness.fetch.calls).toHaveLength(4);
822
+ expect(requestsTo(harness, "/api/auth/websocket-ticket/rpc")).toBe(2);
823
+ expect(requestsTo(harness, "/oauth/token")).toBe(1);
824
}),
825
);
826
@@ -768,9 +837,9 @@ describe("RemoteEnvironmentAuthorization", () => {
837
});
838
const harness = yield* makeHarness({
839
initialToken: cached,
840
+ descriptor: DESCRIPTOR,
841
responses: [
842
new Response("endpoint unavailable", { status: 503 }),
773
- Response.json(DESCRIPTOR),
843
accessToken("replacement-access-token"),
844
websocketTicket("replacement-ticket"),
845
],
@@ -790,7 +859,8 @@ describe("RemoteEnvironmentAuthorization", () => {
859
accessToken: "replacement-access-token",
860
}),
861
);
793
- expect(harness.fetch.calls).toHaveLength(4);
862
+ expect(requestsTo(harness, "/api/auth/websocket-ticket/rpc")).toBe(2);
863
+ expect(requestsTo(harness, "/oauth/token")).toBe(1);
864
}),
865
);
866
packages/client-runtime/src/authorization/service.ts
+75
-37
@@ -22,6 +22,7 @@ import {
22
ConnectionTransientError,
23
type ConnectionAttemptError,
24
} from "../connection/model.ts";
25
+import { isNearbyAddress } from "../connection/routes.ts";
26
import { fetchRemoteEnvironmentDescriptor } from "../environment/descriptor.ts";
27
import { environmentEndpointUrl } from "../environment/endpoint.ts";
28
import * as ClientCapabilities from "../platform/capabilities.ts";
@@ -82,10 +83,14 @@ export class RemoteEnvironmentAuthorization extends Context.Service<
83
readonly expectedEnvironmentId: EnvironmentId;
84
readonly rejectedAccessToken?: string;
85
}) => Effect.Effect<AuthorizedRemoteHttpEnvironment, ConnectionAttemptError>;
85
- /** Lets the next direct authorization of this address skip its own descriptor read. */
86
+ /**
87
+ * Lets the next direct authorization of this address skip its own descriptor
88
+ * read. Kept only where the address reaches the same machine on every network.
89
+ */
90
readonly rememberCheckedDescriptor: (
91
httpBaseUrl: string,
92
descriptor: Option.Option<ExecutionEnvironmentDescriptor>,
93
+ keepsItsHost: boolean,
94
) => Effect.Effect<void>;
95
/**
96
* Renews a live socket's token before it expires, so the server moves the
@@ -101,6 +106,14 @@ export class RemoteEnvironmentAuthorization extends Context.Service<
106
const CACHED_ENDPOINT_SOCKET_TIMEOUT_MS = 3_000;
107
const DPOP_AUTHORIZATION_TIMEOUT_MS = 30_000;
108
const CHECKED_DESCRIPTOR_TTL_NANOS = 5_000_000_000n;
109
+// A loopback or LAN host answers well inside this, so a dead one frees its attempt sooner.
110
+const NEARBY_DESCRIPTOR_TIMEOUT_MS = 3_000;
111
+
112
+/** A token with the descriptor its mint read. A cached token has none. */
113
+interface IssuedDpopToken {
114
+ readonly token: TokenStore.RemoteDpopAccessToken;
115
+ readonly descriptor?: ExecutionEnvironmentDescriptor;
116
+}
117
118
function mapDpopSocketError(error: RemoteEnvironmentAuthError | ConnectionAttemptError) {
119
return error._tag === "ConnectionTransientError" || error._tag === "ConnectionBlockedError"
@@ -111,10 +124,12 @@ function mapDpopSocketError(error: RemoteEnvironmentAuthError | ConnectionAttemp
124
const fetchDescriptor = Effect.fn("clientRuntime.connection.remote.fetchDescriptor")(function* (
125
httpBaseUrl: string,
126
connectionMethod: ClientConnectionMethod,
127
+ timeoutMs?: number,
128
) {
115
- return yield* fetchRemoteEnvironmentDescriptor({ httpBaseUrl }).pipe(
116
- Effect.mapError((error) => mapRemoteEnvironmentError(error, connectionMethod)),
117
- );
129
+ return yield* fetchRemoteEnvironmentDescriptor({
130
+ httpBaseUrl,
131
+ ...(timeoutMs === undefined ? {} : { timeoutMs }),
132
+ }).pipe(Effect.mapError((error) => mapRemoteEnvironmentError(error, connectionMethod)));
133
});
134
135
export const make = Effect.gen(function* () {
@@ -137,13 +152,13 @@ export const make = Effect.gen(function* () {
152
readonly id: object;
153
readonly identity: ClientCapabilities.CloudSessionIdentity;
154
readonly thumbprint: string;
140
- readonly token: Effect.Effect<TokenStore.RemoteDpopAccessToken, ConnectionAttemptError>;
155
+ readonly issued: Effect.Effect<IssuedDpopToken, ConnectionAttemptError>;
156
}
157
>();
158
159
// A route or host check hands its descriptor to the connect it leads to, once,
145
- // so the connect skips a round trip. Only over HTTPS: a plain LAN address can
146
- // belong to another machine after a network change.
160
+ // so the connect skips a round trip. Only over HTTPS or loopback: any other
161
+ // plain http address can belong to another machine after a network change.
162
const checkedDescriptors = new Map<
163
string,
164
{ readonly descriptor: ExecutionEnvironmentDescriptor; readonly at: bigint }
@@ -167,9 +182,14 @@ export const make = Effect.gen(function* () {
182
checked.descriptor.environmentId === input.expectedEnvironmentId &&
183
(yield* Clock.monotonicTimeNanos) - checked.at <= CHECKED_DESCRIPTOR_TTL_NANOS
184
? checked.descriptor
170
- : yield* fetchDescriptor(input.httpBaseUrl, input.connectionMethod).pipe(
171
- Effect.provideService(HttpClient.HttpClient, httpClient),
172
- );
185
+ : yield* fetchDescriptor(
186
+ input.httpBaseUrl,
187
+ input.connectionMethod,
188
+ // An SSH forward listens on loopback but reaches a remote host.
189
+ input.connectionMethod === "direct" && isNearbyAddress(input.httpBaseUrl)
190
+ ? NEARBY_DESCRIPTOR_TIMEOUT_MS
191
+ : undefined,
192
+ ).pipe(Effect.provideService(HttpClient.HttpClient, httpClient));
193
if (descriptor.environmentId !== input.expectedEnvironmentId) {
194
return yield* environmentMismatchError({
195
expected: input.expectedEnvironmentId,
@@ -352,7 +372,7 @@ export const make = Effect.gen(function* () {
372
Effect.withSpan("environment.authorization.accessToken.exchange"),
373
);
374
const issuedAt = yield* Clock.currentTimeMillis;
355
- return new TokenStore.RemoteDpopAccessToken({
375
+ const token = new TokenStore.RemoteDpopAccessToken({
376
environmentId: descriptor.environmentId,
377
accountId: identity.accountId,
378
label: descriptor.label,
@@ -361,6 +381,7 @@ export const make = Effect.gen(function* () {
381
expiresAtEpochMs: issuedAt + access.expires_in * 1_000,
382
dpopThumbprint: thumbprint,
383
});
384
+ return { token, descriptor } satisfies IssuedDpopToken;
385
},
386
);
387
@@ -403,14 +424,17 @@ export const make = Effect.gen(function* () {
424
accessToken: cached.value.accessToken,
425
identity,
426
});
406
- return { token: Effect.succeed(cached.value), fromCache: true };
427
+ return {
428
+ issued: Effect.succeed<IssuedDpopToken>({ token: cached.value }),
429
+ fromCache: true,
430
+ };
431
}
432
if (input.rejectedAccessToken !== undefined) {
433
yield* removeRejectedToken(input.expectedEnvironmentId, input.rejectedAccessToken);
434
}
435
const pending = pendingTokens.get(input.expectedEnvironmentId);
436
if (pending?.identity === identity && pending.thumbprint === thumbprint) {
413
- return { token: pending.token, fromCache: false };
437
+ return { issued: pending.issued, fromCache: false };
438
}
439
440
const id = {};
@@ -419,9 +443,10 @@ export const make = Effect.gen(function* () {
443
thumbprint,
444
identity,
445
).pipe(
422
- Effect.flatMap((token) =>
446
+ Effect.flatMap((issued) =>
447
tokenLock.withPermits(1)(
448
Effect.gen(function* () {
449
+ const { token } = issued;
450
yield* assertSession(identity);
451
yield* tokenStore
452
.put(token)
@@ -438,7 +463,7 @@ export const make = Effect.gen(function* () {
463
if (pendingTokens.get(input.expectedEnvironmentId)?.id === id) {
464
pendingTokens.delete(input.expectedEnvironmentId);
465
}
441
- return token;
466
+ return issued;
467
}),
468
),
469
),
@@ -464,17 +489,17 @@ export const make = Effect.gen(function* () {
489
// One caller's timeout must not cancel renewal for other HTTP requests.
490
Effect.forkIn(serviceScope),
491
);
467
- const token = Fiber.join(fiber);
468
- pendingTokens.set(input.expectedEnvironmentId, { id, identity, thumbprint, token });
469
- return { token, fromCache: false };
492
+ const issued = Fiber.join(fiber);
493
+ pendingTokens.set(input.expectedEnvironmentId, { id, identity, thumbprint, issued });
494
+ return { issued, fromCache: false };
495
}),
496
);
497
yield* Effect.annotateCurrentSpan({
498
"connection.remote_token_cache": selected.fromCache ? "hit" : "miss",
499
});
475
- const token = yield* selected.token;
500
+ const issued = yield* selected.issued;
501
yield* assertSession(identity);
477
- return { token, fromCache: selected.fromCache, identity };
502
+ return { ...issued, fromCache: selected.fromCache, identity };
503
});
504
505
const httpAuthorization = (
@@ -495,7 +520,7 @@ export const make = Effect.gen(function* () {
520
input: Parameters<RemoteEnvironmentAuthorization["Service"]["authorizeDpop"]>[0],
521
) {
522
const endpoint = input.directEndpoint;
498
- const descriptor =
523
+ const directDescriptor =
524
endpoint === undefined
525
? undefined
526
: yield* verifyDirectEndpoint({
@@ -503,21 +528,36 @@ export const make = Effect.gen(function* () {
528
httpBaseUrl: endpoint.httpBaseUrl,
529
connectionMethod: "direct",
530
});
506
- const authorized = (token: TokenStore.RemoteDpopAccessToken, socketUrl: string) => ({
507
- ...httpAuthorization(token, endpoint?.httpBaseUrl),
508
- socketUrl,
509
- ...(descriptor === undefined ? {} : { descriptor }),
531
+ const authorized = (
532
+ token: TokenStore.RemoteDpopAccessToken,
533
+ socketUrl: string,
534
+ descriptor: ExecutionEnvironmentDescriptor,
535
+ ) => ({ ...httpAuthorization(token, endpoint?.httpBaseUrl), socketUrl, descriptor });
536
+ // A cached token comes without a descriptor. That read needs no ticket, so
537
+ // it runs beside the ticket request, and a failed ticket abandons it.
538
+ const openSocket = Effect.fnUntraced(function* (issued: IssuedDpopToken, timeoutMs?: number) {
539
+ const socket = createDpopSocketUrl(issued.token, timeoutMs, endpoint).pipe(Effect.result);
540
+ const known = directDescriptor ?? issued.descriptor;
541
+ if (known !== undefined) {
542
+ return Result.map(yield* socket, (socketUrl) => authorized(issued.token, socketUrl, known));
543
+ }
544
+ const reading = yield* fetchDescriptor(issued.token.endpoint.httpBaseUrl, "relay").pipe(
545
+ Effect.provideService(HttpClient.HttpClient, httpClient),
546
+ Effect.forkChild,
547
+ );
548
+ const opened = yield* socket;
549
+ if (Result.isFailure(opened)) {
550
+ yield* Fiber.interrupt(reading);
551
+ return Result.fail(opened.failure);
552
+ }
553
+ return Result.succeed(authorized(issued.token, opened.success, yield* Fiber.join(reading)));
554
});
555
let selected = yield* getDpopToken(input);
556
if (selected.fromCache) {
513
- const cachedSocket = yield* createDpopSocketUrl(
514
- selected.token,
515
- CACHED_ENDPOINT_SOCKET_TIMEOUT_MS,
516
- endpoint,
517
- ).pipe(Effect.result);
557
+ const cachedSocket = yield* openSocket(selected, CACHED_ENDPOINT_SOCKET_TIMEOUT_MS);
558
if (Result.isSuccess(cachedSocket)) {
559
yield* assertSession(selected.identity);
520
- return authorized(selected.token, cachedSocket.success);
560
+ return cachedSocket.success;
561
}
562
if (cachedSocket.failure._tag === "ConnectionBlockedError") {
563
return yield* mapDpopSocketError(cachedSocket.failure);
@@ -533,9 +573,7 @@ export const make = Effect.gen(function* () {
573
rejectedAccessToken: selected.token.accessToken,
574
});
575
}
536
- const socket = yield* createDpopSocketUrl(selected.token, undefined, endpoint).pipe(
537
- Effect.result,
538
- );
576
+ const socket = yield* openSocket(selected);
577
if (Result.isFailure(socket)) {
578
if (endpoint === undefined || socket.failure._tag === "ConnectionBlockedError") {
579
yield* tokenLock.withPermits(1)(
@@ -545,14 +583,14 @@ export const make = Effect.gen(function* () {
583
return yield* mapDpopSocketError(socket.failure);
584
}
585
yield* assertSession(selected.identity);
548
- return authorized(selected.token, socket.success);
586
+ return socket.success;
587
});
588
589
return RemoteEnvironmentAuthorization.of({
590
authorizeBearer,
591
// Untraced: it runs on every route and host check.
554
- rememberCheckedDescriptor: Effect.fnUntraced(function* (httpBaseUrl, descriptor) {
555
- if (Option.isNone(descriptor) || !httpBaseUrl.startsWith("https:")) {
592
+ rememberCheckedDescriptor: Effect.fnUntraced(function* (httpBaseUrl, descriptor, keepsItsHost) {
593
+ if (Option.isNone(descriptor) || !keepsItsHost) {
594
checkedDescriptors.delete(httpBaseUrl);
595
return;
596
}
packages/client-runtime/src/connection/driver.ts
+11
-4
@@ -20,6 +20,7 @@ import {
20
connectionRoutes,
21
routeEntry,
22
routeHttpBaseUrl,
23
+ routeKeepsItsHost,
24
sendsTokenInClear,
25
usableRoutes,
26
} from "./routes.ts";
@@ -148,7 +149,7 @@ export const make = Effect.gen(function* () {
149
150
// The descriptor is public, so this sends no credential to whatever
151
// answers at a saved LAN address on a different network.
151
- const checkDescriptor = (httpBaseUrl: string, environmentId: string) =>
152
+ const checkDescriptor = (httpBaseUrl: string, environmentId: string, keepsItsHost: boolean) =>
153
fetchRemoteEnvironmentDescriptor({
154
httpBaseUrl,
155
timeoutMs: ROUTE_CHECK_TIMEOUT_MS,
@@ -156,7 +157,9 @@ export const make = Effect.gen(function* () {
157
Effect.option,
158
// The connect this check leads to skips its own read. Every answer, even a
159
// failure, replaces what an older check remembered.
159
- Effect.tap((descriptor) => remote.rememberCheckedDescriptor(httpBaseUrl, descriptor)),
160
+ Effect.tap((descriptor) =>
161
+ remote.rememberCheckedDescriptor(httpBaseUrl, descriptor, keepsItsHost),
162
+ ),
163
Effect.map((descriptor): RouteCheck =>
164
Option.exists(descriptor, (value) => value.environmentId === environmentId)
165
? "answered"
@@ -170,7 +173,7 @@ export const make = Effect.gen(function* () {
173
const checkRoute = (entry: ConnectionCatalogEntry, route: ConnectionRoute) => {
174
const httpBaseUrl = routeHttpBaseUrl(route);
175
if (httpBaseUrl === null) return Effect.succeed<RouteCheck>("unchecked");
173
- return checkDescriptor(httpBaseUrl, entry.target.environmentId).pipe(
176
+ return checkDescriptor(httpBaseUrl, entry.target.environmentId, routeKeepsItsHost(route)).pipe(
177
Effect.withSpan("ConnectionDriver.checkRoute", {
178
attributes: { "connection.target.kind": route.target._tag },
179
}),
@@ -178,7 +181,11 @@ export const make = Effect.gen(function* () {
181
};
182
183
const checkHost = (prepared: PreparedConnection) =>
181
- checkDescriptor(prepared.httpBaseUrl, prepared.environmentId).pipe(
184
+ checkDescriptor(
185
+ prepared.httpBaseUrl,
186
+ prepared.environmentId,
187
+ prepared.httpBaseUrl.startsWith("https:"),
188
+ ).pipe(
189
Effect.withSpan("ConnectionDriver.checkHost", {
190
attributes: { "connection.target.kind": prepared.target._tag },
191
}),
packages/client-runtime/src/connection/model.ts
+3
-1
@@ -1,4 +1,4 @@
1
-import { EnvironmentId } from "@t3tools/contracts";
1
+import { EnvironmentId, type ExecutionEnvironmentDescriptor } from "@t3tools/contracts";
2
import { RelayEnvironmentEndpointFailure } from "@t3tools/contracts/relay";
3
import * as Schema from "effect/Schema";
4
@@ -154,6 +154,8 @@ export interface PreparedConnection {
154
readonly socketUrl: string;
155
readonly httpAuthorization: PreparedHttpAuthorization | null;
156
readonly target: ConnectionTarget;
157
+ /** What the server said about itself while this connection was prepared. */
158
+ readonly descriptor?: ExecutionEnvironmentDescriptor;
159
}
160
161
export type SupervisorConnectionPhase =
packages/client-runtime/src/connection/resolver.test.ts
+9
@@ -206,6 +206,15 @@ describe("ConnectionResolver", () => {
206
"ws://127.0.0.1:3777/ws?clientSurface=web&clientDeviceType=desktop&connectionMethod=direct&orchestrationProtocol=2&clientCapabilities=settingsSecretReferences",
207
httpAuthorization: null,
208
target,
209
+ // The shell reads the server's build and shell flags here before the socket opens.
210
+ descriptor: {
211
+ environmentId: ENVIRONMENT_ID,
212
+ label: "Compatible environment",
213
+ platform: { os: "linux", arch: "x64" },
214
+ serverVersion: "0.0.0-test",
215
+ orchestrationProtocolVersion: ORCHESTRATION_PROTOCOL_VERSION,
216
+ capabilities: { repositoryIdentity: true },
217
+ },
218
});
219
}),
220
);
packages/client-runtime/src/connection/resolver.ts
+1
@@ -321,6 +321,7 @@ export const make = Effect.gen(function* () {
321
}
322
return {
323
...prepared,
324
+ descriptor,
325
socketUrl: appendOrchestrationProtocol(prepared.socketUrl),
326
};
327
});
packages/client-runtime/src/connection/routes.test.ts
+31
@@ -16,9 +16,11 @@ import {
16
credentialConnectionId,
17
insertRoute,
18
entryWithRoutes,
19
+ isNearbyAddress,
20
mergeLearnedRoutes,
21
pendingRouteOrder,
22
removedWithRelay,
23
+ routeKeepsItsHost,
24
routesAfterRemoving,
25
sendsTokenInClear,
26
upsertRoute,
@@ -77,6 +79,35 @@ describe("connection routes", () => {
79
expect(connectionRouteLabel(RELAY)).toBe("T3 Connect");
80
});
81
82
+ it("knows which addresses reach the same machine on every network", () => {
83
+ const tailnetIp = direct("ts-ip", "http://100.101.102.103:3773/");
84
+ const profile = Option.getOrThrow(tailnetIp.profile) as BearerConnectionProfile;
85
+ const marked = {
86
+ target: tailnetIp.target,
87
+ profile: Option.some(new BearerConnectionProfile({ ...profile, network: "tailscale" })),
88
+ };
89
+ expect(routeKeepsItsHost(PUBLIC)).toBe(true);
90
+ expect(routeKeepsItsHost(direct("ts-tls", "https://desk.tail1234.ts.net/"))).toBe(true);
91
+ expect(routeKeepsItsHost(direct("lo", "http://127.0.0.1:3773/"))).toBe(true);
92
+ // With Tailscale down, a plain http tailnet address can reach another host:
93
+ // carrier networks use 100.64.0.0/10 too, and nothing proves who answers.
94
+ expect(routeKeepsItsHost(direct("ts", "http://desk.tail1234.ts.net:3773/"))).toBe(false);
95
+ expect(routeKeepsItsHost(marked)).toBe(false);
96
+ expect(routeKeepsItsHost(tailnetIp)).toBe(false);
97
+ expect(routeKeepsItsHost(LAN)).toBe(false);
98
+ expect(routeKeepsItsHost(direct("http", "http://desk.example.com/"))).toBe(false);
99
+ expect(routeKeepsItsHost(RELAY)).toBe(false);
100
+ });
101
+
102
+ it("treats only loopback and LAN addresses as nearby", () => {
103
+ expect(isNearbyAddress("http://127.0.0.1:3773/")).toBe(true);
104
+ expect(isNearbyAddress("http://192.168.1.10:3773/")).toBe(true);
105
+ expect(isNearbyAddress("http://100.101.102.103:3773/")).toBe(false);
106
+ expect(isNearbyAddress("https://desk.tail1234.ts.net/")).toBe(false);
107
+ expect(isNearbyAddress("https://desk.example.com/")).toBe(false);
108
+ expect(isNearbyAddress("not a url")).toBe(false);
109
+ });
110
+
111
it("places a new route after faster kinds and ahead of T3 Connect", () => {
112
expect(insertRoute([RELAY], LAN)).toEqual([LAN, RELAY]);
113
expect(insertRoute([LAN, RELAY], TAILNET)).toEqual([LAN, TAILNET, RELAY]);
packages/client-runtime/src/connection/routes.ts
+32
-6
@@ -81,6 +81,13 @@ function routeHostname(route: ConnectionRoute): string | null {
81
}
82
}
83
84
+function directAddressKind(hostname: string, tailscale = false) {
85
+ if (isLocalLoopbackHost(hostname)) return "loopback";
86
+ if (tailscale || isTailnetHost(hostname)) return "tailnet";
87
+ if (isSharedAddressSpaceHost(hostname)) return "vpn";
88
+ return isPrivateNetworkHost(hostname) ? "lan" : "public";
89
+}
90
+
91
export function connectionRouteKind(route: ConnectionRoute): ConnectionRouteKind {
92
switch (route.target._tag) {
93
case "RelayConnectionTarget":
@@ -91,17 +98,36 @@ export function connectionRouteKind(route: ConnectionRoute): ConnectionRouteKind
98
case "BearerConnectionTarget": {
99
const hostname = routeHostname(route);
100
if (hostname === null) return "public";
94
- if (isLocalLoopbackHost(hostname)) return "loopback";
101
const profile = Option.getOrNull(route.profile);
96
- const tailscale =
97
- profile?._tag === "BearerConnectionProfile" && profile.network === "tailscale";
98
- if (tailscale || isTailnetHost(hostname)) return "tailnet";
99
- if (isSharedAddressSpaceHost(hostname)) return "vpn";
100
- return isPrivateNetworkHost(hostname) ? "lan" : "public";
102
+ return directAddressKind(
103
+ hostname,
104
+ profile?._tag === "BearerConnectionProfile" && profile.network === "tailscale",
105
+ );
106
}
107
}
108
}
109
110
+/**
111
+ * Whether the route's address reaches the same machine on every network: TLS
112
+ * proves who answers, and loopback cannot move. Any other plain http address,
113
+ * a tailnet one too once Tailscale is down, can belong to another machine after
114
+ * a network change, so an answer from it is not kept.
115
+ */
116
+export function routeKeepsItsHost(route: ConnectionRoute): boolean {
117
+ if (routeHttpBaseUrl(route)?.startsWith("https:") === true) return true;
118
+ return connectionRouteKind(route) === "loopback";
119
+}
120
+
121
+/** A loopback or LAN host answers within a few seconds or is not there. */
122
+export function isNearbyAddress(httpBaseUrl: string): boolean {
123
+ try {
124
+ const kind = directAddressKind(new URL(httpBaseUrl).hostname);
125
+ return kind === "loopback" || kind === "lan";
126
+ } catch {
127
+ return false;
128
+ }
129
+}
130
+
131
const ROUTE_KIND_RANK: Record<ConnectionRouteKind, number> = {
132
loopback: 0,
133
lan: 1,
packages/client-runtime/src/state/environmentHttpAuth.test.ts
+5
-1
@@ -223,7 +223,11 @@ const LOADERS: ReadonlyArray<{
223
expected: SHELL,
224
load: (input: HttpInput) =>
225
fetchEnvironmentShellSnapshot(input).pipe(
226
- Effect.map(({ loadPullRequests: _links, ...snapshot }) => snapshot),
226
+ Effect.map((answer) => {
227
+ if ("resumable" in answer) return answer;
228
+ const { loadPullRequests: _links, ...snapshot } = answer;
229
+ return snapshot;
230
+ }),
231
),
232
},
233
{
packages/client-runtime/src/state/shell-sync.test.ts
+301
-5
@@ -1,5 +1,6 @@
1
import {
2
EnvironmentId,
3
+ type ExecutionEnvironmentDescriptor,
4
ORCHESTRATION_V2_WS_METHODS,
5
type OrchestrationV2ShellSnapshot,
6
type OrchestrationV2ShellStreamItem,
@@ -1151,8 +1152,27 @@ describe("environment shell cold start", () => {
1152
};
1153
const RECENT_ORIGIN = { ...CURRENT_ORIGIN, snapshotAt: NOW - HOUR };
1154
1155
+ // A server that names its shell flags in the descriptor, and the matching live config.
1156
+ const FAMILY_DESCRIPTOR = {
1157
+ serverVersion: "1.0.0",
1158
+ capabilities: { shellThreadFamilies: true, shellOlderSettled: true },
1159
+ } as ExecutionEnvironmentDescriptor;
1160
+ const FAMILY_CONFIG = {
1161
+ shellThreadFamilies: true,
1162
+ shellOlderSettled: true,
1163
+ environment: { serverVersion: "1.0.0" },
1164
+ };
1165
+ const FAMILY_FLAGS = { omitIdleSubagents: true, omitOlderSettled: true };
1166
+
1167
const makeColdStart = Effect.fn("test.makeColdStartShell")(function* (
1168
origin: Persistence.ShellCacheOrigin | undefined,
1169
+ // Set for a connection that is prepared but has no session yet.
1170
+ // Each held load waits for one `releaseLoads`. An "uninterruptible" hold stands for a load
1171
+ // that applies its answer, which an interrupt has to wait for.
1172
+ early?: {
1173
+ readonly serverResumes: boolean;
1174
+ readonly holdLoads?: boolean | "uninterruptible";
1175
+ },
1176
) {
1177
yield* TestClock.setTime(NOW);
1178
const subscriptions = yield* Queue.unbounded<{
@@ -1164,6 +1184,18 @@ describe("environment shell cold start", () => {
1184
readonly origin: Persistence.ShellCacheOrigin | undefined;
1185
}>();
1186
const httpLoads = yield* Ref.make(0);
1187
+ const loadOptions = yield* Queue.unbounded<ShellSnapshotLoader.ShellSnapshotOptions>();
1188
+ const releases = yield* Queue.unbounded<void>();
1189
+ const scope = yield* Effect.scope;
1190
+ const holdLoad =
1191
+ early?.holdLoads === "uninterruptible"
1192
+ ? // Registered after the load's fiber, so the end of a failed test does not wait for it.
1193
+ Scope.addFinalizer(scope, Queue.shutdown(releases)).pipe(
1194
+ Effect.andThen(Effect.uninterruptible(Queue.take(releases))),
1195
+ )
1196
+ : early?.holdLoads === true
1197
+ ? Queue.take(releases)
1198
+ : Effect.void;
1199
const client = {
1200
[ORCHESTRATION_V2_WS_METHODS.subscribeShell]: (input: unknown) =>
1201
Stream.unwrap(
@@ -1174,12 +1206,17 @@ describe("environment shell cold start", () => {
1206
}),
1207
),
1208
} as unknown as WsRpcProtocolClient;
1177
- const activeSession = yield* SubscriptionRef.make(Option.some(session(client)));
1209
+ const activeSession = yield* SubscriptionRef.make(
1210
+ early === undefined ? Option.some(session(client)) : Option.none<RpcSession.RpcSession>(),
1211
+ );
1212
+ const prepared = yield* SubscriptionRef.make(
1213
+ Option.some(early === undefined ? PREPARED : { ...PREPARED, descriptor: FAMILY_DESCRIPTOR }),
1214
+ );
1215
const supervisor = EnvironmentSupervisor.EnvironmentSupervisor.of({
1216
target: TARGET,
1217
state: yield* SubscriptionRef.make(AVAILABLE_CONNECTION_STATE),
1218
session: activeSession,
1182
- prepared: yield* SubscriptionRef.make(Option.some(PREPARED)),
1219
+ prepared,
1220
connect: Effect.void,
1221
disconnect: Effect.void,
1222
retryNow: Effect.void,
@@ -1205,10 +1242,20 @@ describe("environment shell cold start", () => {
1242
clear: () => Effect.void,
1243
});
1244
const snapshotLoader = ShellSnapshotLoader.ShellSnapshotLoader.of({
1208
- load: () =>
1209
- Ref.updateAndGet(httpLoads, (count) => count + 1).pipe(
1245
+ load: (_prepared, options = {}) =>
1246
+ Queue.offer(loadOptions, options).pipe(
1247
+ Effect.andThen(holdLoad),
1248
+ Effect.andThen(Ref.updateAndGet(httpLoads, (count) => count + 1)),
1249
Effect.map((count) =>
1211
- Option.some({ ...v2ShellSnapshot, snapshotSequence: HTTP_SEQUENCE + count - 1 }),
1250
+ Option.some<ShellSnapshotLoader.ShellHttpAnswer>(
1251
+ early?.serverResumes === true && options.resumeFrom !== undefined
1252
+ ? { resumable: true }
1253
+ : {
1254
+ ...v2ShellSnapshot,
1255
+ snapshotSequence: HTTP_SEQUENCE + count - 1,
1256
+ projectIdentities: `identities-${count}`,
1257
+ },
1258
+ ),
1259
),
1260
),
1261
});
@@ -1217,11 +1264,45 @@ describe("environment shell cold start", () => {
1264
Effect.provideService(Persistence.EnvironmentCacheStore, cache),
1265
Effect.provideService(ShellSnapshotLoader.ShellSnapshotLoader, snapshotLoader),
1266
);
1267
+ const nextConnection = Effect.suspend(() =>
1268
+ SubscriptionRef.set(
1269
+ prepared,
1270
+ Option.some({ ...PREPARED, descriptor: { ...FAMILY_DESCRIPTOR } }),
1271
+ ),
1272
+ );
1273
return {
1274
+ shellState,
1275
subscriptions,
1276
saved,
1277
httpLoads,
1278
+ loadOptions,
1279
+ releaseLoads: Queue.offer(releases, undefined),
1280
+ nextConnection,
1281
+ // A connection attempt that ended, then the next one.
1282
+ prepareAgain: SubscriptionRef.set(prepared, Option.none()).pipe(
1283
+ Effect.andThen(nextConnection),
1284
+ ),
1285
+ // Resolves `configRead` when the subscription reads the config, just before it claims the load.
1286
+ connect: (config: object, configRead?: Deferred.Deferred<void>) =>
1287
+ SubscriptionRef.set(
1288
+ activeSession,
1289
+ Option.some({
1290
+ ...session(client),
1291
+ initialConfig: (configRead === undefined
1292
+ ? Effect.void
1293
+ : Deferred.succeed(configRead, undefined)
1294
+ ).pipe(Effect.as(config as never)),
1295
+ }),
1296
+ ),
1297
reconnect: SubscriptionRef.set(activeSession, Option.some(session(client))),
1298
+ // The registry swaps a rotated bearer into the prepared connection of a live session.
1299
+ rotateCredential: SubscriptionRef.update(
1300
+ prepared,
1301
+ Option.map((current) => ({
1302
+ ...current,
1303
+ httpAuthorization: { _tag: "Bearer" as const, token: "rotated" },
1304
+ })),
1305
+ ),
1306
liveAt: (sequence: number) =>
1307
SubscriptionRef.changes(shellState).pipe(
1308
Stream.filter(
@@ -1284,6 +1365,221 @@ describe("environment shell cold start", () => {
1365
}),
1366
);
1367
1368
+ it.effect.each([
1369
+ {
1370
+ label: "no rows to resume",
1371
+ origin: undefined,
1372
+ serverResumes: false,
1373
+ config: FAMILY_CONFIG,
1374
+ loads: [FAMILY_FLAGS],
1375
+ subscribes: { afterSequence: HTTP_SEQUENCE, knownProjectIdentities: "identities-1" },
1376
+ },
1377
+ {
1378
+ label: "rows the server can replay from",
1379
+ origin: { ...RECENT_ORIGIN, ...FAMILY_FLAGS },
1380
+ serverResumes: true,
1381
+ config: FAMILY_CONFIG,
1382
+ loads: [{ ...FAMILY_FLAGS, resumeFrom: CACHED_SEQUENCE }],
1383
+ subscribes: { afterSequence: CACHED_SEQUENCE, httpSnapshotFallback: true },
1384
+ },
1385
+ {
1386
+ label: "rows the server cannot replay from",
1387
+ origin: { ...RECENT_ORIGIN, ...FAMILY_FLAGS },
1388
+ serverResumes: false,
1389
+ config: FAMILY_CONFIG,
1390
+ loads: [{ ...FAMILY_FLAGS, resumeFrom: CACHED_SEQUENCE }],
1391
+ subscribes: { afterSequence: HTTP_SEQUENCE, knownProjectIdentities: "identities-1" },
1392
+ },
1393
+ {
1394
+ // The server answered that it can replay, then the rows passed their age limit.
1395
+ label: "rows that expire before the socket is ready",
1396
+ origin: { ...RECENT_ORIGIN, ...FAMILY_FLAGS, snapshotAt: NOW - 12 * HOUR + 1 },
1397
+ serverResumes: true,
1398
+ config: FAMILY_CONFIG,
1399
+ loads: [{ ...FAMILY_FLAGS, resumeFrom: CACHED_SEQUENCE }],
1400
+ subscribes: {},
1401
+ expires: true,
1402
+ },
1403
+ {
1404
+ // The rows loaded with the descriptor's omissions are replaced, never kept.
1405
+ label: "a live config with other omissions",
1406
+ origin: undefined,
1407
+ serverResumes: false,
1408
+ config: { environment: { serverVersion: "1.0.0" } },
1409
+ loads: [FAMILY_FLAGS, { omitIdleSubagents: false, omitOlderSettled: false }],
1410
+ subscribes: { afterSequence: HTTP_SEQUENCE + 1, knownProjectIdentities: "identities-2" },
1411
+ },
1412
+ ])(
1413
+ "asks for the shell once the connection is prepared, with $label",
1414
+ ({ origin, serverResumes, config, loads, subscribes, expires }) =>
1415
+ Effect.gen(function* () {
1416
+ const h = yield* makeColdStart(origin, { serverResumes });
1417
+ expect(yield* Queue.take(h.loadOptions)).toEqual(loads[0]);
1418
+ if (!serverResumes) {
1419
+ // The rows are on screen before a socket exists, and are not live yet.
1420
+ const early = yield* SubscriptionRef.changes(h.shellState).pipe(
1421
+ Stream.filter((state) =>
1422
+ Option.exists(
1423
+ state.snapshot,
1424
+ (snapshot) => snapshot.snapshotSequence === HTTP_SEQUENCE,
1425
+ ),
1426
+ ),
1427
+ Stream.runHead,
1428
+ Effect.map(Option.getOrThrow),
1429
+ );
1430
+ expect(early.status).not.toBe("live");
1431
+ expect(early.generation).toBeGreaterThan(0);
1432
+ }
1433
+ expect(yield* Queue.size(h.subscriptions)).toBe(0);
1434
+
1435
+ if (expires) yield* TestClock.adjust("1 millis");
1436
+ yield* h.connect(config);
1437
+ const first = yield* Queue.take(h.subscriptions);
1438
+ expect(first.input).toMatchObject(subscribes);
1439
+ for (const key of [
1440
+ "afterSequence",
1441
+ "httpSnapshotFallback",
1442
+ "knownProjectIdentities",
1443
+ ] as const) {
1444
+ if (!(key in subscribes)) expect(first.input).not.toHaveProperty(key);
1445
+ }
1446
+ expect(yield* Queue.clear(h.loadOptions)).toEqual(loads.slice(1));
1447
+ expect(yield* Ref.get(h.httpLoads)).toBe(loads.length);
1448
+ // The socket sends the full snapshot these rows asked for; the rest needs a cursor.
1449
+ if (!("afterSequence" in subscribes)) return;
1450
+
1451
+ yield* Queue.offer(first.events, { kind: "synchronized" });
1452
+ yield* h.liveAt(subscribes.afterSequence);
1453
+ if (loads.length > 1) {
1454
+ yield* TestClock.adjust("500 millis");
1455
+ const saves = yield* Queue.takeAll(h.saved);
1456
+ expect(saves.at(-1)).toMatchObject({
1457
+ snapshot: { snapshotSequence: subscribes.afterSequence },
1458
+ origin: { ...CURRENT_ORIGIN, snapshotAt: NOW },
1459
+ });
1460
+ }
1461
+
1462
+ // A rotated credential keeps the session, so its events still apply and nothing reloads.
1463
+ yield* h.rotateCredential;
1464
+ yield* Queue.offer(first.events, {
1465
+ kind: "thread.removed",
1466
+ sequence: subscribes.afterSequence + 1,
1467
+ location: "active",
1468
+ threadId: ThreadId.make("no-such-thread"),
1469
+ });
1470
+ yield* h.liveAt(subscribes.afterSequence + 1);
1471
+ expect(yield* Ref.get(h.httpLoads)).toBe(loads.length);
1472
+ }),
1473
+ );
1474
+
1475
+ it.effect("a replaced session waits for the load its connection already started", () =>
1476
+ Effect.gen(function* () {
1477
+ const h = yield* makeColdStart(undefined, {
1478
+ serverResumes: false,
1479
+ holdLoads: "uninterruptible",
1480
+ });
1481
+ expect(yield* Queue.take(h.loadOptions)).toEqual(FAMILY_FLAGS);
1482
+ const claimed = yield* Deferred.make<void>();
1483
+ yield* h.connect(FAMILY_CONFIG, claimed);
1484
+ yield* Deferred.await(claimed);
1485
+ // Lets the first subscription run on from its config read to its wait for the load.
1486
+ yield* TestClock.adjust("1 millis");
1487
+ // The first subscription waits for the load. A second request could answer first, and
1488
+ // the older answer would then replace its rows under a cursor the socket has passed.
1489
+ yield* h.connect(FAMILY_CONFIG);
1490
+ yield* TestClock.adjust("1 millis");
1491
+ expect(yield* Queue.size(h.loadOptions)).toBe(0);
1492
+
1493
+ // The connection changes twice while the first load ends. The next session reads the
1494
+ // last connection before its load exists, so it replaces the load of the one between.
1495
+ yield* h.nextConnection;
1496
+ yield* TestClock.adjust("1 millis");
1497
+ yield* h.nextConnection;
1498
+ yield* h.connect(FAMILY_CONFIG);
1499
+ yield* TestClock.adjust("1 millis");
1500
+ yield* h.releaseLoads;
1501
+ expect(yield* Queue.take(h.loadOptions)).toEqual(FAMILY_FLAGS);
1502
+ yield* TestClock.adjust("1 millis");
1503
+ // That session is replaced while it waits for the load between to end. The load it then
1504
+ // starts must stay the one load of the last connection.
1505
+ yield* h.connect(FAMILY_CONFIG);
1506
+ yield* TestClock.adjust("1 millis");
1507
+ yield* h.releaseLoads;
1508
+ expect(yield* Queue.take(h.loadOptions)).toEqual(FAMILY_FLAGS);
1509
+ yield* TestClock.adjust("1 millis");
1510
+ expect(yield* Queue.size(h.loadOptions)).toBe(0);
1511
+
1512
+ yield* h.releaseLoads;
1513
+ const subscribed = yield* Queue.take(h.subscriptions);
1514
+ expect(subscribed.input).toMatchObject({ afterSequence: HTTP_SEQUENCE });
1515
+ expect(yield* Queue.size(h.loadOptions)).toBe(0);
1516
+ expect(yield* Ref.get(h.httpLoads)).toBe(1);
1517
+ }),
1518
+ );
1519
+
1520
+ it.effect("resumes held rows on the socket when the HTTP answer does not come", () =>
1521
+ Effect.gen(function* () {
1522
+ const h = yield* makeColdStart(
1523
+ { ...RECENT_ORIGIN, ...FAMILY_FLAGS },
1524
+ { serverResumes: false, holdLoads: true },
1525
+ );
1526
+ yield* Queue.take(h.loadOptions);
1527
+ const claimed = yield* Deferred.make<void>();
1528
+ yield* h.connect(FAMILY_CONFIG, claimed);
1529
+ yield* Deferred.await(claimed);
1530
+ // Lets the subscription run on from its config read to its wait for the load.
1531
+ yield* TestClock.adjust("1 millis");
1532
+ expect(yield* Queue.size(h.subscriptions)).toBe(0);
1533
+
1534
+ yield* TestClock.adjust("3 seconds");
1535
+ const first = yield* Queue.take(h.subscriptions);
1536
+ expect(first.input).toMatchObject({
1537
+ afterSequence: CACHED_SEQUENCE,
1538
+ httpSnapshotFallback: true,
1539
+ });
1540
+ yield* Queue.offer(first.events, {
1541
+ kind: "thread.removed",
1542
+ sequence: HTTP_SEQUENCE + 1,
1543
+ location: "active",
1544
+ threadId: ThreadId.make("no-such-thread"),
1545
+ });
1546
+ yield* Queue.offer(first.events, { kind: "synchronized" });
1547
+ yield* h.liveAt(HTTP_SEQUENCE + 1);
1548
+
1549
+ // The late answer is an older snapshot. It must not replace the rows the socket moved on.
1550
+ yield* h.releaseLoads;
1551
+ yield* TestClock.adjust("1 millis");
1552
+ expect(yield* Ref.get(h.httpLoads)).toBe(0);
1553
+ expect(yield* SubscriptionRef.get(h.shellState)).toMatchObject({
1554
+ status: "live",
1555
+ snapshot: Option.some({ snapshotSequence: HTTP_SEQUENCE + 1 }),
1556
+ });
1557
+ }),
1558
+ );
1559
+
1560
+ it.effect("drops the load of a connection that ended and waits for the next one", () =>
1561
+ Effect.gen(function* () {
1562
+ const h = yield* makeColdStart(undefined, { serverResumes: false, holdLoads: true });
1563
+ yield* Queue.take(h.loadOptions);
1564
+ yield* h.prepareAgain;
1565
+ yield* Queue.take(h.loadOptions);
1566
+
1567
+ // The socket is ready while the second load is in flight.
1568
+ yield* h.connect(FAMILY_CONFIG);
1569
+ yield* h.releaseLoads;
1570
+ const first = yield* Queue.take(h.subscriptions);
1571
+ expect(first.input).toMatchObject({
1572
+ afterSequence: HTTP_SEQUENCE,
1573
+ knownProjectIdentities: "identities-1",
1574
+ });
1575
+ // The first load ended with its connection and never answered.
1576
+ expect(yield* Ref.get(h.httpLoads)).toBe(1);
1577
+
1578
+ yield* Queue.offer(first.events, { kind: "synchronized" });
1579
+ yield* h.liveAt(HTTP_SEQUENCE);
1580
+ }),
1581
+ );
1582
+
1583
it.effect("reloads over HTTP when the server answers a resume with snapshot-required", () =>
1584
Effect.gen(function* () {
1585
const h = yield* makeColdStart(RECENT_ORIGIN);
packages/client-runtime/src/state/shell.ts
+228
-64
@@ -1,6 +1,7 @@
1
import {
2
ORCHESTRATION_V2_WS_METHODS,
3
type EnvironmentId,
4
+ type ExecutionEnvironmentDescriptor,
5
type OrchestrationV2ShellSnapshot,
6
type OrchestrationV2ShellStreamItem,
7
type OrchestrationV2ThreadShell,
@@ -11,6 +12,7 @@ import {
12
import * as Cause from "effect/Cause";
13
import * as Clock from "effect/Clock";
14
import * as Effect from "effect/Effect";
15
+import * as Exit from "effect/Exit";
16
import * as Fiber from "effect/Fiber";
17
import * as Option from "effect/Option";
18
import * as Queue from "effect/Queue";
@@ -18,10 +20,11 @@ import * as Ref from "effect/Ref";
20
import * as Semaphore from "effect/Semaphore";
21
import * as Stream from "effect/Stream";
22
import * as SubscriptionRef from "effect/SubscriptionRef";
23
+import * as SynchronizedRef from "effect/SynchronizedRef";
24
import { AsyncResult, Atom } from "effect/reactivity";
25
26
import * as EnvironmentRegistry from "../connection/registry.ts";
24
-import { connectionProjectionPhase } from "../connection/model.ts";
27
+import { connectionProjectionPhase, type PreparedConnection } from "../connection/model.ts";
28
import * as EnvironmentSupervisor from "../connection/supervisor.ts";
29
import * as ConnectionWakeups from "../connection/wakeups.ts";
30
import { safeErrorLogAttributes } from "../errors/safeLog.ts";
@@ -89,6 +92,11 @@ const SHELL_SYNCHRONIZATION_ERROR_MESSAGE = "Could not synchronize environment d
92
*/
93
const CACHE_RESUME_MAX_AGE_MILLIS = 12 * 60 * 60 * 1000;
94
95
+// How long a ready socket waits for the early HTTP answer before it resumes held rows itself.
96
+// The supervisor's quick probe allows a live host the same time. A proxy that drops `/api`
97
+// without an answer would otherwise hold each connect for the full HTTP timeout.
98
+const EARLY_LOAD_GRACE = "3 seconds";
99
+
100
type ShellOriginIdentity = Omit<Persistence.ShellCacheOrigin, "snapshotAt">;
101
102
interface ShellSubscription {
@@ -97,6 +105,42 @@ interface ShellSubscription {
105
readonly identity: ShellOriginIdentity | null;
106
}
107
108
+function shellIdentity(
109
+ environment: Pick<ExecutionEnvironmentDescriptor, "serverVersion" | "databaseMigration">,
110
+ flags: { readonly shellThreadFamilies?: boolean; readonly shellOlderSettled?: boolean },
111
+): ShellOriginIdentity {
112
+ const omitIdleSubagents = flags.shellThreadFamilies === true;
113
+ return {
114
+ serverVersion: environment.serverVersion,
115
+ // Migrations rewrite projections without events, so a migrated server reloads.
116
+ databaseMigration: environment.databaseMigration ?? null,
117
+ omitIdleSubagents,
118
+ omitOlderSettled: flags.shellOlderSettled === true && omitIdleSubagents,
119
+ };
120
+}
121
+
122
+function sameConnection(left: PreparedConnection, right: PreparedConnection): boolean {
123
+ const keys = Object.keys({ ...left, ...right }) as Array<keyof PreparedConnection>;
124
+ return keys.every((key) => key === "httpAuthorization" || left[key] === right[key]);
125
+}
126
+
127
+/** Null for a server whose descriptor does not name its shell flags; only its config does. */
128
+function descriptorShellIdentity(
129
+ descriptor: ExecutionEnvironmentDescriptor | undefined,
130
+): ShellOriginIdentity | null {
131
+ if (descriptor?.capabilities.shellThreadFamilies === undefined) return null;
132
+ return shellIdentity(descriptor, descriptor.capabilities);
133
+}
134
+
135
+function sameShellIdentity(left: ShellOriginIdentity, right: ShellOriginIdentity): boolean {
136
+ return (
137
+ left.serverVersion === right.serverVersion &&
138
+ left.databaseMigration === right.databaseMigration &&
139
+ left.omitIdleSubagents === right.omitIdleSubagents &&
140
+ left.omitOlderSettled === right.omitOlderSettled
141
+ );
142
+}
143
+
144
function canResumeFrom(
145
origin: Persistence.ShellCacheOrigin | undefined,
146
identity: ShellOriginIdentity | null,
@@ -104,14 +148,21 @@ function canResumeFrom(
148
): boolean {
149
if (origin === undefined || identity === null) return false;
150
const age = now - origin.snapshotAt;
107
- return (
108
- origin.serverVersion === identity.serverVersion &&
109
- origin.databaseMigration === identity.databaseMigration &&
110
- origin.omitIdleSubagents === identity.omitIdleSubagents &&
111
- origin.omitOlderSettled === identity.omitOlderSettled &&
112
- age >= 0 &&
113
- age < CACHE_RESUME_MAX_AGE_MILLIS
114
- );
151
+ return sameShellIdentity(origin, identity) && age >= 0 && age < CACHE_RESUME_MAX_AGE_MILLIS;
152
+}
153
+
154
+type HttpShellOutcome =
155
+ | { readonly kind: "snapshot"; readonly projectIdentities: string | undefined }
156
+ /** The server can replay from the rows the client holds. */
157
+ | { readonly kind: "resumable" }
158
+ | { readonly kind: "failed" };
159
+
160
+interface EarlyShellLoad {
161
+ readonly prepared: PreparedConnection;
162
+ readonly identity: ShellOriginIdentity;
163
+ readonly fiber: Fiber.Fiber<HttpShellOutcome>;
164
+ // One subscription uses a load; a later one on the same connection decides again.
165
+ readonly claimed: boolean;
166
}
167
168
interface LiveShell {
@@ -249,17 +300,17 @@ export const makeEnvironmentShellState = Effect.fn("EnvironmentShellState.make")
300
// time, so neither can overwrite what the other wrote in between.
301
const snapshotWriteLock = yield* Semaphore.make(1);
302
252
- const applyItems = Effect.fn("EnvironmentShellState.applyItems")(function* (
253
- entries: ReadonlyArray<readonly [RpcSession, OrchestrationV2ShellStreamItem]>,
303
+ const applyLocked = Effect.fnUntraced(function* (
304
+ items: ReadonlyArray<OrchestrationV2ShellStreamItem>,
305
+ /** What an authoritative snapshot among the items descends from. */
306
+ identity: ShellOriginIdentity | null,
307
+ /** Set for an HTTP snapshot: the rows that still wait for their pull request links. */
308
pendingLinkIds?: ReadonlySet<ThreadId>,
309
) {
256
- // Read under the lock: items buffered across a session change must not land on the
257
- // rows the next session loaded.
258
- const subscription = yield* Ref.get(activeSubscription);
259
- const items = entries.flatMap(([session, item]) =>
260
- session === subscription?.session ? [item] : [],
261
- );
310
const initial = yield* SubscriptionRef.get(state);
311
+ // No socket has confirmed an HTTP snapshot, so it never makes the shell live.
312
+ const fromHttp = pendingLinkIds !== undefined;
313
+ const disconnected = initial.status === "empty" || initial.status === "cached";
314
let waiting = yield* Ref.get(awaitingCompletion);
315
let next = initial;
316
let receivedAuthoritativeSnapshot = false;
@@ -330,7 +381,8 @@ export const makeEnvironmentShellState = Effect.fn("EnvironmentShellState.make")
381
if (authoritative) receivedAuthoritativeSnapshot = true;
382
next = {
383
snapshot: Option.some(nextSnapshot),
333
- status: waiting ? "synchronizing" : "live",
384
+ status:
385
+ fromHttp && disconnected ? "cached" : waiting || fromHttp ? "synchronizing" : "live",
386
error: Option.none(),
387
generation: authoritative ? ++lastShellGeneration : next.generation,
388
removedAt: authoritative ? EMPTY_REMOVALS : next.removedAt,
@@ -347,9 +399,7 @@ export const makeEnvironmentShellState = Effect.fn("EnvironmentShellState.make")
399
yield* Queue.offer(reloadRequests, undefined);
400
} else if (receivedAuthoritativeSnapshot) {
401
nextOrigin =
350
- subscription?.identity == null
351
- ? undefined
352
- : { ...subscription.identity, snapshotAt: yield* Clock.currentTimeMillis };
402
+ identity === null ? undefined : { ...identity, snapshotAt: yield* Clock.currentTimeMillis };
403
yield* Ref.set(origin, nextOrigin);
404
}
405
if (next === initial) return;
@@ -368,6 +418,18 @@ export const makeEnvironmentShellState = Effect.fn("EnvironmentShellState.make")
418
if (next.snapshot !== initial.snapshot && Option.isSome(next.snapshot)) {
419
yield* Queue.offer(persistence, next.snapshot.value);
420
}
421
+ });
422
+
423
+ const applyItems = Effect.fn("EnvironmentShellState.applyItems")(function* (
424
+ entries: ReadonlyArray<readonly [RpcSession, OrchestrationV2ShellStreamItem]>,
425
+ ) {
426
+ // Read under the lock: items buffered across a session change must not land on the
427
+ // rows the next session loaded.
428
+ const subscription = yield* Ref.get(activeSubscription);
429
+ yield* applyLocked(
430
+ entries.flatMap(([session, item]) => (session === subscription?.session ? [item] : [])),
431
+ subscription?.identity ?? null,
432
+ );
433
}, snapshotWriteLock.withPermit);
434
435
const stateScope = yield* Effect.scope;
@@ -437,6 +499,120 @@ export const makeEnvironmentShellState = Effect.fn("EnvironmentShellState.make")
499
yield* fillDeferredPullRequests(cached.value, cachedSnapshot.value);
500
}
501
502
+ const applyHttpSnapshot = Effect.fnUntraced(function* (
503
+ loaded: ShellSnapshotLoader.ShellHttpSnapshot,
504
+ identity: ShellOriginIdentity | null,
505
+ ) {
506
+ const { projectIdentities: _identities, ...deferred } = loaded;
507
+ yield* cancelActiveFill;
508
+ const previous = yield* SubscriptionRef.get(state);
509
+ const snapshot = reuseUnchangedThreadShells(
510
+ Option.getOrNull(previous.snapshot),
511
+ withoutDeferredPullRequests(deferred),
512
+ );
513
+ const pendingLinkIds = new Set(
514
+ deferred.loadPullRequests === undefined ? [] : deferred.threads.map((thread) => thread.id),
515
+ );
516
+ yield* snapshotWriteLock.withPermit(
517
+ applyLocked([{ kind: "snapshot", snapshot }], identity, pendingLinkIds),
518
+ );
519
+ const current = yield* SubscriptionRef.get(state);
520
+ if (Option.isSome(current.snapshot)) {
521
+ yield* fillDeferredPullRequests(deferred, current.snapshot.value);
522
+ }
523
+ }, Effect.uninterruptible);
524
+
525
+ /**
526
+ * One request to the HTTP shell route. Held rows that may resume are named, so the server
527
+ * answers that they still can, in a few bytes, or sends the snapshot that replaces them.
528
+ */
529
+ const loadOverHttp = Effect.fn("EnvironmentShellState.loadOverHttp")(function* (
530
+ prepared: PreparedConnection,
531
+ identity: ShellOriginIdentity | null,
532
+ ) {
533
+ const held = yield* SubscriptionRef.get(state);
534
+ const resumeFrom =
535
+ Option.isSome(held.snapshot) &&
536
+ canResumeFrom(yield* Ref.get(origin), identity, yield* Clock.currentTimeMillis)
537
+ ? held.snapshot.value.snapshotSequence
538
+ : undefined;
539
+ // HTTP and the socket must agree, or omitted children would return through the socket.
540
+ const answer = yield* snapshotLoader.load(prepared, {
541
+ omitIdleSubagents: identity?.omitIdleSubagents === true,
542
+ omitOlderSettled: identity?.omitOlderSettled === true,
543
+ ...(resumeFrom === undefined ? {} : { resumeFrom }),
544
+ });
545
+ if (Option.isNone(answer)) return { kind: "failed" } satisfies HttpShellOutcome;
546
+ if ("resumable" in answer.value) return { kind: "resumable" } satisfies HttpShellOutcome;
547
+ yield* applyHttpSnapshot(answer.value, identity);
548
+ return {
549
+ kind: "snapshot",
550
+ projectIdentities: answer.value.projectIdentities,
551
+ } satisfies HttpShellOutcome;
552
+ });
553
+
554
+ // A server whose descriptor names its shell flags is asked for the shell as soon as the
555
+ // connection is prepared, while the socket opens and the config downloads. The subscription
556
+ // claims that load, and reloads when the live config disagrees with the descriptor.
557
+ const earlyLoad = yield* SynchronizedRef.make<EarlyShellLoad | null>(null);
558
+ const useEarlyLoad = (prepared: PreparedConnection | null, claim: boolean) =>
559
+ SynchronizedRef.modifyEffect(
560
+ earlyLoad,
561
+ Effect.fnUntraced(function* (current) {
562
+ // A rotated credential replaces the prepared connection of a session that stays up.
563
+ if (current !== null && prepared !== null && sameConnection(current.prepared, prepared)) {
564
+ const next = { ...current, prepared, claimed: current.claimed || claim };
565
+ // A claimed load that still runs is handed out again. A second request could
566
+ // answer first, and this one would then replace its rows under a passed cursor.
567
+ const done = current.fiber.pollUnsafe() !== undefined;
568
+ return [current.claimed && done ? null : current, next] as const;
569
+ }
570
+ if (current !== null) yield* Fiber.interrupt(current.fiber);
571
+ const identity = descriptorShellIdentity(prepared?.descriptor);
572
+ if (prepared === null || identity === null) return [null, null] as const;
573
+ // Items a closed session still has in flight must not land on the rows this loads.
574
+ yield* Ref.set(activeSubscription, null);
575
+ const fiber = yield* loadOverHttp(prepared, identity).pipe(Effect.forkIn(stateScope));
576
+ const next: EarlyShellLoad = { prepared, identity, fiber, claimed: claim };
577
+ return [next, next] as const;
578
+ }),
579
+ ).pipe(
580
+ // Covers the store too. A cancel that lands during the wait above would otherwise take
581
+ // effect before the store, and the load just started would run on with no record.
582
+ Effect.uninterruptible,
583
+ );
584
+ /**
585
+ * The subscription waits for the early load, so its answer is applied before the socket
586
+ * moves the rows on. Held rows that can resume wait a limited time only: after that the
587
+ * load is ended, so a late answer applies nothing, and the socket resumes them.
588
+ */
589
+ const awaitEarlyLoad = Effect.fnUntraced(function* (
590
+ early: EarlyShellLoad,
591
+ identity: ShellOriginIdentity | null,
592
+ ) {
593
+ const resumable =
594
+ Option.isSome((yield* SubscriptionRef.get(state)).snapshot) &&
595
+ canResumeFrom(yield* Ref.get(origin), identity, yield* Clock.currentTimeMillis);
596
+ if (!resumable) return yield* Fiber.await(early.fiber);
597
+ const answered = yield* Fiber.await(early.fiber).pipe(Effect.timeoutOption(EARLY_LOAD_GRACE));
598
+ if (Option.isSome(answered)) return answered.value;
599
+ // Returns after an answer that is being applied has finished, so the cursor below is read
600
+ // from rows that no longer change under it.
601
+ yield* Fiber.interrupt(early.fiber);
602
+ return yield* Fiber.await(early.fiber);
603
+ });
604
+
605
+ yield* SubscriptionRef.changes(supervisor.prepared).pipe(
606
+ // Reads the connection of this moment: a change handled late must not start a load for a
607
+ // connection that ended, which would also drop the items of the session that replaced it.
608
+ Stream.runForEach(() =>
609
+ SubscriptionRef.get(supervisor.prepared).pipe(
610
+ Effect.flatMap((prepared) => useEarlyLoad(Option.getOrNull(prepared), false)),
611
+ ),
612
+ ),
613
+ Effect.forkScoped,
614
+ );
615
+
616
const resubscriptions = Stream.merge(
617
Stream.fromQueue(reloadRequests),
618
Option.match(wakeups, {
@@ -452,37 +628,41 @@ export const makeEnvironmentShellState = Effect.fn("EnvironmentShellState.make")
628
ORCHESTRATION_V2_WS_METHODS.subscribeShell,
629
Effect.fn("EnvironmentShellState.makeSubscribeInput")(function* (session) {
630
const identity = yield* session.initialConfig.pipe(
455
- Effect.map((config): ShellOriginIdentity => {
456
- const omitIdleSubagents = config.shellThreadFamilies === true;
457
- return {
458
- serverVersion: config.environment.serverVersion,
459
- // Migrations rewrite projections without events, so a migrated server reloads.
460
- databaseMigration: config.environment.databaseMigration ?? null,
461
- omitIdleSubagents,
462
- omitOlderSettled: config.shellOlderSettled === true && omitIdleSubagents,
463
- };
464
- }),
631
+ Effect.map((config) => shellIdentity(config.environment, config)),
632
Effect.orElseSucceed(() => null),
633
);
634
const { omitIdleSubagents, omitOlderSettled } = identity ?? {
635
omitIdleSubagents: false,
636
omitOlderSettled: false,
637
};
638
+ const early = yield* useEarlyLoad(
639
+ Option.getOrNull(yield* SubscriptionRef.get(supervisor.prepared)),
640
+ true,
641
+ );
642
+ const earlyExit = early === null ? null : yield* awaitEarlyLoad(early, identity);
643
+ // Rows the descriptor's flags or build loaded are not kept when the live config differs:
644
+ // their origin then cannot resume, so the load below replaces them.
645
+ let loaded =
646
+ early !== null &&
647
+ earlyExit !== null &&
648
+ Exit.isSuccess(earlyExit) &&
649
+ identity !== null &&
650
+ sameShellIdentity(early.identity, identity)
651
+ ? earlyExit.value
652
+ : null;
653
yield* Ref.set(activeSubscription, { session, identity });
654
yield* Ref.set(awaitingCompletion, true);
655
yield* setSynchronizing;
656
657
// Rows from a recent server snapshot with the same omissions and server build resume
658
// from their cursor, in memory or from the cache. Anything else reloads the HTTP
477
- // snapshot so a valid cursor cannot preserve stale or incomplete data.
478
- let current = yield* SubscriptionRef.get(state);
479
- let canResume =
480
- Option.isSome(current.snapshot) &&
659
+ // snapshot so a valid cursor cannot preserve stale or incomplete data. A `resumable`
660
+ // answer is checked again too: the rows can pass their age limit while the socket opens.
661
+ const resumingHeldRows =
662
+ loaded?.kind !== "snapshot" &&
663
+ Option.isSome((yield* SubscriptionRef.get(state)).snapshot) &&
664
canResumeFrom(yield* Ref.get(origin), identity, yield* Clock.currentTimeMillis);
482
- // Only rows the client already held may be refused; a cursor from the snapshot just
483
- // loaded falls back to the socket snapshot, so a refusal cannot loop.
484
- const resumingHeldRows = canResume;
485
- if (!canResume) {
665
+ if (!resumingHeldRows && loaded === null) {
666
const prepared = yield* SubscriptionRef.get(supervisor.prepared).pipe(
667
Effect.flatMap(
668
Option.match({
@@ -497,42 +677,26 @@ export const makeEnvironmentShellState = Effect.fn("EnvironmentShellState.make")
677
}),
678
),
679
);
500
- // HTTP and the socket must agree, or omitted children would return through the socket.
501
- const httpSnapshot = yield* snapshotLoader.load(prepared, {
502
- omitIdleSubagents,
503
- omitOlderSettled,
504
- });
505
- if (Option.isSome(httpSnapshot)) {
506
- yield* cancelActiveFill;
507
- const previous = yield* SubscriptionRef.get(state);
508
- const snapshot = reuseUnchangedThreadShells(
509
- Option.getOrNull(previous.snapshot),
510
- withoutDeferredPullRequests(httpSnapshot.value),
511
- );
512
- const pendingLinkIds = new Set(
513
- httpSnapshot.value.loadPullRequests === undefined
514
- ? []
515
- : httpSnapshot.value.threads.map((thread) => thread.id),
516
- );
517
- yield* applyItems([[session, { kind: "snapshot", snapshot }]], pendingLinkIds);
518
- canResume = true;
519
- current = yield* SubscriptionRef.get(state);
520
- if (Option.isSome(current.snapshot)) {
521
- yield* fillDeferredPullRequests(httpSnapshot.value, current.snapshot.value);
522
- }
523
- }
680
+ loaded = yield* loadOverHttp(prepared, identity);
681
}
682
683
+ const current = yield* SubscriptionRef.get(state);
684
const omission = {
685
...(omitIdleSubagents ? { omitIdleSubagents: true as const } : {}),
686
...(omitOlderSettled ? { omitOlderSettled: true as const } : {}),
687
};
530
- if (!canResume || Option.isNone(current.snapshot)) {
688
+ if (Option.isNone(current.snapshot) || !(resumingHeldRows || loaded?.kind === "snapshot")) {
689
return { ...omission, requestCompletionMarker: true as const };
690
}
691
return {
692
afterSequence: current.snapshot.value.snapshotSequence,
693
+ // Only rows the client already held may be refused; a cursor from the snapshot just
694
+ // loaded falls back to the socket snapshot, so a refusal cannot loop.
695
...(resumingHeldRows ? { httpSnapshotFallback: true as const } : {}),
696
+ // No stream item has touched the snapshot just loaded, so its identities still hold.
697
+ ...(loaded?.kind === "snapshot" && loaded.projectIdentities !== undefined
698
+ ? { knownProjectIdentities: loaded.projectIdentities }
699
+ : {}),
700
...omission,
701
requestCompletionMarker: true as const,
702
};
packages/client-runtime/src/state/shellSnapshotHttp.ts
+49
-19
@@ -1,9 +1,10 @@
1
-import { EnvironmentHttpCommonError } from "@t3tools/contracts";
1
+import { EnvironmentHttpCommonError, OrchestrationV2ShellResumable } from "@t3tools/contracts";
2
import * as Cause from "effect/Cause";
3
import * as Context from "effect/Context";
4
import * as Effect from "effect/Effect";
5
import * as Layer from "effect/Layer";
6
import * as Option from "effect/Option";
7
+import * as Predicate from "effect/Predicate";
8
import * as Schema from "effect/Schema";
9
import { HttpClient, HttpClientError, type HttpClientResponse } from "effect/http";
10
@@ -19,6 +20,20 @@ import {
20
type DeferredShellSnapshot,
21
} from "./shellPullRequests.ts";
22
23
+export interface ShellSnapshotOptions {
24
+ readonly omitIdleSubagents?: boolean;
25
+ readonly omitOlderSettled?: boolean;
26
+ /** The cursor of held rows loaded with these omissions; the server may answer `resumable`. */
27
+ readonly resumeFrom?: number;
28
+}
29
+
30
+/** `projectIdentities` names the repository identities the snapshot carried. */
31
+export interface ShellHttpSnapshot extends DeferredShellSnapshot {
32
+ readonly projectIdentities?: string;
33
+}
34
+
35
+export type ShellHttpAnswer = ShellHttpSnapshot | OrchestrationV2ShellResumable;
36
+
37
// On timeout the socket asks the same server for the same full snapshot, so a
38
// short deadline only throws the first build away. The socket fallback is for
39
// setups where /api fails but /ws works, such as a proxy that blocks /api.
@@ -26,20 +41,21 @@ const DEFAULT_SHELL_SNAPSHOT_TIMEOUT_MS = 20_000;
41
42
export const fetchEnvironmentShellSnapshot = Effect.fn(
43
"clientRuntime.state.fetchEnvironmentShellSnapshot",
29
-)(function* (input: {
30
- readonly prepared: PreparedConnection;
31
- readonly signer: Option.Option<ManagedRelay.ManagedRelayDpopSigner["Service"]>;
32
- readonly remoteAuthorization?: Option.Option<
33
- RemoteEnvironmentAuthorization.RemoteEnvironmentAuthorization["Service"]
34
- >;
35
- readonly timeoutMs?: number;
36
- readonly omitIdleSubagents?: boolean;
37
- readonly omitOlderSettled?: boolean;
38
-}) {
44
+)(function* (
45
+ input: {
46
+ readonly prepared: PreparedConnection;
47
+ readonly signer: Option.Option<ManagedRelay.ManagedRelayDpopSigner["Service"]>;
48
+ readonly remoteAuthorization?: Option.Option<
49
+ RemoteEnvironmentAuthorization.RemoteEnvironmentAuthorization["Service"]
50
+ >;
51
+ readonly timeoutMs?: number;
52
+ } & ShellSnapshotOptions,
53
+) {
54
const endpoint = {
55
query: {
56
...(input.omitIdleSubagents === true ? { omitIdleSubagents: "1" } : {}),
57
...(input.omitOlderSettled === true ? { omitOlderSettled: "1" } : {}),
58
+ ...(input.resumeFrom === undefined ? {} : { resumeFrom: String(input.resumeFrom) }),
59
},
60
};
61
return yield* executeAuthenticatedEnvironmentHttpRequest({
@@ -63,14 +79,29 @@ const decodeEnvironmentHttpCommonError = Schema.decodeUnknownEffect(
79
Schema.toCodecJson(EnvironmentHttpCommonError),
80
);
81
82
+const isShellResumable = Schema.is(OrchestrationV2ShellResumable);
83
+
84
const decodeShellSnapshotResponse = (
85
response: HttpClientResponse.HttpClientResponse,
86
): Effect.Effect<
69
- DeferredShellSnapshot,
87
+ ShellHttpAnswer,
88
HttpClientError.HttpClientError | Schema.SchemaError | EnvironmentHttpCommonError
89
> => {
90
if (response.status === 200) {
73
- return response.json.pipe(Effect.flatMap(decodeShellSnapshotDeferringPullRequests));
91
+ return response.json.pipe(
92
+ Effect.flatMap((json): Effect.Effect<ShellHttpAnswer, Schema.SchemaError> => {
93
+ if (isShellResumable(json)) return Effect.succeed(json);
94
+ const projectIdentities =
95
+ Predicate.isObject(json) && typeof json.projectIdentities === "string"
96
+ ? json.projectIdentities
97
+ : undefined;
98
+ return decodeShellSnapshotDeferringPullRequests(json).pipe(
99
+ Effect.map((snapshot) =>
100
+ projectIdentities === undefined ? snapshot : { ...snapshot, projectIdentities },
101
+ ),
102
+ );
103
+ }),
104
+ );
105
}
106
const statusError = new HttpClientError.HttpClientError({
107
reason: new HttpClientError.StatusCodeError({ request: response.request, response }),
@@ -89,8 +120,8 @@ export class ShellSnapshotLoader extends Context.Service<
120
{
121
readonly load: (
122
prepared: PreparedConnection,
92
- options?: { readonly omitIdleSubagents?: boolean; readonly omitOlderSettled?: boolean },
93
- ) => Effect.Effect<Option.Option<DeferredShellSnapshot>>;
123
+ options?: ShellSnapshotOptions,
124
+ ) => Effect.Effect<Option.Option<ShellHttpAnswer>>;
125
}
126
>()("@t3tools/client-runtime/state/shellSnapshotHttp/ShellSnapshotLoader") {}
127
@@ -110,17 +141,16 @@ export const layer: Layer.Layer<ShellSnapshotLoader, never, HttpClient.HttpClien
141
prepared,
142
signer,
143
remoteAuthorization,
113
- ...(options?.omitIdleSubagents === true ? { omitIdleSubagents: true } : {}),
114
- ...(options?.omitOlderSettled === true ? { omitOlderSettled: true } : {}),
144
+ ...options,
145
}).pipe(
116
- Effect.map(Option.some<DeferredShellSnapshot>),
146
+ Effect.map(Option.some<ShellHttpAnswer>),
147
Effect.provideService(HttpClient.HttpClient, httpClient),
148
Effect.catchCause((cause) =>
149
Effect.logWarning(
150
"Could not load the environment shell snapshot over HTTP; using the socket snapshot instead.",
151
).pipe(
152
Effect.annotateLogs({ cause: Cause.pretty(cause) }),
123
- Effect.as(Option.none<DeferredShellSnapshot>()),
153
+ Effect.as(Option.none<ShellHttpAnswer>()),
154
),
155
),
156
);
packages/contracts/src/environment.ts
+6
@@ -184,6 +184,12 @@ export const ExecutionEnvironmentCapabilities = Schema.Struct({
184
/** Server can transcribe 16 kHz mono PCM audio on the environment machine. */
185
voiceTranscription: Schema.optionalKey(Schema.Boolean),
186
cloudVoiceTranscription: Schema.optionalKey(Schema.Boolean),
187
+ /**
188
+ * The server config's flags of the same names. A server that names them here lets a client
189
+ * load the shell snapshot before the socket opens, and answers its `resumeFrom`.
190
+ */
191
+ shellThreadFamilies: Schema.optionalKey(Schema.Boolean),
192
+ shellOlderSettled: Schema.optionalKey(Schema.Boolean),
193
});
194
export type ExecutionEnvironmentCapabilities = typeof ExecutionEnvironmentCapabilities.Type;
195
packages/contracts/src/environmentHttp.ts
+8
-2
@@ -67,7 +67,8 @@ import {
67
} from "./speech.ts";
68
import { SecretReference } from "./secretReference.ts";
69
import {
70
- OrchestrationV2ShellSnapshot,
70
+ OrchestrationV2ShellHttpSnapshot,
71
+ OrchestrationV2ShellResumable,
72
OrchestrationV2ThreadBoundedSnapshot,
73
OrchestrationV2ThreadDetailSnapshot,
74
OrchestrationV2ThreadHistoryPage,
@@ -685,6 +686,11 @@ const EnvironmentOrchestrationThreadSnapshotParams = Schema.Struct({
686
export const EnvironmentOrchestrationShellSnapshotQuery = Schema.Struct({
687
omitIdleSubagents: Schema.optionalKey(Schema.String),
688
omitOlderSettled: Schema.optionalKey(Schema.String),
689
+ /**
690
+ * The cursor of rows the client holds, loaded with the same omissions. The server answers
691
+ * `resumable` instead of a snapshot when the socket can replay from it.
692
+ */
693
+ resumeFrom: Schema.optionalKey(Schema.String),
694
});
695
696
export const EnvironmentOrchestrationThreadBoundedSnapshotQuery = Schema.Struct({
@@ -714,7 +720,7 @@ class EnvironmentOrchestrationHttpApi extends HttpApiGroup.make("orchestration")
720
HttpApiEndpoint.get("shellSnapshot", "/api/orchestration/shell", {
721
headers: OrchestrationProtocolHeaders,
722
query: EnvironmentOrchestrationShellSnapshotQuery,
717
- success: OrchestrationV2ShellSnapshot,
723
+ success: [OrchestrationV2ShellHttpSnapshot, OrchestrationV2ShellResumable],
724
error: EnvironmentOrchestrationSnapshotErrors,
725
}).middleware(EnvironmentAuthenticatedAuth),
726
)
packages/contracts/src/orchestrationV2.ts
+21
@@ -2292,6 +2292,22 @@ export const OrchestrationV2ShellSnapshot = Schema.Struct({
2292
});
2293
export type OrchestrationV2ShellSnapshot = typeof OrchestrationV2ShellSnapshot.Type;
2294
2295
+/** The HTTP shell snapshot. `projectIdentities` names its repository identities; see `knownProjectIdentities`. */
2296
+export const OrchestrationV2ShellHttpSnapshot = Schema.Struct({
2297
+ ...OrchestrationV2ShellSnapshot.fields,
2298
+ projectIdentities: Schema.optionalKey(Schema.String),
2299
+});
2300
+export type OrchestrationV2ShellHttpSnapshot = typeof OrchestrationV2ShellHttpSnapshot.Type;
2301
+
2302
+/**
2303
+ * The HTTP shell route's answer to `resumeFrom` when the socket can replay from that cursor:
2304
+ * the client keeps its rows and subscribes from the cursor.
2305
+ */
2306
+export const OrchestrationV2ShellResumable = Schema.Struct({
2307
+ resumable: Schema.Literal(true),
2308
+});
2309
+export type OrchestrationV2ShellResumable = typeof OrchestrationV2ShellResumable.Type;
2310
+
2311
export const OrchestrationV2ShellStreamItem = Schema.Union([
2312
Schema.Struct({
2313
kind: Schema.Literal("synchronized"),
@@ -3792,6 +3808,11 @@ export const OrchestrationV2SubscribeShellInput = Schema.Struct({
3808
* a socket snapshot, which clients without transport compression download over gzip HTTP.
3809
*/
3810
httpSnapshotFallback: Schema.optionalKey(Schema.Literal(true)),
3811
+ /**
3812
+ * The `projectIdentities` of the HTTP snapshot whose cursor is `afterSequence`, when the client
3813
+ * changed nothing since. A resume that still has the same identities does not resend them.
3814
+ */
3815
+ knownProjectIdentities: Schema.optionalKey(Schema.String),
3816
requestCompletionMarker: Schema.optionalKey(Schema.Boolean),
3817
/**
3818
* Snapshots leave out idle subagent children; read them with `getThreadFamily`. Send only to