@devries-cameron20 / t3code-1 / commits / 962eeefa03
Fork 0

The best way to control Claude Code, Codex and more from any computer or phone

fix(client-runtime): retry messages after socket loss with the same command ID (#24761)

Julius Marminge committed Nov 11, 2026 at 15:47 UTC 962eeefa0392922d32eb54e0bedbbceedd094bef
3 files changed +282 -34
apps/mobile/src/state/threadOutboxDelivery.ts
+2
@@ -911,6 +911,7 @@ export function createThreadOutboxDelivery(): {
911 interactionMode: sendSettings.interactionMode,
912 createdAt: queuedMessage.createdAt,
913 dispatchMode: "queue",
914 + resendAfterSocketLoss: false,
915 },
916 });
917 if (AsyncResult.isFailure(compactionResult) && !hasAccess()) return true;
@@ -951,6 +952,7 @@ export function createThreadOutboxDelivery(): {
952 createdAt: queuedMessage.createdAt,
953 dispatchMode: queuedMessage.dispatchMode ?? "start",
954 ...(queuedMessage.freshContext ? { freshContext: true } : {}),
955 + resendAfterSocketLoss: false,
956 },
957 });
958 if (AsyncResult.isFailure(deliveryResult) && !hasAccess()) return true;
packages/client-runtime/src/operations/commands.test.ts
+190 -1
@@ -20,16 +20,23 @@ import {
20 type ProjectMutation,
21 } from "@t3tools/contracts";
22 import { describe, expect, it } from "@effect/vitest";
23 +import * as Cause from "effect/Cause";
24 import * as SecureRandom from "effect/crypto/SecureRandom";
25 +import * as Deferred from "effect/Deferred";
26 import * as Effect from "effect/Effect";
27 +import * as Exit from "effect/Exit";
28 +import * as Fiber from "effect/Fiber";
29 import * as Layer from "effect/Layer";
30 import * as Option from "effect/Option";
31 import * as SubscriptionRef from "effect/SubscriptionRef";
32 +import * as TestClock from "effect/testing/TestClock";
33 +import { RpcClientError } from "effect/rpc";
34
35 import {
36 AVAILABLE_CONNECTION_STATE,
37 PrimaryConnectionTarget,
38 type PreparedConnection,
39 + type SupervisorConnectionState,
40 } from "../connection/model.ts";
41 import * as EnvironmentSupervisor from "../connection/supervisor.ts";
42 import * as RpcSession from "../rpc/session.ts";
@@ -55,6 +62,12 @@ const TARGET = new PrimaryConnectionTarget({
62 wsBaseUrl: "wss://environment.example.test",
63 });
64
65 +const CONNECTED_STATE: SupervisorConnectionState = {
66 + ...AVAILABLE_CONNECTION_STATE,
67 + desired: true,
68 + phase: "connected",
69 +};
70 +
71 const makeSupervisor = Effect.fn("TestEnvironmentCommands.makeSupervisor")(function* (input: {
72 readonly commands: OrchestrationV2Command[];
73 readonly projects: ProjectMutation[];
@@ -121,7 +134,7 @@ const makeSupervisor = Effect.fn("TestEnvironmentCommands.makeSupervisor")(funct
134 };
135 return EnvironmentSupervisor.EnvironmentSupervisor.of({
136 target: TARGET,
124 - state: yield* SubscriptionRef.make(AVAILABLE_CONNECTION_STATE),
137 + state: yield* SubscriptionRef.make(CONNECTED_STATE),
138 session: yield* SubscriptionRef.make(Option.some(session)),
139 prepared: yield* SubscriptionRef.make(Option.none<PreparedConnection>()),
140 connect: Effect.void,
@@ -293,6 +306,182 @@ describe("V2 environment commands", () => {
306 }).pipe(Effect.provide(layerTestRandom)),
307 );
308
309 + // A dead socket reaches the caller as a protocol error, or as an interrupt when
310 + // the supervisor closes the session before the protocol reports the error.
311 + it.effect.each([
312 + [
313 + "protocol error",
314 + new RpcClientError.RpcClientError({
315 + reason: new RpcClientError.RpcClientDefect({
316 + message: "socket closed",
317 + cause: new Error("socket closed"),
318 + }),
319 + }),
320 + ],
321 + ["session close", Effect.failCause(Cause.interrupt())],
322 + ] as const)("retries a message dispatch lost to a %s with the same command id", ([, loss]) =>
323 + Effect.gen(function* () {
324 + const commands: OrchestrationV2Command[] = [];
325 + const supervisor = yield* makeSupervisor({ commands, projects: [] });
326 + const healthySession = Option.getOrThrow(yield* SubscriptionRef.get(supervisor.session));
327 + const firstAttemptLost = yield* Deferred.make<void>();
328 + const lostClient = {
329 + ...healthySession.client,
330 + [ORCHESTRATION_V2_WS_METHODS.dispatchCommand]: (command: OrchestrationV2Command) =>
331 + Effect.gen(function* () {
332 + commands.push(command);
333 + yield* SubscriptionRef.set(supervisor.session, Option.none());
334 + yield* Deferred.succeed(firstAttemptLost, undefined);
335 + return yield* loss;
336 + }),
337 + } as unknown as WsRpcProtocolClient;
338 + yield* SubscriptionRef.set(
339 + supervisor.session,
340 + Option.some({ ...healthySession, client: lostClient }),
341 + );
342 +
343 + const sendFiber = yield* startThreadTurn({
344 + commandId: CommandId.make("command-lost-ack"),
345 + threadId: v2ThreadId,
346 + message: {
347 + messageId: MessageId.make("message-lost-ack"),
348 + role: "user",
349 + text: "hello",
350 + attachments: [],
351 + },
352 + runtimeMode: "full-access",
353 + interactionMode: "default",
354 + }).pipe(
355 + Effect.provideService(EnvironmentSupervisor.EnvironmentSupervisor, supervisor),
356 + Effect.forkChild,
357 + );
358 + yield* Deferred.await(firstAttemptLost);
359 + yield* SubscriptionRef.set(supervisor.session, Option.some(healthySession));
360 + const exit = yield* Fiber.await(sendFiber);
361 +
362 + expect(commands.map((command) => [command.type, command.commandId])).toEqual([
363 + ["message.dispatch", "command-lost-ack"],
364 + ["message.dispatch", "command-lost-ack"],
365 + ]);
366 + expect(Exit.isSuccess(exit)).toBe(true);
367 + }).pipe(Effect.provide(layerTestRandom)),
368 + );
369 +
370 + it.effect("stops resending a message whose dispatch keeps losing the socket", () =>
371 + Effect.gen(function* () {
372 + const commands: OrchestrationV2Command[] = [];
373 + const supervisor = yield* makeSupervisor({ commands, projects: [] });
374 + const healthySession = Option.getOrThrow(yield* SubscriptionRef.get(supervisor.session));
375 + const dyingClient: WsRpcProtocolClient = {
376 + ...healthySession.client,
377 + [ORCHESTRATION_V2_WS_METHODS.dispatchCommand]: (command: OrchestrationV2Command) =>
378 + Effect.gen(function* () {
379 + commands.push(command);
380 + yield* SubscriptionRef.set(
381 + supervisor.session,
382 + Option.some({ ...healthySession, client: dyingClient }),
383 + );
384 + return yield* new RpcClientError.RpcClientError({
385 + reason: new RpcClientError.RpcClientDefect({
386 + message: "socket closed",
387 + cause: new Error("socket closed"),
388 + }),
389 + });
390 + }),
391 + } as unknown as WsRpcProtocolClient;
392 + yield* SubscriptionRef.set(
393 + supervisor.session,
394 + Option.some({ ...healthySession, client: dyingClient }),
395 + );
396 +
397 + const error = yield* startThreadTurn({
398 + commandId: CommandId.make("command-socket-killer"),
399 + threadId: v2ThreadId,
400 + message: {
401 + messageId: MessageId.make("message-socket-killer"),
402 + role: "user",
403 + text: "hello",
404 + attachments: [],
405 + },
406 + runtimeMode: "full-access",
407 + interactionMode: "default",
408 + }).pipe(
409 + Effect.provideService(EnvironmentSupervisor.EnvironmentSupervisor, supervisor),
410 + Effect.flip,
411 + );
412 +
413 + expect(commands.map((command) => command.commandId)).toEqual([
414 + "command-socket-killer",
415 + "command-socket-killer",
416 + "command-socket-killer",
417 + ]);
418 + expect(error._tag).toBe("RpcClientError");
419 + }).pipe(Effect.provide(layerTestRandom)),
420 + );
421 +
422 + it.effect("holds a lost message through reconnect backoff until disconnect", () =>
423 + Effect.gen(function* () {
424 + const commands: OrchestrationV2Command[] = [];
425 + const supervisor = yield* makeSupervisor({ commands, projects: [] });
426 + const healthySession = Option.getOrThrow(yield* SubscriptionRef.get(supervisor.session));
427 + const lostClient = {
428 + ...healthySession.client,
429 + [ORCHESTRATION_V2_WS_METHODS.dispatchCommand]: (command: OrchestrationV2Command) =>
430 + Effect.gen(function* () {
431 + commands.push(command);
432 + yield* SubscriptionRef.set(supervisor.session, Option.none());
433 + return yield* new RpcClientError.RpcClientError({
434 + reason: new RpcClientError.RpcClientDefect({
435 + message: "socket closed",
436 + cause: new Error("socket closed"),
437 + }),
438 + });
439 + }),
440 + } as unknown as WsRpcProtocolClient;
441 + const send = (resendAfterSocketLoss: boolean) =>
442 + Effect.gen(function* () {
443 + yield* SubscriptionRef.set(
444 + supervisor.session,
445 + Option.some({ ...healthySession, client: lostClient }),
446 + );
447 + return yield* startThreadTurn({
448 + commandId: CommandId.make(`command-host-gone-${resendAfterSocketLoss}`),
449 + threadId: v2ThreadId,
450 + message: {
451 + messageId: MessageId.make(`message-host-gone-${resendAfterSocketLoss}`),
452 + role: "user",
453 + text: "hello",
454 + attachments: [],
455 + },
456 + runtimeMode: "full-access",
457 + interactionMode: "default",
458 + resendAfterSocketLoss,
459 + }).pipe(
460 + Effect.provideService(EnvironmentSupervisor.EnvironmentSupervisor, supervisor),
461 + Effect.flip,
462 + Effect.forkChild,
463 + );
464 + });
465 +
466 + const optedOut = yield* send(false);
467 + expect((yield* Fiber.join(optedOut))._tag).toBe("RpcClientError");
468 +
469 + const waiting = yield* send(true);
470 + yield* SubscriptionRef.set(supervisor.state, {
471 + ...CONNECTED_STATE,
472 + phase: "backoff" as const,
473 + });
474 + yield* TestClock.adjust("2 minutes");
475 + expect(waiting.pollUnsafe()).toBeUndefined();
476 + yield* SubscriptionRef.set(supervisor.state, AVAILABLE_CONNECTION_STATE);
477 + expect((yield* Fiber.join(waiting))._tag).toBe("RpcClientError");
478 + expect(commands.map((command) => command.commandId)).toEqual([
479 + "command-host-gone-false",
480 + "command-host-gone-true",
481 + ]);
482 + }).pipe(Effect.provide(layerTestRandom)),
483 + );
484 +
485 it.effect("Stop with no run left ends the thread's pull request watches", () =>
486 Effect.gen(function* () {
487 const link = {
packages/client-runtime/src/operations/commands.ts
+90 -33
@@ -31,9 +31,15 @@ import {
31 import { derivePendingBackgroundWork } from "@t3tools/shared/orchestrationV2PendingBackgroundWork";
32 import { visibleThreadPullRequests } from "@t3tools/shared/threadPullRequests";
33 import * as SecureRandom from "effect/crypto/SecureRandom";
34 +import * as Cause from "effect/Cause";
35 import * as Effect from "effect/Effect";
36 +import * as Exit from "effect/Exit";
37 +import * as Option from "effect/Option";
38 +import * as Stream from "effect/Stream";
39 +import * as SubscriptionRef from "effect/SubscriptionRef";
40
36 -import { getInitialServerConfig, request } from "../rpc/client.ts";
41 +import * as EnvironmentSupervisor from "../connection/supervisor.ts";
42 +import { getInitialServerConfig, isRpcClientError, request } from "../rpc/client.ts";
43
44 interface CommandMetadata {
45 readonly commandId?: CommandId;
@@ -185,6 +191,8 @@ export interface StartThreadTurnInput extends ThreadCommandInput {
191 readonly dispatchMode?: "auto" | "queue" | "steer" | "restart" | "start";
192 /** Start a new provider session; needs the server's `freshContext` capability. */
193 readonly freshContext?: boolean;
194 + /** Defaults to true. Callers with their own retry queue opt out so a dead socket fails fast. */
195 + readonly resendAfterSocketLoss?: boolean;
196 }
197
198 export interface InterruptThreadTurnInput extends ThreadCommandInput {
@@ -279,6 +287,46 @@ const allocateCommandId = Effect.fn("EnvironmentCommands.allocateCommandId")(fun
287 const dispatch = (command: OrchestrationV2Command) =>
288 request(ORCHESTRATION_V2_WS_METHODS.dispatchCommand, command);
289
290 +// The socket can close after the server accepted the frame but before it answered.
291 +// The server replays an accepted commandId from its receipt, so the same command
292 +// waits out reconnect backoff until the supervisor stops trying. The resend cap
293 +// stops a frame that kills the socket from looping.
294 +const MAX_LOST_TRANSPORT_RESENDS = 2;
295 +
296 +const dispatchMessage = Effect.fn("EnvironmentCommands.dispatchMessage")(function* (
297 + command: Extract<OrchestrationV2Command, { type: "message.dispatch" }>,
298 + resendAfterSocketLoss: boolean,
299 +) {
300 + if (!resendAfterSocketLoss) return yield* dispatch(command);
301 + const supervisor = yield* EnvironmentSupervisor.EnvironmentSupervisor;
302 + for (let resends = 0; ; resends += 1) {
303 + const attemptSession = yield* SubscriptionRef.get(supervisor.session);
304 + const exit = yield* Effect.exit(dispatch(command));
305 + if (Exit.isSuccess(exit)) return exit.value;
306 + // Closing the session interrupts the request, which can beat the protocol error.
307 + const lostTransport =
308 + Cause.hasInterruptsOnly(exit.cause) ||
309 + exit.cause.reasons.some((reason) => reason._tag === "Fail" && isRpcClientError(reason.error));
310 + if (!lostTransport || resends >= MAX_LOST_TRANSPORT_RESENDS) return yield* exit;
311 + const replacement = yield* Effect.raceFirst(
312 + SubscriptionRef.changes(supervisor.session).pipe(
313 + Stream.filter(
314 + (session) =>
315 + Option.isSome(session) &&
316 + !(Option.isSome(attemptSession) && session.value === attemptSession.value),
317 + ),
318 + Stream.runHead,
319 + ),
320 + SubscriptionRef.changes(supervisor.state).pipe(
321 + Stream.filter((state) => state.phase === "available" || state.phase === "blocked"),
322 + Stream.runHead,
323 + Effect.as(Option.none()),
324 + ),
325 + );
326 + if (Option.isNone(replacement)) return yield* exit;
327 + }
328 +});
329 +
330 const getProjection = (threadId: ThreadId) =>
331 request(ORCHESTRATION_V2_WS_METHODS.getThreadProjection, { threadId });
332
@@ -733,56 +781,65 @@ export const startThreadTurn = Effect.fn("EnvironmentCommands.startThreadTurn")(
781 }
782
783 const requestedMode = input.dispatchMode ?? "auto";
784 + const resendAfterSocketLoss = input.resendAfterSocketLoss ?? true;
785 if (requestedMode === "start") {
737 - return yield* dispatch({
786 + return yield* dispatchMessage(
787 + {
788 + type: "message.dispatch",
789 + commandId,
790 + createdBy: "user",
791 + creationSource: input.creationSource ?? "web",
792 + threadId: input.threadId,
793 + messageId: input.message.messageId,
794 + ...(input.manualContinuationOfRunId === undefined
795 + ? {}
796 + : { manualContinuationOfRunId: input.manualContinuationOfRunId }),
797 + text: input.message.text,
798 + ...(context ? { context } : {}),
799 + ...(input.message.idleCompaction === undefined
800 + ? {}
801 + : { idleCompaction: input.message.idleCompaction }),
802 + attachments,
803 + ...(input.modelSelection === undefined ? {} : { modelSelection: input.modelSelection }),
804 + ...(input.sourceProposedPlan === undefined
805 + ? {}
806 + : { sourcePlanRef: input.sourceProposedPlan }),
807 + ...(input.freshContext ? { freshContext: true } : {}),
808 + dispatchMode: { type: "start_immediately" },
809 + },
810 + resendAfterSocketLoss,
811 + );
812 + }
813 +
814 + const dispatchMode =
815 + requestedMode === "queue"
816 + ? ({ type: "queue_after_active" } as const)
817 + : ({ type: "start_immediately" } as const);
818 + return yield* dispatchMessage(
819 + {
820 type: "message.dispatch",
821 commandId,
822 createdBy: "user",
823 creationSource: input.creationSource ?? "web",
824 threadId: input.threadId,
825 messageId: input.message.messageId,
744 - ...(input.manualContinuationOfRunId === undefined
745 - ? {}
746 - : { manualContinuationOfRunId: input.manualContinuationOfRunId }),
826 text: input.message.text,
827 ...(context ? { context } : {}),
828 ...(input.message.idleCompaction === undefined
829 ? {}
830 : { idleCompaction: input.message.idleCompaction }),
831 attachments,
832 + ...(input.titleSeed === undefined ? {} : { titleSeed: input.titleSeed }),
833 ...(input.modelSelection === undefined ? {} : { modelSelection: input.modelSelection }),
834 ...(input.sourceProposedPlan === undefined
835 ? {}
836 : { sourcePlanRef: input.sourceProposedPlan }),
837 ...(input.freshContext ? { freshContext: true } : {}),
758 - dispatchMode: { type: "start_immediately" },
759 - });
760 - }
761 -
762 - const dispatchMode =
763 - requestedMode === "queue"
764 - ? ({ type: "queue_after_active" } as const)
765 - : ({ type: "start_immediately" } as const);
766 - return yield* dispatch({
767 - type: "message.dispatch",
768 - commandId,
769 - createdBy: "user",
770 - creationSource: input.creationSource ?? "web",
771 - threadId: input.threadId,
772 - messageId: input.message.messageId,
773 - text: input.message.text,
774 - ...(context ? { context } : {}),
775 - ...(input.message.idleCompaction === undefined
776 - ? {}
777 - : { idleCompaction: input.message.idleCompaction }),
778 - attachments,
779 - ...(input.titleSeed === undefined ? {} : { titleSeed: input.titleSeed }),
780 - ...(input.modelSelection === undefined ? {} : { modelSelection: input.modelSelection }),
781 - ...(input.sourceProposedPlan === undefined ? {} : { sourcePlanRef: input.sourceProposedPlan }),
782 - ...(input.freshContext ? { freshContext: true } : {}),
783 - ...(requestedMode !== "queue" ? { deliveryIntent: requestedMode } : {}),
784 - dispatchMode,
785 - });
838 + ...(requestedMode !== "queue" ? { deliveryIntent: requestedMode } : {}),
839 + dispatchMode,
840 + },
841 + resendAfterSocketLoss,
842 + );
843 });
844
845 export const interruptThreadTurn = Effect.fn("EnvironmentCommands.interruptThreadTurn")(function* (