The best way to control Claude Code, Codex and more from any computer or phone
Mirrored from https://github.com/Camo67/t3code.git
Synced Nov 21, 21:00 UTC
fix(client-runtime): retry messages after socket loss with the same command ID (#24761)
Julius Marminge committed
Nov 11, 2026 at 15:47 UTC
962eeefa0392922d32eb54e0bedbbceedd094bef
3 files changed
+282
-34
apps/mobile/src/state/threadOutboxDelivery.ts
+2
@@ -911,6 +911,7 @@ export function createThreadOutboxDelivery(): {
911
interactionMode: sendSettings.interactionMode,
912
createdAt: queuedMessage.createdAt,
913
dispatchMode: "queue",
914
+ resendAfterSocketLoss: false,
915
},
916
});
917
if (AsyncResult.isFailure(compactionResult) && !hasAccess()) return true;
@@ -951,6 +952,7 @@ export function createThreadOutboxDelivery(): {
952
createdAt: queuedMessage.createdAt,
953
dispatchMode: queuedMessage.dispatchMode ?? "start",
954
...(queuedMessage.freshContext ? { freshContext: true } : {}),
955
+ resendAfterSocketLoss: false,
956
},
957
});
958
if (AsyncResult.isFailure(deliveryResult) && !hasAccess()) return true;
packages/client-runtime/src/operations/commands.test.ts
+190
-1
@@ -20,16 +20,23 @@ import {
20
type ProjectMutation,
21
} from "@t3tools/contracts";
22
import { describe, expect, it } from "@effect/vitest";
23
+import * as Cause from "effect/Cause";
24
import * as SecureRandom from "effect/crypto/SecureRandom";
25
+import * as Deferred from "effect/Deferred";
26
import * as Effect from "effect/Effect";
27
+import * as Exit from "effect/Exit";
28
+import * as Fiber from "effect/Fiber";
29
import * as Layer from "effect/Layer";
30
import * as Option from "effect/Option";
31
import * as SubscriptionRef from "effect/SubscriptionRef";
32
+import * as TestClock from "effect/testing/TestClock";
33
+import { RpcClientError } from "effect/rpc";
34
35
import {
36
AVAILABLE_CONNECTION_STATE,
37
PrimaryConnectionTarget,
38
type PreparedConnection,
39
+ type SupervisorConnectionState,
40
} from "../connection/model.ts";
41
import * as EnvironmentSupervisor from "../connection/supervisor.ts";
42
import * as RpcSession from "../rpc/session.ts";
@@ -55,6 +62,12 @@ const TARGET = new PrimaryConnectionTarget({
62
wsBaseUrl: "wss://environment.example.test",
63
});
64
65
+const CONNECTED_STATE: SupervisorConnectionState = {
66
+ ...AVAILABLE_CONNECTION_STATE,
67
+ desired: true,
68
+ phase: "connected",
69
+};
70
+
71
const makeSupervisor = Effect.fn("TestEnvironmentCommands.makeSupervisor")(function* (input: {
72
readonly commands: OrchestrationV2Command[];
73
readonly projects: ProjectMutation[];
@@ -121,7 +134,7 @@ const makeSupervisor = Effect.fn("TestEnvironmentCommands.makeSupervisor")(funct
134
};
135
return EnvironmentSupervisor.EnvironmentSupervisor.of({
136
target: TARGET,
124
- state: yield* SubscriptionRef.make(AVAILABLE_CONNECTION_STATE),
137
+ state: yield* SubscriptionRef.make(CONNECTED_STATE),
138
session: yield* SubscriptionRef.make(Option.some(session)),
139
prepared: yield* SubscriptionRef.make(Option.none<PreparedConnection>()),
140
connect: Effect.void,
@@ -293,6 +306,182 @@ describe("V2 environment commands", () => {
306
}).pipe(Effect.provide(layerTestRandom)),
307
);
308
309
+ // A dead socket reaches the caller as a protocol error, or as an interrupt when
310
+ // the supervisor closes the session before the protocol reports the error.
311
+ it.effect.each([
312
+ [
313
+ "protocol error",
314
+ new RpcClientError.RpcClientError({
315
+ reason: new RpcClientError.RpcClientDefect({
316
+ message: "socket closed",
317
+ cause: new Error("socket closed"),
318
+ }),
319
+ }),
320
+ ],
321
+ ["session close", Effect.failCause(Cause.interrupt())],
322
+ ] as const)("retries a message dispatch lost to a %s with the same command id", ([, loss]) =>
323
+ Effect.gen(function* () {
324
+ const commands: OrchestrationV2Command[] = [];
325
+ const supervisor = yield* makeSupervisor({ commands, projects: [] });
326
+ const healthySession = Option.getOrThrow(yield* SubscriptionRef.get(supervisor.session));
327
+ const firstAttemptLost = yield* Deferred.make<void>();
328
+ const lostClient = {
329
+ ...healthySession.client,
330
+ [ORCHESTRATION_V2_WS_METHODS.dispatchCommand]: (command: OrchestrationV2Command) =>
331
+ Effect.gen(function* () {
332
+ commands.push(command);
333
+ yield* SubscriptionRef.set(supervisor.session, Option.none());
334
+ yield* Deferred.succeed(firstAttemptLost, undefined);
335
+ return yield* loss;
336
+ }),
337
+ } as unknown as WsRpcProtocolClient;
338
+ yield* SubscriptionRef.set(
339
+ supervisor.session,
340
+ Option.some({ ...healthySession, client: lostClient }),
341
+ );
342
+
343
+ const sendFiber = yield* startThreadTurn({
344
+ commandId: CommandId.make("command-lost-ack"),
345
+ threadId: v2ThreadId,
346
+ message: {
347
+ messageId: MessageId.make("message-lost-ack"),
348
+ role: "user",
349
+ text: "hello",
350
+ attachments: [],
351
+ },
352
+ runtimeMode: "full-access",
353
+ interactionMode: "default",
354
+ }).pipe(
355
+ Effect.provideService(EnvironmentSupervisor.EnvironmentSupervisor, supervisor),
356
+ Effect.forkChild,
357
+ );
358
+ yield* Deferred.await(firstAttemptLost);
359
+ yield* SubscriptionRef.set(supervisor.session, Option.some(healthySession));
360
+ const exit = yield* Fiber.await(sendFiber);
361
+
362
+ expect(commands.map((command) => [command.type, command.commandId])).toEqual([
363
+ ["message.dispatch", "command-lost-ack"],
364
+ ["message.dispatch", "command-lost-ack"],
365
+ ]);
366
+ expect(Exit.isSuccess(exit)).toBe(true);
367
+ }).pipe(Effect.provide(layerTestRandom)),
368
+ );
369
+
370
+ it.effect("stops resending a message whose dispatch keeps losing the socket", () =>
371
+ Effect.gen(function* () {
372
+ const commands: OrchestrationV2Command[] = [];
373
+ const supervisor = yield* makeSupervisor({ commands, projects: [] });
374
+ const healthySession = Option.getOrThrow(yield* SubscriptionRef.get(supervisor.session));
375
+ const dyingClient: WsRpcProtocolClient = {
376
+ ...healthySession.client,
377
+ [ORCHESTRATION_V2_WS_METHODS.dispatchCommand]: (command: OrchestrationV2Command) =>
378
+ Effect.gen(function* () {
379
+ commands.push(command);
380
+ yield* SubscriptionRef.set(
381
+ supervisor.session,
382
+ Option.some({ ...healthySession, client: dyingClient }),
383
+ );
384
+ return yield* new RpcClientError.RpcClientError({
385
+ reason: new RpcClientError.RpcClientDefect({
386
+ message: "socket closed",
387
+ cause: new Error("socket closed"),
388
+ }),
389
+ });
390
+ }),
391
+ } as unknown as WsRpcProtocolClient;
392
+ yield* SubscriptionRef.set(
393
+ supervisor.session,
394
+ Option.some({ ...healthySession, client: dyingClient }),
395
+ );
396
+
397
+ const error = yield* startThreadTurn({
398
+ commandId: CommandId.make("command-socket-killer"),
399
+ threadId: v2ThreadId,
400
+ message: {
401
+ messageId: MessageId.make("message-socket-killer"),
402
+ role: "user",
403
+ text: "hello",
404
+ attachments: [],
405
+ },
406
+ runtimeMode: "full-access",
407
+ interactionMode: "default",
408
+ }).pipe(
409
+ Effect.provideService(EnvironmentSupervisor.EnvironmentSupervisor, supervisor),
410
+ Effect.flip,
411
+ );
412
+
413
+ expect(commands.map((command) => command.commandId)).toEqual([
414
+ "command-socket-killer",
415
+ "command-socket-killer",
416
+ "command-socket-killer",
417
+ ]);
418
+ expect(error._tag).toBe("RpcClientError");
419
+ }).pipe(Effect.provide(layerTestRandom)),
420
+ );
421
+
422
+ it.effect("holds a lost message through reconnect backoff until disconnect", () =>
423
+ Effect.gen(function* () {
424
+ const commands: OrchestrationV2Command[] = [];
425
+ const supervisor = yield* makeSupervisor({ commands, projects: [] });
426
+ const healthySession = Option.getOrThrow(yield* SubscriptionRef.get(supervisor.session));
427
+ const lostClient = {
428
+ ...healthySession.client,
429
+ [ORCHESTRATION_V2_WS_METHODS.dispatchCommand]: (command: OrchestrationV2Command) =>
430
+ Effect.gen(function* () {
431
+ commands.push(command);
432
+ yield* SubscriptionRef.set(supervisor.session, Option.none());
433
+ return yield* new RpcClientError.RpcClientError({
434
+ reason: new RpcClientError.RpcClientDefect({
435
+ message: "socket closed",
436
+ cause: new Error("socket closed"),
437
+ }),
438
+ });
439
+ }),
440
+ } as unknown as WsRpcProtocolClient;
441
+ const send = (resendAfterSocketLoss: boolean) =>
442
+ Effect.gen(function* () {
443
+ yield* SubscriptionRef.set(
444
+ supervisor.session,
445
+ Option.some({ ...healthySession, client: lostClient }),
446
+ );
447
+ return yield* startThreadTurn({
448
+ commandId: CommandId.make(`command-host-gone-${resendAfterSocketLoss}`),
449
+ threadId: v2ThreadId,
450
+ message: {
451
+ messageId: MessageId.make(`message-host-gone-${resendAfterSocketLoss}`),
452
+ role: "user",
453
+ text: "hello",
454
+ attachments: [],
455
+ },
456
+ runtimeMode: "full-access",
457
+ interactionMode: "default",
458
+ resendAfterSocketLoss,
459
+ }).pipe(
460
+ Effect.provideService(EnvironmentSupervisor.EnvironmentSupervisor, supervisor),
461
+ Effect.flip,
462
+ Effect.forkChild,
463
+ );
464
+ });
465
+
466
+ const optedOut = yield* send(false);
467
+ expect((yield* Fiber.join(optedOut))._tag).toBe("RpcClientError");
468
+
469
+ const waiting = yield* send(true);
470
+ yield* SubscriptionRef.set(supervisor.state, {
471
+ ...CONNECTED_STATE,
472
+ phase: "backoff" as const,
473
+ });
474
+ yield* TestClock.adjust("2 minutes");
475
+ expect(waiting.pollUnsafe()).toBeUndefined();
476
+ yield* SubscriptionRef.set(supervisor.state, AVAILABLE_CONNECTION_STATE);
477
+ expect((yield* Fiber.join(waiting))._tag).toBe("RpcClientError");
478
+ expect(commands.map((command) => command.commandId)).toEqual([
479
+ "command-host-gone-false",
480
+ "command-host-gone-true",
481
+ ]);
482
+ }).pipe(Effect.provide(layerTestRandom)),
483
+ );
484
+
485
it.effect("Stop with no run left ends the thread's pull request watches", () =>
486
Effect.gen(function* () {
487
const link = {
packages/client-runtime/src/operations/commands.ts
+90
-33
@@ -31,9 +31,15 @@ import {
31
import { derivePendingBackgroundWork } from "@t3tools/shared/orchestrationV2PendingBackgroundWork";
32
import { visibleThreadPullRequests } from "@t3tools/shared/threadPullRequests";
33
import * as SecureRandom from "effect/crypto/SecureRandom";
34
+import * as Cause from "effect/Cause";
35
import * as Effect from "effect/Effect";
36
+import * as Exit from "effect/Exit";
37
+import * as Option from "effect/Option";
38
+import * as Stream from "effect/Stream";
39
+import * as SubscriptionRef from "effect/SubscriptionRef";
40
36
-import { getInitialServerConfig, request } from "../rpc/client.ts";
41
+import * as EnvironmentSupervisor from "../connection/supervisor.ts";
42
+import { getInitialServerConfig, isRpcClientError, request } from "../rpc/client.ts";
43
44
interface CommandMetadata {
45
readonly commandId?: CommandId;
@@ -185,6 +191,8 @@ export interface StartThreadTurnInput extends ThreadCommandInput {
191
readonly dispatchMode?: "auto" | "queue" | "steer" | "restart" | "start";
192
/** Start a new provider session; needs the server's `freshContext` capability. */
193
readonly freshContext?: boolean;
194
+ /** Defaults to true. Callers with their own retry queue opt out so a dead socket fails fast. */
195
+ readonly resendAfterSocketLoss?: boolean;
196
}
197
198
export interface InterruptThreadTurnInput extends ThreadCommandInput {
@@ -279,6 +287,46 @@ const allocateCommandId = Effect.fn("EnvironmentCommands.allocateCommandId")(fun
287
const dispatch = (command: OrchestrationV2Command) =>
288
request(ORCHESTRATION_V2_WS_METHODS.dispatchCommand, command);
289
290
+// The socket can close after the server accepted the frame but before it answered.
291
+// The server replays an accepted commandId from its receipt, so the same command
292
+// waits out reconnect backoff until the supervisor stops trying. The resend cap
293
+// stops a frame that kills the socket from looping.
294
+const MAX_LOST_TRANSPORT_RESENDS = 2;
295
+
296
+const dispatchMessage = Effect.fn("EnvironmentCommands.dispatchMessage")(function* (
297
+ command: Extract<OrchestrationV2Command, { type: "message.dispatch" }>,
298
+ resendAfterSocketLoss: boolean,
299
+) {
300
+ if (!resendAfterSocketLoss) return yield* dispatch(command);
301
+ const supervisor = yield* EnvironmentSupervisor.EnvironmentSupervisor;
302
+ for (let resends = 0; ; resends += 1) {
303
+ const attemptSession = yield* SubscriptionRef.get(supervisor.session);
304
+ const exit = yield* Effect.exit(dispatch(command));
305
+ if (Exit.isSuccess(exit)) return exit.value;
306
+ // Closing the session interrupts the request, which can beat the protocol error.
307
+ const lostTransport =
308
+ Cause.hasInterruptsOnly(exit.cause) ||
309
+ exit.cause.reasons.some((reason) => reason._tag === "Fail" && isRpcClientError(reason.error));
310
+ if (!lostTransport || resends >= MAX_LOST_TRANSPORT_RESENDS) return yield* exit;
311
+ const replacement = yield* Effect.raceFirst(
312
+ SubscriptionRef.changes(supervisor.session).pipe(
313
+ Stream.filter(
314
+ (session) =>
315
+ Option.isSome(session) &&
316
+ !(Option.isSome(attemptSession) && session.value === attemptSession.value),
317
+ ),
318
+ Stream.runHead,
319
+ ),
320
+ SubscriptionRef.changes(supervisor.state).pipe(
321
+ Stream.filter((state) => state.phase === "available" || state.phase === "blocked"),
322
+ Stream.runHead,
323
+ Effect.as(Option.none()),
324
+ ),
325
+ );
326
+ if (Option.isNone(replacement)) return yield* exit;
327
+ }
328
+});
329
+
330
const getProjection = (threadId: ThreadId) =>
331
request(ORCHESTRATION_V2_WS_METHODS.getThreadProjection, { threadId });
332
@@ -733,56 +781,65 @@ export const startThreadTurn = Effect.fn("EnvironmentCommands.startThreadTurn")(
781
}
782
783
const requestedMode = input.dispatchMode ?? "auto";
784
+ const resendAfterSocketLoss = input.resendAfterSocketLoss ?? true;
785
if (requestedMode === "start") {
737
- return yield* dispatch({
786
+ return yield* dispatchMessage(
787
+ {
788
+ type: "message.dispatch",
789
+ commandId,
790
+ createdBy: "user",
791
+ creationSource: input.creationSource ?? "web",
792
+ threadId: input.threadId,
793
+ messageId: input.message.messageId,
794
+ ...(input.manualContinuationOfRunId === undefined
795
+ ? {}
796
+ : { manualContinuationOfRunId: input.manualContinuationOfRunId }),
797
+ text: input.message.text,
798
+ ...(context ? { context } : {}),
799
+ ...(input.message.idleCompaction === undefined
800
+ ? {}
801
+ : { idleCompaction: input.message.idleCompaction }),
802
+ attachments,
803
+ ...(input.modelSelection === undefined ? {} : { modelSelection: input.modelSelection }),
804
+ ...(input.sourceProposedPlan === undefined
805
+ ? {}
806
+ : { sourcePlanRef: input.sourceProposedPlan }),
807
+ ...(input.freshContext ? { freshContext: true } : {}),
808
+ dispatchMode: { type: "start_immediately" },
809
+ },
810
+ resendAfterSocketLoss,
811
+ );
812
+ }
813
+
814
+ const dispatchMode =
815
+ requestedMode === "queue"
816
+ ? ({ type: "queue_after_active" } as const)
817
+ : ({ type: "start_immediately" } as const);
818
+ return yield* dispatchMessage(
819
+ {
820
type: "message.dispatch",
821
commandId,
822
createdBy: "user",
823
creationSource: input.creationSource ?? "web",
824
threadId: input.threadId,
825
messageId: input.message.messageId,
744
- ...(input.manualContinuationOfRunId === undefined
745
- ? {}
746
- : { manualContinuationOfRunId: input.manualContinuationOfRunId }),
826
text: input.message.text,
827
...(context ? { context } : {}),
828
...(input.message.idleCompaction === undefined
829
? {}
830
: { idleCompaction: input.message.idleCompaction }),
831
attachments,
832
+ ...(input.titleSeed === undefined ? {} : { titleSeed: input.titleSeed }),
833
...(input.modelSelection === undefined ? {} : { modelSelection: input.modelSelection }),
834
...(input.sourceProposedPlan === undefined
835
? {}
836
: { sourcePlanRef: input.sourceProposedPlan }),
837
...(input.freshContext ? { freshContext: true } : {}),
758
- dispatchMode: { type: "start_immediately" },
759
- });
760
- }
761
-
762
- const dispatchMode =
763
- requestedMode === "queue"
764
- ? ({ type: "queue_after_active" } as const)
765
- : ({ type: "start_immediately" } as const);
766
- return yield* dispatch({
767
- type: "message.dispatch",
768
- commandId,
769
- createdBy: "user",
770
- creationSource: input.creationSource ?? "web",
771
- threadId: input.threadId,
772
- messageId: input.message.messageId,
773
- text: input.message.text,
774
- ...(context ? { context } : {}),
775
- ...(input.message.idleCompaction === undefined
776
- ? {}
777
- : { idleCompaction: input.message.idleCompaction }),
778
- attachments,
779
- ...(input.titleSeed === undefined ? {} : { titleSeed: input.titleSeed }),
780
- ...(input.modelSelection === undefined ? {} : { modelSelection: input.modelSelection }),
781
- ...(input.sourceProposedPlan === undefined ? {} : { sourcePlanRef: input.sourceProposedPlan }),
782
- ...(input.freshContext ? { freshContext: true } : {}),
783
- ...(requestedMode !== "queue" ? { deliveryIntent: requestedMode } : {}),
784
- dispatchMode,
785
- });
838
+ ...(requestedMode !== "queue" ? { deliveryIntent: requestedMode } : {}),
839
+ dispatchMode,
840
+ },
841
+ resendAfterSocketLoss,
842
+ );
843
});
844
845
export const interruptThreadTurn = Effect.fn("EnvironmentCommands.interruptThreadTurn")(function* (