The best way to control Claude Code, Codex and more from any computer or phone
Mirrored from https://github.com/Camo67/t3code.git
Synced Nov 21, 17:15 UTC
fix(server): a remote delegated task no longer stays running when its follower hits a defect (#32331)
Julius Marminge committed
Nov 13, 2026 at 10:19 UTC
571b319643611fb21db2c6ab7a0c38697b8cffd9
2 files changed
+255
-17
apps/server/src/peer/RemoteDelegation.test.ts
+199
@@ -25,6 +25,7 @@ import * as Context from "effect/Context";
25
import * as DateTime from "effect/DateTime";
26
import * as Deferred from "effect/Deferred";
27
import * as Effect from "effect/Effect";
28
+import * as Fiber from "effect/Fiber";
29
import * as Layer from "effect/Layer";
30
import * as Option from "effect/Option";
31
import * as Queue from "effect/Queue";
@@ -644,6 +645,8 @@ const makeLaptop = (
645
>,
646
/** The laptop's links from before a restart, which keep their sessions there. */
647
kept?: Context.Context<Layer.Success<ReturnType<typeof layerLinkingEnvironment>>>,
648
+ /** Runs as each follow attempt here starts, on its follower's fiber. */
649
+ beforeFollow: Effect.Effect<void, Orchestrator.OrchestratorV2Error> = Effect.void,
650
) =>
651
Effect.gen(function* () {
652
const linking = kept ?? (yield* layerLinkingEnvironment(laptop).pipe(Layer.build));
@@ -672,6 +675,19 @@ const makeLaptop = (
675
Layer.provideMerge(McpHttpServer.layerInProcessMcpServer),
676
Layer.provideMerge(
677
RemoteDelegation.layer.pipe(
678
+ Layer.provide(
679
+ Layer.effect(
680
+ ThreadManagement.ThreadManagementService,
681
+ Effect.gen(function* () {
682
+ const threads = yield* ThreadManagement.ThreadManagementService;
683
+ return {
684
+ ...threads,
685
+ getThreadEventSequence: (threadId: ThreadId) =>
686
+ beforeFollow.pipe(Effect.andThen(threads.getThreadEventSequence(threadId))),
687
+ };
688
+ }),
689
+ ),
690
+ ),
691
Layer.provideMerge(PeerForwarding.layer),
692
Layer.provide(
693
Layer.mock(ProjectService.ProjectService)({
@@ -1784,6 +1800,108 @@ const steppedRetries = Effect.gen(function* () {
1800
return { failed, retry, schedule };
1801
});
1802
1803
+it.effect("a follower that keeps hitting defects fails the task and wakes the parent", () =>
1804
+ Effect.scoped(
1805
+ Effect.gen(function* () {
1806
+ const thread = yield* makeBoxThread;
1807
+ const b = yield* serveBox(thread);
1808
+ const offers = yield* SubscriptionRef.make<
1809
+ ReadonlyArray<ProviderContinuationRequests.ProviderContinuationRequest>
1810
+ >([]);
1811
+ const database = yield* SqlitePersistence.layerMemory.pipe(Layer.build);
1812
+ const retries = yield* steppedRetries;
1813
+ const attempts = yield* Ref.make(0);
1814
+ // The defect carries internal detail that the task's result must not.
1815
+ const beforeFollow = Ref.update(attempts, (count) => count + 1).pipe(
1816
+ Effect.andThen(Effect.die(new Error("A follow attempt hit a bug: defect-detail-4f1c9e"))),
1817
+ );
1818
+ const a = yield* makeLaptop(database, offers, undefined, beforeFollow).pipe(
1819
+ Effect.provideService(RemoteDelegation.FollowRetrySchedule, retries.schedule),
1820
+ );
1821
+ yield* linkTo(a.links, b, "full-access");
1822
+ yield* seedParent(a);
1823
+ yield* Queue.take(retries.failed).pipe(
1824
+ Effect.andThen(Queue.offer(retries.retry, undefined)),
1825
+ Effect.forever,
1826
+ Effect.forkScoped,
1827
+ );
1828
+
1829
+ const delegated = yield* delegateToBox(a);
1830
+ const task = yield* taskResult(a, delegated.taskId);
1831
+ expect(yield* Ref.get(attempts)).toBe(3);
1832
+ expect(task?.status).toBe("failed");
1833
+ expect(task?.result).toContain(delegated.remoteChild.threadId);
1834
+ expect(task?.result).not.toContain("defect-detail-4f1c9e");
1835
+ const offered = yield* SubscriptionRef.changes(offers).pipe(
1836
+ Stream.filter((seen) => seen.some((offer) => offer.threadId === parentThreadId)),
1837
+ Stream.runHead,
1838
+ );
1839
+ expect(Option.isSome(offered)).toBe(true);
1840
+ }),
1841
+ ).pipe(Effect.provide(NodeHttpServer.layerTest)),
1842
+);
1843
+
1844
+it.effect.each(["a still-running wait", "a failed attempt"] as const)(
1845
+ "defects separated by %s do not add up to giving up",
1846
+ (between) =>
1847
+ Effect.scoped(
1848
+ Effect.gen(function* () {
1849
+ const thread = yield* makeBoxThread;
1850
+ const b = yield* serveBox(thread);
1851
+ const offers = yield* SubscriptionRef.make<
1852
+ ReadonlyArray<ProviderContinuationRequests.ProviderContinuationRequest>
1853
+ >([]);
1854
+ const database = yield* SqlitePersistence.layerMemory.pipe(Layer.build);
1855
+ const retries = yield* steppedRetries;
1856
+ const attempts = yield* Ref.make(0);
1857
+ const defect = Effect.die(new Error("A follow attempt hit a bug."));
1858
+ const script = [
1859
+ defect,
1860
+ between === "a still-running wait"
1861
+ ? Effect.void
1862
+ : Effect.fail(
1863
+ new Orchestrator.OrchestratorProjectionError({ threadId: parentThreadId }),
1864
+ ),
1865
+ defect,
1866
+ between === "a still-running wait"
1867
+ ? Effect.void
1868
+ : Effect.fail(
1869
+ new Orchestrator.OrchestratorProjectionError({ threadId: parentThreadId }),
1870
+ ),
1871
+ defect,
1872
+ ];
1873
+ // Past the script, the run there has ended, so the next attempt completes the task.
1874
+ const beforeFollow = Ref.getAndUpdate(attempts, (count) => count + 1).pipe(
1875
+ Effect.flatMap(
1876
+ (count) =>
1877
+ script[count] ??
1878
+ Ref.set(thread.reply, "All green.").pipe(
1879
+ Effect.andThen(Ref.set(thread.status, "completed")),
1880
+ ),
1881
+ ),
1882
+ );
1883
+ // Each wait there answers at once that its run is still running.
1884
+ yield* Deferred.succeed(thread.finished, undefined);
1885
+ const a = yield* makeLaptop(database, offers, undefined, beforeFollow).pipe(
1886
+ Effect.provideService(RemoteDelegation.FollowRetrySchedule, retries.schedule),
1887
+ );
1888
+ yield* linkTo(a.links, b, "full-access");
1889
+ yield* seedParent(a);
1890
+ yield* Queue.take(retries.failed).pipe(
1891
+ Effect.andThen(Queue.offer(retries.retry, undefined)),
1892
+ Effect.forever,
1893
+ Effect.forkScoped,
1894
+ );
1895
+
1896
+ const delegated = yield* delegateToBox(a);
1897
+ expect(yield* taskResult(a, delegated.taskId)).toMatchObject({
1898
+ status: "completed",
1899
+ result: "All green.",
1900
+ });
1901
+ }),
1902
+ ).pipe(Effect.provide(NodeHttpServer.layerTest)),
1903
+);
1904
+
1905
/** Waits until the task here records that the `stopCount`th Stop after its result reached there. */
1906
const stoppedThere = (
1907
laptopEnv: Effect.Success<ReturnType<typeof makeLaptop>>,
@@ -1813,6 +1931,87 @@ const stoppedThere = (
1931
),
1932
);
1933
1934
+it.effect("a follower that keeps hitting defects leaves a stop it owes owed", () =>
1935
+ Effect.scoped(
1936
+ Effect.gen(function* () {
1937
+ const thread = yield* makeBoxThread;
1938
+ const b = yield* serveBox(thread);
1939
+ const offers = yield* SubscriptionRef.make<
1940
+ ReadonlyArray<ProviderContinuationRequests.ProviderContinuationRequest>
1941
+ >([]);
1942
+ const database = yield* SqlitePersistence.layerMemory.pipe(Layer.build);
1943
+ const retries = yield* steppedRetries;
1944
+ // Follow attempts hit a defect only once the task has its result.
1945
+ const broken = yield* Ref.make(false);
1946
+ const follower = yield* Deferred.make<Fiber.Fiber<unknown, unknown>>();
1947
+ const beforeFollow = Ref.get(broken).pipe(
1948
+ Effect.flatMap((isBroken) =>
1949
+ isBroken
1950
+ ? Effect.withFiber((fiber) => Deferred.succeed(follower, fiber)).pipe(
1951
+ Effect.andThen(Effect.die(new Error("A follow attempt hit a bug."))),
1952
+ )
1953
+ : Effect.void,
1954
+ ),
1955
+ );
1956
+ const a = yield* makeLaptop(database, offers, undefined, beforeFollow).pipe(
1957
+ Effect.provideService(RemoteDelegation.FollowRetrySchedule, retries.schedule),
1958
+ );
1959
+ yield* a.remote.start();
1960
+ yield* linkTo(a.links, b, "full-access");
1961
+ yield* seedParent(a);
1962
+ const delegated = yield* delegateToBox(a);
1963
+ yield* Ref.set(thread.reply, "All green.");
1964
+ yield* Ref.set(thread.status, "completed");
1965
+ yield* Deferred.succeed(thread.finished, undefined);
1966
+ yield* taskResult(a, delegated.taskId);
1967
+ yield* Ref.set(broken, true);
1968
+ yield* Queue.take(retries.failed).pipe(
1969
+ Effect.andThen(Queue.offer(retries.retry, undefined)),
1970
+ Effect.forever,
1971
+ Effect.forkScoped,
1972
+ );
1973
+
1974
+ // The Stop owes one stop there, which every attempt to send fails on.
1975
+ yield* a.threads.stopDelegatedTasks({
1976
+ threadId: parentThreadId,
1977
+ commandId: CommandId.make("command:stop:laptop-parent-defective-follower"),
1978
+ });
1979
+ const owedAgain = a.orchestrator
1980
+ .streamStoredEventsFrom({
1981
+ threadId: parentThreadId,
1982
+ afterSequence: 0,
1983
+ eventType: "subagent.updated",
1984
+ })
1985
+ .pipe(
1986
+ Stream.filter(
1987
+ (stored) =>
1988
+ stored.event.type === "subagent.updated" &&
1989
+ stored.event.payload.id === delegated.taskId &&
1990
+ (stored.event.payload.remoteChild?.stopCount ?? 0) > 1,
1991
+ ),
1992
+ Stream.runHead,
1993
+ Effect.flatMap((owed) =>
1994
+ Option.isSome(owed) ? Effect.succeed("owed another stop" as const) : Effect.never,
1995
+ ),
1996
+ );
1997
+ const ended = yield* Deferred.await(follower).pipe(
1998
+ Effect.flatMap(Fiber.await),
1999
+ Effect.as("follower ended" as const),
2000
+ Effect.raceFirst(owedAgain),
2001
+ );
2002
+ expect(ended).toBe("follower ended");
2003
+ const { subagents } = yield* a.threads.getThreadRecords(parentThreadId, ["subagents"], {
2004
+ subagentIds: [delegated.taskId],
2005
+ });
2006
+ expect(subagents[0]).toMatchObject({
2007
+ status: "completed",
2008
+ result: "All green.",
2009
+ remoteChild: { stoppedThere: false, stopCount: 1 },
2010
+ });
2011
+ }),
2012
+ ).pipe(Effect.provide(NodeHttpServer.layerTest)),
2013
+);
2014
+
2015
it.effect("stopping the parent stops follow-up work on a task there that has its result", () =>
2016
Effect.scoped(
2017
Effect.gen(function* () {
apps/server/src/peer/RemoteDelegation.ts
+56
-17
@@ -19,6 +19,7 @@ import {
19
type RunId,
20
type ThreadId,
21
} from "@t3tools/contracts";
22
+import * as Cause from "effect/Cause";
23
import * as Context from "effect/Context";
24
import * as SecureRandom from "effect/crypto/SecureRandom";
25
import * as DateTime from "effect/DateTime";
@@ -53,6 +54,8 @@ import * as PeerLinks from "./PeerLinks.ts";
54
const FOLLOW_WAIT_MS = 10 * 60 * 1_000;
55
/** A linked environment that stops answering is retried, backing off to this. */
56
const FOLLOW_MAX_BACKOFF = Duration.minutes(5);
57
+/** A follower that hits this many defects in a row fails its task instead of retrying. */
58
+const FOLLOW_MAX_DEFECTS = 3;
59
/** How a follower retries; tests step it by hand. */
60
export const FollowRetrySchedule = Context.Reference<Schedule.Schedule<unknown>>(
61
"t3/peer/RemoteDelegation/FollowRetrySchedule",
@@ -673,26 +676,54 @@ const make = Effect.gen(function* () {
676
/**
677
* Waits for the task's run there to end, completes the task here, and stops it there if it
678
* ended here first. A revoked, expired or forgotten link, or a thread or run gone there,
676
- * fails the task with nothing left to stop; anything else (the other machine asleep, a
677
- * dropped route) is retried with back-off, and so is completing.
679
+ * fails the task with nothing left to stop; repeated defects here fail it too. Anything else
680
+ * (the other machine asleep, a dropped route) is retried with back-off, and so is completing.
681
*/
682
const follow = (parentThreadId: ThreadId, taskId: NodeId): Effect.Effect<void> =>
680
- followOnce(parentThreadId, taskId).pipe(
681
- // `false` means it is still running there: follow again straight away.
682
- Effect.repeat({ while: (done) => !done }),
683
- Effect.catchTags({
684
- OrchestratorMcpFailure: Effect.fn("RemoteDelegation.followFailed")(function* (error) {
685
- const reason = yield* finalReason(parentThreadId, taskId, error);
686
- if (reason === undefined) return yield* error;
687
- // Gone there, or no link to reach it by: nothing left to stop there.
688
- const { task } = yield* readTask(parentThreadId, taskId);
689
- const stopCount = task?.remoteChild?.stopCount;
690
- yield* complete(parentThreadId, taskId, "failed", reason, "stopped", stopCount);
683
+ Effect.suspend(() => {
684
+ let defects = 0;
685
+ const noDefect = Effect.sync(() => {
686
+ defects = 0;
687
+ });
688
+ return followOnce(parentThreadId, taskId).pipe(
689
+ Effect.tap(noDefect),
690
+ // `false` means it is still running there: follow again straight away.
691
+ Effect.repeat({ while: (done) => !done }),
692
+ Effect.catchTags({
693
+ OrchestratorMcpFailure: Effect.fn("RemoteDelegation.followFailed")(function* (error) {
694
+ const reason = yield* finalReason(parentThreadId, taskId, error);
695
+ if (reason === undefined) return yield* error;
696
+ // Gone there, or no link to reach it by: nothing left to stop there.
697
+ const { task } = yield* readTask(parentThreadId, taskId);
698
+ const stopCount = task?.remoteChild?.stopCount;
699
+ yield* complete(parentThreadId, taskId, "failed", reason, "stopped", stopCount);
700
+ }),
701
}),
692
- }),
693
- Effect.retry(followRetry),
694
- Effect.ignoreCause({ log: true }),
695
- );
702
+ Effect.tapError(() => noDefect),
703
+ Effect.tapDefect(() =>
704
+ Effect.sync(() => {
705
+ defects += 1;
706
+ }),
707
+ ),
708
+ failOnDefect,
709
+ Effect.retry({ schedule: followRetry, while: () => defects < FOLLOW_MAX_DEFECTS }),
710
+ Effect.catch(
711
+ Effect.fn("RemoteDelegation.followGaveUp")(
712
+ function* () {
713
+ const { task } = yield* readTask(parentThreadId, taskId);
714
+ const remote = task?.remoteChild;
715
+ // A task with its result keeps it; completing it again would owe another stop there.
716
+ if (remote === undefined || task?.result !== null) return;
717
+ const result = `Following the thread ${remote.threadId} in ${remote.label} failed repeatedly, so it may still run there (task_cancel stops it).`;
718
+ yield* complete(parentThreadId, taskId, "failed", result, undefined);
719
+ },
720
+ failOnDefect,
721
+ Effect.retry(followRetry),
722
+ ),
723
+ ),
724
+ Effect.ignoreCause({ log: true }),
725
+ );
726
+ });
727
728
/**
729
* One wait on the task's thread there; true once the task here is done.
@@ -976,6 +1007,14 @@ const make = Effect.gen(function* () {
1007
return RemoteDelegation.of({ delegate, followUp, cancel, start });
1008
});
1009
1010
+/** A defect, logged, as a failure, so a retry covers it. */
1011
+const failOnDefect = <A, E, R>(effect: Effect.Effect<A, E, R>) =>
1012
+ Effect.catchDefect(effect, (defect) =>
1013
+ Effect.logWarning("A remote task's follower hit a defect", Cause.die(defect)).pipe(
1014
+ Effect.andThen(Effect.fail(new Cause.UnknownError(defect, "A follower hit a defect."))),
1015
+ ),
1016
+ );
1017
+
1018
const isTerminal = (status: string) =>
1019
status === "completed" ||
1020
status === "failed" ||