@devries-cameron20 / t3code Mirror · read-only / commits / 571b319643
Fork 0

The best way to control Claude Code, Codex and more from any computer or phone

Mirrored from https://github.com/Camo67/t3code.git

Synced Nov 21, 17:15 UTC

fix(server): a remote delegated task no longer stays running when its follower hits a defect (#32331)

Julius Marminge committed Nov 13, 2026 at 10:19 UTC 571b319643611fb21db2c6ab7a0c38697b8cffd9
2 files changed +255 -17
apps/server/src/peer/RemoteDelegation.test.ts
+199
@@ -25,6 +25,7 @@ import * as Context from "effect/Context";
25 import * as DateTime from "effect/DateTime";
26 import * as Deferred from "effect/Deferred";
27 import * as Effect from "effect/Effect";
28 +import * as Fiber from "effect/Fiber";
29 import * as Layer from "effect/Layer";
30 import * as Option from "effect/Option";
31 import * as Queue from "effect/Queue";
@@ -644,6 +645,8 @@ const makeLaptop = (
645 >,
646 /** The laptop's links from before a restart, which keep their sessions there. */
647 kept?: Context.Context<Layer.Success<ReturnType<typeof layerLinkingEnvironment>>>,
648 + /** Runs as each follow attempt here starts, on its follower's fiber. */
649 + beforeFollow: Effect.Effect<void, Orchestrator.OrchestratorV2Error> = Effect.void,
650 ) =>
651 Effect.gen(function* () {
652 const linking = kept ?? (yield* layerLinkingEnvironment(laptop).pipe(Layer.build));
@@ -672,6 +675,19 @@ const makeLaptop = (
675 Layer.provideMerge(McpHttpServer.layerInProcessMcpServer),
676 Layer.provideMerge(
677 RemoteDelegation.layer.pipe(
678 + Layer.provide(
679 + Layer.effect(
680 + ThreadManagement.ThreadManagementService,
681 + Effect.gen(function* () {
682 + const threads = yield* ThreadManagement.ThreadManagementService;
683 + return {
684 + ...threads,
685 + getThreadEventSequence: (threadId: ThreadId) =>
686 + beforeFollow.pipe(Effect.andThen(threads.getThreadEventSequence(threadId))),
687 + };
688 + }),
689 + ),
690 + ),
691 Layer.provideMerge(PeerForwarding.layer),
692 Layer.provide(
693 Layer.mock(ProjectService.ProjectService)({
@@ -1784,6 +1800,108 @@ const steppedRetries = Effect.gen(function* () {
1800 return { failed, retry, schedule };
1801 });
1802
1803 +it.effect("a follower that keeps hitting defects fails the task and wakes the parent", () =>
1804 + Effect.scoped(
1805 + Effect.gen(function* () {
1806 + const thread = yield* makeBoxThread;
1807 + const b = yield* serveBox(thread);
1808 + const offers = yield* SubscriptionRef.make<
1809 + ReadonlyArray<ProviderContinuationRequests.ProviderContinuationRequest>
1810 + >([]);
1811 + const database = yield* SqlitePersistence.layerMemory.pipe(Layer.build);
1812 + const retries = yield* steppedRetries;
1813 + const attempts = yield* Ref.make(0);
1814 + // The defect carries internal detail that the task's result must not.
1815 + const beforeFollow = Ref.update(attempts, (count) => count + 1).pipe(
1816 + Effect.andThen(Effect.die(new Error("A follow attempt hit a bug: defect-detail-4f1c9e"))),
1817 + );
1818 + const a = yield* makeLaptop(database, offers, undefined, beforeFollow).pipe(
1819 + Effect.provideService(RemoteDelegation.FollowRetrySchedule, retries.schedule),
1820 + );
1821 + yield* linkTo(a.links, b, "full-access");
1822 + yield* seedParent(a);
1823 + yield* Queue.take(retries.failed).pipe(
1824 + Effect.andThen(Queue.offer(retries.retry, undefined)),
1825 + Effect.forever,
1826 + Effect.forkScoped,
1827 + );
1828 +
1829 + const delegated = yield* delegateToBox(a);
1830 + const task = yield* taskResult(a, delegated.taskId);
1831 + expect(yield* Ref.get(attempts)).toBe(3);
1832 + expect(task?.status).toBe("failed");
1833 + expect(task?.result).toContain(delegated.remoteChild.threadId);
1834 + expect(task?.result).not.toContain("defect-detail-4f1c9e");
1835 + const offered = yield* SubscriptionRef.changes(offers).pipe(
1836 + Stream.filter((seen) => seen.some((offer) => offer.threadId === parentThreadId)),
1837 + Stream.runHead,
1838 + );
1839 + expect(Option.isSome(offered)).toBe(true);
1840 + }),
1841 + ).pipe(Effect.provide(NodeHttpServer.layerTest)),
1842 +);
1843 +
1844 +it.effect.each(["a still-running wait", "a failed attempt"] as const)(
1845 + "defects separated by %s do not add up to giving up",
1846 + (between) =>
1847 + Effect.scoped(
1848 + Effect.gen(function* () {
1849 + const thread = yield* makeBoxThread;
1850 + const b = yield* serveBox(thread);
1851 + const offers = yield* SubscriptionRef.make<
1852 + ReadonlyArray<ProviderContinuationRequests.ProviderContinuationRequest>
1853 + >([]);
1854 + const database = yield* SqlitePersistence.layerMemory.pipe(Layer.build);
1855 + const retries = yield* steppedRetries;
1856 + const attempts = yield* Ref.make(0);
1857 + const defect = Effect.die(new Error("A follow attempt hit a bug."));
1858 + const script = [
1859 + defect,
1860 + between === "a still-running wait"
1861 + ? Effect.void
1862 + : Effect.fail(
1863 + new Orchestrator.OrchestratorProjectionError({ threadId: parentThreadId }),
1864 + ),
1865 + defect,
1866 + between === "a still-running wait"
1867 + ? Effect.void
1868 + : Effect.fail(
1869 + new Orchestrator.OrchestratorProjectionError({ threadId: parentThreadId }),
1870 + ),
1871 + defect,
1872 + ];
1873 + // Past the script, the run there has ended, so the next attempt completes the task.
1874 + const beforeFollow = Ref.getAndUpdate(attempts, (count) => count + 1).pipe(
1875 + Effect.flatMap(
1876 + (count) =>
1877 + script[count] ??
1878 + Ref.set(thread.reply, "All green.").pipe(
1879 + Effect.andThen(Ref.set(thread.status, "completed")),
1880 + ),
1881 + ),
1882 + );
1883 + // Each wait there answers at once that its run is still running.
1884 + yield* Deferred.succeed(thread.finished, undefined);
1885 + const a = yield* makeLaptop(database, offers, undefined, beforeFollow).pipe(
1886 + Effect.provideService(RemoteDelegation.FollowRetrySchedule, retries.schedule),
1887 + );
1888 + yield* linkTo(a.links, b, "full-access");
1889 + yield* seedParent(a);
1890 + yield* Queue.take(retries.failed).pipe(
1891 + Effect.andThen(Queue.offer(retries.retry, undefined)),
1892 + Effect.forever,
1893 + Effect.forkScoped,
1894 + );
1895 +
1896 + const delegated = yield* delegateToBox(a);
1897 + expect(yield* taskResult(a, delegated.taskId)).toMatchObject({
1898 + status: "completed",
1899 + result: "All green.",
1900 + });
1901 + }),
1902 + ).pipe(Effect.provide(NodeHttpServer.layerTest)),
1903 +);
1904 +
1905 /** Waits until the task here records that the `stopCount`th Stop after its result reached there. */
1906 const stoppedThere = (
1907 laptopEnv: Effect.Success<ReturnType<typeof makeLaptop>>,
@@ -1813,6 +1931,87 @@ const stoppedThere = (
1931 ),
1932 );
1933
1934 +it.effect("a follower that keeps hitting defects leaves a stop it owes owed", () =>
1935 + Effect.scoped(
1936 + Effect.gen(function* () {
1937 + const thread = yield* makeBoxThread;
1938 + const b = yield* serveBox(thread);
1939 + const offers = yield* SubscriptionRef.make<
1940 + ReadonlyArray<ProviderContinuationRequests.ProviderContinuationRequest>
1941 + >([]);
1942 + const database = yield* SqlitePersistence.layerMemory.pipe(Layer.build);
1943 + const retries = yield* steppedRetries;
1944 + // Follow attempts hit a defect only once the task has its result.
1945 + const broken = yield* Ref.make(false);
1946 + const follower = yield* Deferred.make<Fiber.Fiber<unknown, unknown>>();
1947 + const beforeFollow = Ref.get(broken).pipe(
1948 + Effect.flatMap((isBroken) =>
1949 + isBroken
1950 + ? Effect.withFiber((fiber) => Deferred.succeed(follower, fiber)).pipe(
1951 + Effect.andThen(Effect.die(new Error("A follow attempt hit a bug."))),
1952 + )
1953 + : Effect.void,
1954 + ),
1955 + );
1956 + const a = yield* makeLaptop(database, offers, undefined, beforeFollow).pipe(
1957 + Effect.provideService(RemoteDelegation.FollowRetrySchedule, retries.schedule),
1958 + );
1959 + yield* a.remote.start();
1960 + yield* linkTo(a.links, b, "full-access");
1961 + yield* seedParent(a);
1962 + const delegated = yield* delegateToBox(a);
1963 + yield* Ref.set(thread.reply, "All green.");
1964 + yield* Ref.set(thread.status, "completed");
1965 + yield* Deferred.succeed(thread.finished, undefined);
1966 + yield* taskResult(a, delegated.taskId);
1967 + yield* Ref.set(broken, true);
1968 + yield* Queue.take(retries.failed).pipe(
1969 + Effect.andThen(Queue.offer(retries.retry, undefined)),
1970 + Effect.forever,
1971 + Effect.forkScoped,
1972 + );
1973 +
1974 + // The Stop owes one stop there, which every attempt to send fails on.
1975 + yield* a.threads.stopDelegatedTasks({
1976 + threadId: parentThreadId,
1977 + commandId: CommandId.make("command:stop:laptop-parent-defective-follower"),
1978 + });
1979 + const owedAgain = a.orchestrator
1980 + .streamStoredEventsFrom({
1981 + threadId: parentThreadId,
1982 + afterSequence: 0,
1983 + eventType: "subagent.updated",
1984 + })
1985 + .pipe(
1986 + Stream.filter(
1987 + (stored) =>
1988 + stored.event.type === "subagent.updated" &&
1989 + stored.event.payload.id === delegated.taskId &&
1990 + (stored.event.payload.remoteChild?.stopCount ?? 0) > 1,
1991 + ),
1992 + Stream.runHead,
1993 + Effect.flatMap((owed) =>
1994 + Option.isSome(owed) ? Effect.succeed("owed another stop" as const) : Effect.never,
1995 + ),
1996 + );
1997 + const ended = yield* Deferred.await(follower).pipe(
1998 + Effect.flatMap(Fiber.await),
1999 + Effect.as("follower ended" as const),
2000 + Effect.raceFirst(owedAgain),
2001 + );
2002 + expect(ended).toBe("follower ended");
2003 + const { subagents } = yield* a.threads.getThreadRecords(parentThreadId, ["subagents"], {
2004 + subagentIds: [delegated.taskId],
2005 + });
2006 + expect(subagents[0]).toMatchObject({
2007 + status: "completed",
2008 + result: "All green.",
2009 + remoteChild: { stoppedThere: false, stopCount: 1 },
2010 + });
2011 + }),
2012 + ).pipe(Effect.provide(NodeHttpServer.layerTest)),
2013 +);
2014 +
2015 it.effect("stopping the parent stops follow-up work on a task there that has its result", () =>
2016 Effect.scoped(
2017 Effect.gen(function* () {
apps/server/src/peer/RemoteDelegation.ts
+56 -17
@@ -19,6 +19,7 @@ import {
19 type RunId,
20 type ThreadId,
21 } from "@t3tools/contracts";
22 +import * as Cause from "effect/Cause";
23 import * as Context from "effect/Context";
24 import * as SecureRandom from "effect/crypto/SecureRandom";
25 import * as DateTime from "effect/DateTime";
@@ -53,6 +54,8 @@ import * as PeerLinks from "./PeerLinks.ts";
54 const FOLLOW_WAIT_MS = 10 * 60 * 1_000;
55 /** A linked environment that stops answering is retried, backing off to this. */
56 const FOLLOW_MAX_BACKOFF = Duration.minutes(5);
57 +/** A follower that hits this many defects in a row fails its task instead of retrying. */
58 +const FOLLOW_MAX_DEFECTS = 3;
59 /** How a follower retries; tests step it by hand. */
60 export const FollowRetrySchedule = Context.Reference<Schedule.Schedule<unknown>>(
61 "t3/peer/RemoteDelegation/FollowRetrySchedule",
@@ -673,26 +676,54 @@ const make = Effect.gen(function* () {
676 /**
677 * Waits for the task's run there to end, completes the task here, and stops it there if it
678 * ended here first. A revoked, expired or forgotten link, or a thread or run gone there,
676 - * fails the task with nothing left to stop; anything else (the other machine asleep, a
677 - * dropped route) is retried with back-off, and so is completing.
679 + * fails the task with nothing left to stop; repeated defects here fail it too. Anything else
680 + * (the other machine asleep, a dropped route) is retried with back-off, and so is completing.
681 */
682 const follow = (parentThreadId: ThreadId, taskId: NodeId): Effect.Effect<void> =>
680 - followOnce(parentThreadId, taskId).pipe(
681 - // `false` means it is still running there: follow again straight away.
682 - Effect.repeat({ while: (done) => !done }),
683 - Effect.catchTags({
684 - OrchestratorMcpFailure: Effect.fn("RemoteDelegation.followFailed")(function* (error) {
685 - const reason = yield* finalReason(parentThreadId, taskId, error);
686 - if (reason === undefined) return yield* error;
687 - // Gone there, or no link to reach it by: nothing left to stop there.
688 - const { task } = yield* readTask(parentThreadId, taskId);
689 - const stopCount = task?.remoteChild?.stopCount;
690 - yield* complete(parentThreadId, taskId, "failed", reason, "stopped", stopCount);
683 + Effect.suspend(() => {
684 + let defects = 0;
685 + const noDefect = Effect.sync(() => {
686 + defects = 0;
687 + });
688 + return followOnce(parentThreadId, taskId).pipe(
689 + Effect.tap(noDefect),
690 + // `false` means it is still running there: follow again straight away.
691 + Effect.repeat({ while: (done) => !done }),
692 + Effect.catchTags({
693 + OrchestratorMcpFailure: Effect.fn("RemoteDelegation.followFailed")(function* (error) {
694 + const reason = yield* finalReason(parentThreadId, taskId, error);
695 + if (reason === undefined) return yield* error;
696 + // Gone there, or no link to reach it by: nothing left to stop there.
697 + const { task } = yield* readTask(parentThreadId, taskId);
698 + const stopCount = task?.remoteChild?.stopCount;
699 + yield* complete(parentThreadId, taskId, "failed", reason, "stopped", stopCount);
700 + }),
701 }),
692 - }),
693 - Effect.retry(followRetry),
694 - Effect.ignoreCause({ log: true }),
695 - );
702 + Effect.tapError(() => noDefect),
703 + Effect.tapDefect(() =>
704 + Effect.sync(() => {
705 + defects += 1;
706 + }),
707 + ),
708 + failOnDefect,
709 + Effect.retry({ schedule: followRetry, while: () => defects < FOLLOW_MAX_DEFECTS }),
710 + Effect.catch(
711 + Effect.fn("RemoteDelegation.followGaveUp")(
712 + function* () {
713 + const { task } = yield* readTask(parentThreadId, taskId);
714 + const remote = task?.remoteChild;
715 + // A task with its result keeps it; completing it again would owe another stop there.
716 + if (remote === undefined || task?.result !== null) return;
717 + const result = `Following the thread ${remote.threadId} in ${remote.label} failed repeatedly, so it may still run there (task_cancel stops it).`;
718 + yield* complete(parentThreadId, taskId, "failed", result, undefined);
719 + },
720 + failOnDefect,
721 + Effect.retry(followRetry),
722 + ),
723 + ),
724 + Effect.ignoreCause({ log: true }),
725 + );
726 + });
727
728 /**
729 * One wait on the task's thread there; true once the task here is done.
@@ -976,6 +1007,14 @@ const make = Effect.gen(function* () {
1007 return RemoteDelegation.of({ delegate, followUp, cancel, start });
1008 });
1009
1010 +/** A defect, logged, as a failure, so a retry covers it. */
1011 +const failOnDefect = <A, E, R>(effect: Effect.Effect<A, E, R>) =>
1012 + Effect.catchDefect(effect, (defect) =>
1013 + Effect.logWarning("A remote task's follower hit a defect", Cause.die(defect)).pipe(
1014 + Effect.andThen(Effect.fail(new Cause.UnknownError(defect, "A follower hit a defect."))),
1015 + ),
1016 + );
1017 +
1018 const isTerminal = (status: string) =>
1019 status === "completed" ||
1020 status === "failed" ||