@devries-cameron20 / t3code-1 / commits / 15d1ef5e09
Fork 0

The best way to control Claude Code, Codex and more from any computer or phone

fix(server): Stop waits for the turn to settle instead of polling (#35948)

Co-authored-by: Claude Opus 5.5 <noreply@anthropic.com>

Julius Marminge committed Nov 12, 2026 at 21:38 UTC 15d1ef5e090579c86aab1cd137b2c51f5394f15a
7 files changed +142 -34
apps/server/src/orchestration-v2/BackgroundWorkStop.integration.test.ts
+12 -3
@@ -20,6 +20,7 @@ import * as Effect from "effect/Effect";
20 import * as Fiber from "effect/Fiber";
21 import * as Queue from "effect/Queue";
22 import * as Stream from "effect/Stream";
23 +import * as TestClock from "effect/testing/TestClock";
24 import { CodexProviderCapabilitiesV2 } from "./Adapters/CodexAdapterV2.ts";
25 import * as EffectWorker from "./EffectWorker.ts";
26 import * as EventSink from "./EventSink.ts";
@@ -56,6 +57,7 @@ const stopEarlierBackgroundWork = ({
57 const events = yield* Queue.unbounded<ProviderAdapter.ProviderAdapterV2Event>();
58 const started: ProviderAdapter.ProviderAdapterV2TurnInput[] = [];
59 const interrupts: ProviderAdapter.ProviderAdapterV2InterruptInput[] = [];
60 + const interruptCalls = yield* Queue.unbounded<void>();
61 const adapter: ProviderAdapter.ProviderAdapterV2["Service"] = {
62 instanceId,
63 driver,
@@ -143,7 +145,7 @@ const stopEarlierBackgroundWork = ({
145 interruptTurn: (interrupt) =>
146 Effect.sync(() => {
147 interrupts.push(interrupt);
146 - }),
148 + }).pipe(Effect.andThen(Queue.offer(interruptCalls, undefined))),
149 respondToRuntimeRequest: () => Effect.die("unused"),
150 readThreadSnapshot: () => Effect.die("unused"),
151 rollbackThread: () => Effect.die("unused"),
@@ -156,6 +158,13 @@ const stopEarlierBackgroundWork = ({
158 const worker = yield* EffectWorker.OrchestrationEffectWorkerV2;
159 const sink = yield* EventSink.EventSinkV2;
160 const threadId = ThreadId.make("thread:background-work-stop");
161 + // This provider never confirms Stop, so Stop gives up waiting at its bound.
162 + const drainUnconfirmedStop = Effect.gen(function* () {
163 + const drained = yield* Effect.forkChild(worker.drain());
164 + yield* Queue.take(interruptCalls);
165 + yield* TestClock.adjust("2 seconds");
166 + yield* Fiber.join(drained);
167 + });
168 const watch = (predicate: (event: OrchestrationV2DomainEvent) => boolean) =>
169 orchestrator.streamDomainEvents.pipe(
170 Stream.filter(predicate),
@@ -219,7 +228,7 @@ const stopEarlierBackgroundWork = ({
228 assert.isTrue(followUp?.queueHeld);
229 assert.isTrue(wake?.queueHeld);
230
222 - yield* worker.drain();
231 + yield* drainUnconfirmedStop;
232 yield* orchestrator.resumeQueuedRuns;
233 yield* worker.drain();
234 const after = yield* orchestrator.getThreadProjection(threadId);
@@ -404,7 +413,7 @@ const stopEarlierBackgroundWork = ({
413 threadId,
414 },
415 );
407 - yield* worker.drain();
416 + yield* stalledRun === "returned-interrupt" ? drainUnconfirmedStop : worker.drain();
417 const after = yield* orchestrator.getThreadProjection(threadId);
418 const interrupted = stalledRun !== "superseded-attempt";
419 assert.equal(after.runs[0]?.status, interrupted ? "interrupted" : "running");
apps/server/src/orchestration-v2/ProjectionControlReads.test.ts
+55 -7
@@ -14,12 +14,18 @@ import {
14 RuntimeRequestId,
15 ThreadId,
16 type OrchestrationV2DomainEvent,
17 + type OrchestrationV2StoredEvent,
18 } from "@t3tools/contracts";
19 import * as DateTime from "effect/DateTime";
20 import * as Effect from "effect/Effect";
21 +import * as Fiber from "effect/Fiber";
22 import * as Layer from "effect/Layer";
23 +import * as Queue from "effect/Queue";
24 +import * as Stream from "effect/Stream";
25 +import * as TestClock from "effect/testing/TestClock";
26 import * as SqlClient from "effect/sql/SqlClient";
27 import * as SqlitePersistence from "../persistence/Sqlite.ts";
28 +import * as EventSink from "./EventSink.ts";
29 import * as ProjectionStore from "./ProjectionStore.ts";
30 import * as ProviderSessionManager from "./ProviderSessionManager.ts";
31 import * as ProviderTurnControlService from "./ProviderTurnControlService.ts";
@@ -295,10 +301,30 @@ it.effect.each(storageCases)(
301 }),
302 } as never),
303 });
304 + // Each pull is announced, so a second pull proves Stop judged the first event.
305 + const pulls = yield* Queue.unbounded<void>();
306 + const committed = yield* Queue.unbounded<OrchestrationV2StoredEvent>();
307 + const layerEvents = Layer.mock(EventSink.EventSinkV2)({
308 + latestSequence: () => Effect.succeed(0),
309 + stream: () =>
310 + Stream.fromEffectRepeat(
311 + Queue.offer(pulls, undefined).pipe(Effect.andThen(Queue.take(committed))),
312 + ),
313 + });
314 yield* Effect.gen(function* () {
315 const control = yield* ProviderTurnControlService.ProviderTurnControlServiceV2;
316 const reply = yield* RuntimeRequestService.RuntimeRequestServiceV2;
301 - yield* control.interrupt({ threadId, providerThreadId, providerTurnId, providerSessionId });
317 + const interrupt = control.interrupt({
318 + threadId,
319 + providerThreadId,
320 + providerTurnId,
321 + providerSessionId,
322 + });
323 + // The provider never reports the turn settled, so Stop returns at the bound.
324 + const unconfirmed = yield* Effect.forkChild(interrupt);
325 + yield* Queue.take(pulls);
326 + yield* TestClock.adjust("2 seconds");
327 + yield* Fiber.join(unconfirmed);
328 yield* control.steer({
329 threadId,
330 providerThreadId,
@@ -332,20 +358,42 @@ it.effect.each(storageCases)(
358 "Failure",
359 );
360 assert.lengthOf(calls, 3);
335 - const turn = context.providerTurn!;
336 - yield* store.apply({
361 + const confirmed = yield* Effect.forkChild(interrupt);
362 + yield* Queue.take(pulls);
363 + const ingest = (sequence: number, event: OrchestrationV2DomainEvent) =>
364 + store
365 + .apply(event)
366 + .pipe(Effect.andThen(Queue.offer(committed, { sequence, commandId: null, event })));
367 + yield* ingest(1, {
368 id: EventId.make("control:turn-completed"),
369 type: "provider-turn.updated",
370 threadId,
371 occurredAt: now,
341 - payload: { ...turn, status: "completed", completedAt: now },
372 + payload: { ...context.providerTurn!, status: "completed", completedAt: now },
373 + });
374 + // A settled provider turn is not enough; Stop waits for its attempt to finalize.
375 + yield* Queue.take(pulls);
376 + assert.isUndefined(confirmed.pollUnsafe());
377 + yield* ingest(2, {
378 + id: EventId.make("control:attempt-completed"),
379 + type: "run-attempt.updated",
380 + threadId,
381 + occurredAt: now,
382 + payload: { ...context.attempt!, status: "completed", completedAt: now },
383 });
343 - yield* control.interrupt({ threadId, providerThreadId, providerTurnId, providerSessionId });
344 - assert.deepEqual(calls, ["interrupt", "Use the smaller fix.", "reply", "interrupt"]);
384 + yield* Fiber.join(confirmed);
385 + yield* interrupt;
386 + assert.deepEqual(calls, [
387 + "interrupt",
388 + "Use the smaller fix.",
389 + "reply",
390 + "interrupt",
391 + "interrupt",
392 + ]);
393 }).pipe(
394 Effect.provide(
395 Layer.merge(ProviderTurnControlService.layer, RuntimeRequestService.layer).pipe(
348 - Layer.provide(layerSessions),
396 + Layer.provide(Layer.merge(layerSessions, layerEvents)),
397 ),
398 ),
399 );
apps/server/src/orchestration-v2/ProviderTurnControlService.test.ts
+8 -1
@@ -23,6 +23,7 @@ import * as Ref from "effect/Ref";
23 import * as Stream from "effect/Stream";
24
25 import { CodexProviderCapabilitiesV2 } from "./Adapters/CodexAdapterV2.ts";
26 +import * as EventSink from "./EventSink.ts";
27 import * as ProjectionStore from "./ProjectionStore.ts";
28 import type * as ProviderAdapter from "@t3tools/provider-core/server/ProviderAdapter";
29 import * as ProviderSessionManager from "./ProviderSessionManager.ts";
@@ -301,7 +302,13 @@ it.effect(
302 }),
303 );
304 const layerControl = ProviderTurnControlService.layer.pipe(
304 - Layer.provide(Layer.merge(layerProjection, layerSessionManager)),
305 + Layer.provide(
306 + Layer.mergeAll(
307 + layerProjection,
308 + layerSessionManager,
309 + Layer.mock(EventSink.EventSinkV2)({}),
310 + ),
311 + ),
312 );
313
314 const [ordinaryInterrupt, unrelatedRestart] = yield* Effect.gen(function* () {
apps/server/src/orchestration-v2/ProviderTurnControlService.ts
+40 -19
@@ -9,15 +9,16 @@ import {
9 RunAttemptId,
10 ThreadId,
11 } from "@t3tools/contracts";
12 -import * as Clock from "effect/Clock";
12 import type * as DateTime from "effect/DateTime";
13 import * as Context from "effect/Context";
14 import * as Effect from "effect/Effect";
15 import * as Layer from "effect/Layer";
16 import * as Option from "effect/Option";
17 import * as Schema from "effect/Schema";
18 +import * as Stream from "effect/Stream";
19
20 import * as ProviderAdapter from "@t3tools/provider-core/server/ProviderAdapter";
21 +import * as EventSink from "./EventSink.ts";
22 import * as ProjectionStore from "./ProjectionStore.ts";
23 import * as ProviderSessionManager from "./ProviderSessionManager.ts";
24
@@ -101,10 +102,13 @@ export class ProviderTurnControlServiceV2 extends Context.Service<
102 export const layer: Layer.Layer<
103 ProviderTurnControlServiceV2,
104 never,
104 - ProjectionStore.ProjectionStoreV2 | ProviderSessionManager.ProviderSessionManagerV2
105 + | EventSink.EventSinkV2
106 + | ProjectionStore.ProjectionStoreV2
107 + | ProviderSessionManager.ProviderSessionManagerV2
108 > = Layer.effect(
109 ProviderTurnControlServiceV2,
110 Effect.gen(function* () {
111 + const eventSink = yield* EventSink.EventSinkV2;
112 const projections = yield* ProjectionStore.ProjectionStoreV2;
113 const sessions = yield* ProviderSessionManager.ProviderSessionManagerV2;
114
@@ -199,6 +203,31 @@ export const layer: Layer.Layer<
203 return { context, providerThread: interruptProviderThread, providerTurn, session };
204 });
205
206 + const awaitSettled = Effect.fn("ProviderTurnControlService.awaitSettled")(function* (
207 + threadId: ThreadId,
208 + target: ProjectionStore.ProjectionProviderControlTarget,
209 + ) {
210 + const isSettled = projections
211 + .getProviderControlContext(threadId, target)
212 + .pipe(
213 + Effect.map(
214 + ({ providerTurn, attempt }) =>
215 + providerTurn?.status !== "running" && attempt?.status !== "running",
216 + ),
217 + );
218 + const afterSequence = yield* eventSink.latestSequence({ threadId });
219 + if (yield* isSettled) return;
220 + yield* eventSink.stream({ threadId, afterSequence }).pipe(
221 + Stream.filter(
222 + ({ event }) =>
223 + event.type === "provider-turn.updated" || event.type === "run-attempt.updated",
224 + ),
225 + Stream.mapEffect(() => isSettled),
226 + Stream.takeUntil((settled) => settled),
227 + Stream.runDrain,
228 + );
229 + });
230 +
231 return ProviderTurnControlServiceV2.of({
232 applyModelSelection: Effect.fn("ProviderTurnControlService.applyModelSelection")(
233 function* (input) {
@@ -275,23 +304,15 @@ export const layer: Layer.Layer<
304 ),
305 }),
306 );
278 - // Use real time; ingestion fibers do not advance TestClock and would leave Stop blocked.
279 - yield* Effect.gen(function* () {
280 - const deadline = (yield* Clock.currentTimeMillis) + 2_000;
281 - while (
282 - loaded.providerTurn.status === "running" &&
283 - (yield* Clock.currentTimeMillis) < deadline
284 - ) {
285 - const current = yield* projections.getProviderControlContext(input.threadId, input);
286 - if (
287 - current.providerTurn?.status !== "running" &&
288 - current.attempt?.status !== "running"
289 - ) {
290 - return;
291 - }
292 - yield* Effect.sleep("10 millis");
293 - }
294 - }).pipe(Effect.provideService(Clock.Clock, Clock.Clock.defaultValue()));
307 + if (loaded.providerTurn.status === "running") {
308 + // Let native terminal ingestion finalize the attempt before the Stop follow-up repairs it.
309 + const { runAttemptId } = loaded.providerTurn;
310 + yield* awaitSettled(input.threadId, {
311 + providerThreadId: input.providerThreadId,
312 + providerTurnId: input.providerTurnId,
313 + ...(runAttemptId === null ? {} : { attemptId: runAttemptId }),
314 + }).pipe(Effect.timeoutOption("2 seconds"));
315 + }
316 },
317 (effect, input) =>
318 effect.pipe(
apps/server/src/orchestration-v2/SteeringCompletion.integration.test.ts
+18 -1
@@ -27,6 +27,7 @@ import { CodexProviderCapabilitiesV2 } from "./Adapters/CodexAdapterV2.ts";
27 import * as EffectWorker from "./EffectWorker.ts";
28 import * as EffectOutbox from "./EffectOutbox.ts";
29 import * as EventSink from "./EventSink.ts";
30 +import * as ThreadCommandExecutor from "./ThreadCommandExecutor.ts";
31 import * as Orchestrator from "./Orchestrator.ts";
32 import * as ProviderAdapterRegistry from "./ProviderAdapterRegistry.ts";
33 import * as ProviderReplayHarness from "./testkit/ProviderReplayHarness.ts";
@@ -618,7 +619,21 @@ const nextTurnSelectionHarness = Effect.fn("nextTurnSelectionHarness")(function*
619 (turn) =>
620 ProviderTurnId.make(`provider-turn:${turn.attemptId}`) === providerTurnId,
621 ),
621 - (turn) => turnUpdated(turn, "interrupted"),
622 + (turn) =>
623 + turnUpdated(turn, "interrupted").pipe(
624 + Effect.andThen(
625 + Queue.offer(events, {
626 + type: "turn.terminal",
627 + driver,
628 + providerThreadId: turn.providerThread.id,
629 + providerTurnId,
630 + runOrdinal: turn.runOrdinal,
631 + status: "interrupted",
632 + failure: null,
633 + threadDisposition: "reusable",
634 + }),
635 + ),
636 + ),
637 { discard: true },
638 ),
639 ),
@@ -675,6 +690,8 @@ const nextTurnSelectionHarness = Effect.fn("nextTurnSelectionHarness")(function*
690 });
691 yield* worker.drain();
692 yield* Fiber.join(running);
693 + // Ingestion checks turn ownership under this lock after committing the running turn.
694 + yield* (yield* ThreadCommandExecutor.ThreadCommandExecutor).withLock(threadId, Effect.void);
695 return threadId;
696 });
697 return { cwd, events, started, steered, steeredSelections, interrupted, layer, startFirstTurn };
apps/server/src/orchestration-v2/runtimeLayer.ts
+3 -1
@@ -182,7 +182,9 @@ const layerProviderTurnStartServiceProvided = ProviderTurnStartService.layer.pip
182 );
183
184 const layerProviderTurnControlServiceProvided = ProviderTurnControlService.layer.pipe(
185 - Layer.provide(Layer.merge(ProjectionStore.layer, layerProviderSessionManagerProvided)),
185 + Layer.provide(
186 + Layer.mergeAll(layerEventSink, ProjectionStore.layer, layerProviderSessionManagerProvided),
187 + ),
188 );
189 const layerRuntimeRequestServiceProvided = RuntimeRequestService.layer.pipe(
190 Layer.provide(Layer.merge(ProjectionStore.layer, layerProviderSessionManagerProvided)),
apps/server/src/orchestration-v2/testkit/ProviderReplayHarness.ts
+6 -2
@@ -295,7 +295,8 @@ export function layerWithRegistry<Error>(
295 | EffectWorker.OrchestrationEffectWorkerV2
296 | EventSink.EventSinkV2
297 | ProviderSessionManager.ProviderSessionManagerV2
298 - | CommandOutputRecorder.CommandOutputRecorder,
298 + | CommandOutputRecorder.CommandOutputRecorder
299 + | ThreadCommandExecutor.ThreadCommandExecutor,
300 Error | MigrationError | PlatformError.PlatformError | SqlError
301 > {
302 const layerServerConfig = Layer.effect(
@@ -452,7 +453,9 @@ export function layerWithRegistry<Error>(
453 ),
454 );
455 const layerProviderTurnControlServiceProvided = ProviderTurnControlService.layer.pipe(
455 - Layer.provide(Layer.merge(layerStores, layerProviderSessionManagerProvided)),
456 + Layer.provide(
457 + Layer.mergeAll(layerEventSinkProvided, layerStores, layerProviderSessionManagerProvided),
458 + ),
459 );
460 const layerRuntimeRequestServiceProvided = RuntimeRequestService.layer.pipe(
461 Layer.provide(Layer.merge(layerStores, layerProviderSessionManagerProvided)),
@@ -563,6 +566,7 @@ export function layerWithRegistry<Error>(
566 layerEventSinkProvided,
567 layerContinuationWorkerProvided,
568 layerCommandOutputHubProvided,
569 + ThreadCommandExecutor.layer,
570 ).pipe(
571 Layer.provide(ProviderTurnStartServiceTestkit.layer),
572 Layer.provide(Layer.mergeAll(NodeServices.layer, NodeCrypto.layer)),