The best way to control Claude Code, Codex and more from any computer or phone
Mirrored from https://github.com/Camo67/t3code.git
Synced Nov 21, 17:15 UTC
fix(server): Stop waits for the turn to settle instead of polling (#35948)
Co-authored-by: Claude Opus 5.5 <noreply@anthropic.com>
Julius Marminge committed
Nov 12, 2026 at 21:38 UTC
15d1ef5e090579c86aab1cd137b2c51f5394f15a
7 files changed
+142
-34
apps/server/src/orchestration-v2/BackgroundWorkStop.integration.test.ts
+12
-3
@@ -20,6 +20,7 @@ import * as Effect from "effect/Effect";
20
import * as Fiber from "effect/Fiber";
21
import * as Queue from "effect/Queue";
22
import * as Stream from "effect/Stream";
23
+import * as TestClock from "effect/testing/TestClock";
24
import { CodexProviderCapabilitiesV2 } from "./Adapters/CodexAdapterV2.ts";
25
import * as EffectWorker from "./EffectWorker.ts";
26
import * as EventSink from "./EventSink.ts";
@@ -56,6 +57,7 @@ const stopEarlierBackgroundWork = ({
57
const events = yield* Queue.unbounded<ProviderAdapter.ProviderAdapterV2Event>();
58
const started: ProviderAdapter.ProviderAdapterV2TurnInput[] = [];
59
const interrupts: ProviderAdapter.ProviderAdapterV2InterruptInput[] = [];
60
+ const interruptCalls = yield* Queue.unbounded<void>();
61
const adapter: ProviderAdapter.ProviderAdapterV2["Service"] = {
62
instanceId,
63
driver,
@@ -143,7 +145,7 @@ const stopEarlierBackgroundWork = ({
145
interruptTurn: (interrupt) =>
146
Effect.sync(() => {
147
interrupts.push(interrupt);
146
- }),
148
+ }).pipe(Effect.andThen(Queue.offer(interruptCalls, undefined))),
149
respondToRuntimeRequest: () => Effect.die("unused"),
150
readThreadSnapshot: () => Effect.die("unused"),
151
rollbackThread: () => Effect.die("unused"),
@@ -156,6 +158,13 @@ const stopEarlierBackgroundWork = ({
158
const worker = yield* EffectWorker.OrchestrationEffectWorkerV2;
159
const sink = yield* EventSink.EventSinkV2;
160
const threadId = ThreadId.make("thread:background-work-stop");
161
+ // This provider never confirms Stop, so Stop gives up waiting at its bound.
162
+ const drainUnconfirmedStop = Effect.gen(function* () {
163
+ const drained = yield* Effect.forkChild(worker.drain());
164
+ yield* Queue.take(interruptCalls);
165
+ yield* TestClock.adjust("2 seconds");
166
+ yield* Fiber.join(drained);
167
+ });
168
const watch = (predicate: (event: OrchestrationV2DomainEvent) => boolean) =>
169
orchestrator.streamDomainEvents.pipe(
170
Stream.filter(predicate),
@@ -219,7 +228,7 @@ const stopEarlierBackgroundWork = ({
228
assert.isTrue(followUp?.queueHeld);
229
assert.isTrue(wake?.queueHeld);
230
222
- yield* worker.drain();
231
+ yield* drainUnconfirmedStop;
232
yield* orchestrator.resumeQueuedRuns;
233
yield* worker.drain();
234
const after = yield* orchestrator.getThreadProjection(threadId);
@@ -404,7 +413,7 @@ const stopEarlierBackgroundWork = ({
413
threadId,
414
},
415
);
407
- yield* worker.drain();
416
+ yield* stalledRun === "returned-interrupt" ? drainUnconfirmedStop : worker.drain();
417
const after = yield* orchestrator.getThreadProjection(threadId);
418
const interrupted = stalledRun !== "superseded-attempt";
419
assert.equal(after.runs[0]?.status, interrupted ? "interrupted" : "running");
apps/server/src/orchestration-v2/ProjectionControlReads.test.ts
+55
-7
@@ -14,12 +14,18 @@ import {
14
RuntimeRequestId,
15
ThreadId,
16
type OrchestrationV2DomainEvent,
17
+ type OrchestrationV2StoredEvent,
18
} from "@t3tools/contracts";
19
import * as DateTime from "effect/DateTime";
20
import * as Effect from "effect/Effect";
21
+import * as Fiber from "effect/Fiber";
22
import * as Layer from "effect/Layer";
23
+import * as Queue from "effect/Queue";
24
+import * as Stream from "effect/Stream";
25
+import * as TestClock from "effect/testing/TestClock";
26
import * as SqlClient from "effect/sql/SqlClient";
27
import * as SqlitePersistence from "../persistence/Sqlite.ts";
28
+import * as EventSink from "./EventSink.ts";
29
import * as ProjectionStore from "./ProjectionStore.ts";
30
import * as ProviderSessionManager from "./ProviderSessionManager.ts";
31
import * as ProviderTurnControlService from "./ProviderTurnControlService.ts";
@@ -295,10 +301,30 @@ it.effect.each(storageCases)(
301
}),
302
} as never),
303
});
304
+ // Each pull is announced, so a second pull proves Stop judged the first event.
305
+ const pulls = yield* Queue.unbounded<void>();
306
+ const committed = yield* Queue.unbounded<OrchestrationV2StoredEvent>();
307
+ const layerEvents = Layer.mock(EventSink.EventSinkV2)({
308
+ latestSequence: () => Effect.succeed(0),
309
+ stream: () =>
310
+ Stream.fromEffectRepeat(
311
+ Queue.offer(pulls, undefined).pipe(Effect.andThen(Queue.take(committed))),
312
+ ),
313
+ });
314
yield* Effect.gen(function* () {
315
const control = yield* ProviderTurnControlService.ProviderTurnControlServiceV2;
316
const reply = yield* RuntimeRequestService.RuntimeRequestServiceV2;
301
- yield* control.interrupt({ threadId, providerThreadId, providerTurnId, providerSessionId });
317
+ const interrupt = control.interrupt({
318
+ threadId,
319
+ providerThreadId,
320
+ providerTurnId,
321
+ providerSessionId,
322
+ });
323
+ // The provider never reports the turn settled, so Stop returns at the bound.
324
+ const unconfirmed = yield* Effect.forkChild(interrupt);
325
+ yield* Queue.take(pulls);
326
+ yield* TestClock.adjust("2 seconds");
327
+ yield* Fiber.join(unconfirmed);
328
yield* control.steer({
329
threadId,
330
providerThreadId,
@@ -332,20 +358,42 @@ it.effect.each(storageCases)(
358
"Failure",
359
);
360
assert.lengthOf(calls, 3);
335
- const turn = context.providerTurn!;
336
- yield* store.apply({
361
+ const confirmed = yield* Effect.forkChild(interrupt);
362
+ yield* Queue.take(pulls);
363
+ const ingest = (sequence: number, event: OrchestrationV2DomainEvent) =>
364
+ store
365
+ .apply(event)
366
+ .pipe(Effect.andThen(Queue.offer(committed, { sequence, commandId: null, event })));
367
+ yield* ingest(1, {
368
id: EventId.make("control:turn-completed"),
369
type: "provider-turn.updated",
370
threadId,
371
occurredAt: now,
341
- payload: { ...turn, status: "completed", completedAt: now },
372
+ payload: { ...context.providerTurn!, status: "completed", completedAt: now },
373
+ });
374
+ // A settled provider turn is not enough; Stop waits for its attempt to finalize.
375
+ yield* Queue.take(pulls);
376
+ assert.isUndefined(confirmed.pollUnsafe());
377
+ yield* ingest(2, {
378
+ id: EventId.make("control:attempt-completed"),
379
+ type: "run-attempt.updated",
380
+ threadId,
381
+ occurredAt: now,
382
+ payload: { ...context.attempt!, status: "completed", completedAt: now },
383
});
343
- yield* control.interrupt({ threadId, providerThreadId, providerTurnId, providerSessionId });
344
- assert.deepEqual(calls, ["interrupt", "Use the smaller fix.", "reply", "interrupt"]);
384
+ yield* Fiber.join(confirmed);
385
+ yield* interrupt;
386
+ assert.deepEqual(calls, [
387
+ "interrupt",
388
+ "Use the smaller fix.",
389
+ "reply",
390
+ "interrupt",
391
+ "interrupt",
392
+ ]);
393
}).pipe(
394
Effect.provide(
395
Layer.merge(ProviderTurnControlService.layer, RuntimeRequestService.layer).pipe(
348
- Layer.provide(layerSessions),
396
+ Layer.provide(Layer.merge(layerSessions, layerEvents)),
397
),
398
),
399
);
apps/server/src/orchestration-v2/ProviderTurnControlService.test.ts
+8
-1
@@ -23,6 +23,7 @@ import * as Ref from "effect/Ref";
23
import * as Stream from "effect/Stream";
24
25
import { CodexProviderCapabilitiesV2 } from "./Adapters/CodexAdapterV2.ts";
26
+import * as EventSink from "./EventSink.ts";
27
import * as ProjectionStore from "./ProjectionStore.ts";
28
import type * as ProviderAdapter from "@t3tools/provider-core/server/ProviderAdapter";
29
import * as ProviderSessionManager from "./ProviderSessionManager.ts";
@@ -301,7 +302,13 @@ it.effect(
302
}),
303
);
304
const layerControl = ProviderTurnControlService.layer.pipe(
304
- Layer.provide(Layer.merge(layerProjection, layerSessionManager)),
305
+ Layer.provide(
306
+ Layer.mergeAll(
307
+ layerProjection,
308
+ layerSessionManager,
309
+ Layer.mock(EventSink.EventSinkV2)({}),
310
+ ),
311
+ ),
312
);
313
314
const [ordinaryInterrupt, unrelatedRestart] = yield* Effect.gen(function* () {
apps/server/src/orchestration-v2/ProviderTurnControlService.ts
+40
-19
@@ -9,15 +9,16 @@ import {
9
RunAttemptId,
10
ThreadId,
11
} from "@t3tools/contracts";
12
-import * as Clock from "effect/Clock";
12
import type * as DateTime from "effect/DateTime";
13
import * as Context from "effect/Context";
14
import * as Effect from "effect/Effect";
15
import * as Layer from "effect/Layer";
16
import * as Option from "effect/Option";
17
import * as Schema from "effect/Schema";
18
+import * as Stream from "effect/Stream";
19
20
import * as ProviderAdapter from "@t3tools/provider-core/server/ProviderAdapter";
21
+import * as EventSink from "./EventSink.ts";
22
import * as ProjectionStore from "./ProjectionStore.ts";
23
import * as ProviderSessionManager from "./ProviderSessionManager.ts";
24
@@ -101,10 +102,13 @@ export class ProviderTurnControlServiceV2 extends Context.Service<
102
export const layer: Layer.Layer<
103
ProviderTurnControlServiceV2,
104
never,
104
- ProjectionStore.ProjectionStoreV2 | ProviderSessionManager.ProviderSessionManagerV2
105
+ | EventSink.EventSinkV2
106
+ | ProjectionStore.ProjectionStoreV2
107
+ | ProviderSessionManager.ProviderSessionManagerV2
108
> = Layer.effect(
109
ProviderTurnControlServiceV2,
110
Effect.gen(function* () {
111
+ const eventSink = yield* EventSink.EventSinkV2;
112
const projections = yield* ProjectionStore.ProjectionStoreV2;
113
const sessions = yield* ProviderSessionManager.ProviderSessionManagerV2;
114
@@ -199,6 +203,31 @@ export const layer: Layer.Layer<
203
return { context, providerThread: interruptProviderThread, providerTurn, session };
204
});
205
206
+ const awaitSettled = Effect.fn("ProviderTurnControlService.awaitSettled")(function* (
207
+ threadId: ThreadId,
208
+ target: ProjectionStore.ProjectionProviderControlTarget,
209
+ ) {
210
+ const isSettled = projections
211
+ .getProviderControlContext(threadId, target)
212
+ .pipe(
213
+ Effect.map(
214
+ ({ providerTurn, attempt }) =>
215
+ providerTurn?.status !== "running" && attempt?.status !== "running",
216
+ ),
217
+ );
218
+ const afterSequence = yield* eventSink.latestSequence({ threadId });
219
+ if (yield* isSettled) return;
220
+ yield* eventSink.stream({ threadId, afterSequence }).pipe(
221
+ Stream.filter(
222
+ ({ event }) =>
223
+ event.type === "provider-turn.updated" || event.type === "run-attempt.updated",
224
+ ),
225
+ Stream.mapEffect(() => isSettled),
226
+ Stream.takeUntil((settled) => settled),
227
+ Stream.runDrain,
228
+ );
229
+ });
230
+
231
return ProviderTurnControlServiceV2.of({
232
applyModelSelection: Effect.fn("ProviderTurnControlService.applyModelSelection")(
233
function* (input) {
@@ -275,23 +304,15 @@ export const layer: Layer.Layer<
304
),
305
}),
306
);
278
- // Use real time; ingestion fibers do not advance TestClock and would leave Stop blocked.
279
- yield* Effect.gen(function* () {
280
- const deadline = (yield* Clock.currentTimeMillis) + 2_000;
281
- while (
282
- loaded.providerTurn.status === "running" &&
283
- (yield* Clock.currentTimeMillis) < deadline
284
- ) {
285
- const current = yield* projections.getProviderControlContext(input.threadId, input);
286
- if (
287
- current.providerTurn?.status !== "running" &&
288
- current.attempt?.status !== "running"
289
- ) {
290
- return;
291
- }
292
- yield* Effect.sleep("10 millis");
293
- }
294
- }).pipe(Effect.provideService(Clock.Clock, Clock.Clock.defaultValue()));
307
+ if (loaded.providerTurn.status === "running") {
308
+ // Let native terminal ingestion finalize the attempt before the Stop follow-up repairs it.
309
+ const { runAttemptId } = loaded.providerTurn;
310
+ yield* awaitSettled(input.threadId, {
311
+ providerThreadId: input.providerThreadId,
312
+ providerTurnId: input.providerTurnId,
313
+ ...(runAttemptId === null ? {} : { attemptId: runAttemptId }),
314
+ }).pipe(Effect.timeoutOption("2 seconds"));
315
+ }
316
},
317
(effect, input) =>
318
effect.pipe(
apps/server/src/orchestration-v2/SteeringCompletion.integration.test.ts
+18
-1
@@ -27,6 +27,7 @@ import { CodexProviderCapabilitiesV2 } from "./Adapters/CodexAdapterV2.ts";
27
import * as EffectWorker from "./EffectWorker.ts";
28
import * as EffectOutbox from "./EffectOutbox.ts";
29
import * as EventSink from "./EventSink.ts";
30
+import * as ThreadCommandExecutor from "./ThreadCommandExecutor.ts";
31
import * as Orchestrator from "./Orchestrator.ts";
32
import * as ProviderAdapterRegistry from "./ProviderAdapterRegistry.ts";
33
import * as ProviderReplayHarness from "./testkit/ProviderReplayHarness.ts";
@@ -618,7 +619,21 @@ const nextTurnSelectionHarness = Effect.fn("nextTurnSelectionHarness")(function*
619
(turn) =>
620
ProviderTurnId.make(`provider-turn:${turn.attemptId}`) === providerTurnId,
621
),
621
- (turn) => turnUpdated(turn, "interrupted"),
622
+ (turn) =>
623
+ turnUpdated(turn, "interrupted").pipe(
624
+ Effect.andThen(
625
+ Queue.offer(events, {
626
+ type: "turn.terminal",
627
+ driver,
628
+ providerThreadId: turn.providerThread.id,
629
+ providerTurnId,
630
+ runOrdinal: turn.runOrdinal,
631
+ status: "interrupted",
632
+ failure: null,
633
+ threadDisposition: "reusable",
634
+ }),
635
+ ),
636
+ ),
637
{ discard: true },
638
),
639
),
@@ -675,6 +690,8 @@ const nextTurnSelectionHarness = Effect.fn("nextTurnSelectionHarness")(function*
690
});
691
yield* worker.drain();
692
yield* Fiber.join(running);
693
+ // Ingestion checks turn ownership under this lock after committing the running turn.
694
+ yield* (yield* ThreadCommandExecutor.ThreadCommandExecutor).withLock(threadId, Effect.void);
695
return threadId;
696
});
697
return { cwd, events, started, steered, steeredSelections, interrupted, layer, startFirstTurn };
apps/server/src/orchestration-v2/runtimeLayer.ts
+3
-1
@@ -182,7 +182,9 @@ const layerProviderTurnStartServiceProvided = ProviderTurnStartService.layer.pip
182
);
183
184
const layerProviderTurnControlServiceProvided = ProviderTurnControlService.layer.pipe(
185
- Layer.provide(Layer.merge(ProjectionStore.layer, layerProviderSessionManagerProvided)),
185
+ Layer.provide(
186
+ Layer.mergeAll(layerEventSink, ProjectionStore.layer, layerProviderSessionManagerProvided),
187
+ ),
188
);
189
const layerRuntimeRequestServiceProvided = RuntimeRequestService.layer.pipe(
190
Layer.provide(Layer.merge(ProjectionStore.layer, layerProviderSessionManagerProvided)),
apps/server/src/orchestration-v2/testkit/ProviderReplayHarness.ts
+6
-2
@@ -295,7 +295,8 @@ export function layerWithRegistry<Error>(
295
| EffectWorker.OrchestrationEffectWorkerV2
296
| EventSink.EventSinkV2
297
| ProviderSessionManager.ProviderSessionManagerV2
298
- | CommandOutputRecorder.CommandOutputRecorder,
298
+ | CommandOutputRecorder.CommandOutputRecorder
299
+ | ThreadCommandExecutor.ThreadCommandExecutor,
300
Error | MigrationError | PlatformError.PlatformError | SqlError
301
> {
302
const layerServerConfig = Layer.effect(
@@ -452,7 +453,9 @@ export function layerWithRegistry<Error>(
453
),
454
);
455
const layerProviderTurnControlServiceProvided = ProviderTurnControlService.layer.pipe(
455
- Layer.provide(Layer.merge(layerStores, layerProviderSessionManagerProvided)),
456
+ Layer.provide(
457
+ Layer.mergeAll(layerEventSinkProvided, layerStores, layerProviderSessionManagerProvided),
458
+ ),
459
);
460
const layerRuntimeRequestServiceProvided = RuntimeRequestService.layer.pipe(
461
Layer.provide(Layer.merge(layerStores, layerProviderSessionManagerProvided)),
@@ -563,6 +566,7 @@ export function layerWithRegistry<Error>(
566
layerEventSinkProvided,
567
layerContinuationWorkerProvided,
568
layerCommandOutputHubProvided,
569
+ ThreadCommandExecutor.layer,
570
).pipe(
571
Layer.provide(ProviderTurnStartServiceTestkit.layer),
572
Layer.provide(Layer.mergeAll(NodeServices.layer, NodeCrypto.layer)),