The best way to control Claude Code, Codex and more from any computer or phone
Mirrored from https://github.com/Camo67/t3code.git
Synced Nov 21, 04:45 UTC
| 1 | import * as DocumentText from "../provider/DocumentText.ts"; |
| 2 | import { NodeHttpServer } from "@effect/platform-node"; |
| 3 | import * as NodeCrypto from "@effect/platform-node/NodeCrypto"; |
| 4 | import * as NodeServices from "@effect/platform-node/NodeServices"; |
| 5 | import { |
| 6 | CommandId, |
| 7 | EventId, |
| 8 | MessageId, |
| 9 | type ModelSelection, |
| 10 | NodeId, |
| 11 | type OrchestrationV2Run, |
| 12 | type OrchestratorMcpThreadReadInput, |
| 13 | ProjectId, |
| 14 | ProviderDriverKind, |
| 15 | ProviderInstanceId, |
| 16 | ProviderThreadId, |
| 17 | type Project, |
| 18 | type RepositoryIdentity, |
| 19 | RunId, |
| 20 | type ServerProvider, |
| 21 | ThreadId, |
| 22 | TurnItemId, |
| 23 | } from "@t3tools/contracts"; |
| 24 | import { expect, it } from "@effect/vitest"; |
| 25 | import * as Context from "effect/Context"; |
| 26 | import * as DateTime from "effect/DateTime"; |
| 27 | import * as Deferred from "effect/Deferred"; |
| 28 | import * as Effect from "effect/Effect"; |
| 29 | import * as Fiber from "effect/Fiber"; |
| 30 | import * as Layer from "effect/Layer"; |
| 31 | import * as Option from "effect/Option"; |
| 32 | import * as Queue from "effect/Queue"; |
| 33 | import * as Ref from "effect/Ref"; |
| 34 | import * as Schedule from "effect/Schedule"; |
| 35 | import * as Stream from "effect/Stream"; |
| 36 | import * as SubscriptionRef from "effect/SubscriptionRef"; |
| 37 | import * as Tracer from "effect/Tracer"; |
| 38 | import * as FetchHttpClient from "effect/http/FetchHttpClient"; |
| 39 | import * as TestClock from "effect/testing/TestClock"; |
| 40 | import type * as SqlClient from "effect/sql/SqlClient"; |
| 41 | import { McpSchema, McpServer } from "effect/ai"; |
| 42 | |
| 43 | import * as ServerConfig from "../config.ts"; |
| 44 | import * as McpHttpServer from "../mcp/McpHttpServer.ts"; |
| 45 | import * as McpInvocationContext from "../mcp/McpInvocationContext.ts"; |
| 46 | import { idleThreadProjection, liveThreadShell } from "../mcp/McpToolAccess.testkit.ts"; |
| 47 | import { CodexProviderCapabilitiesV2 } from "../orchestration-v2/Adapters/CodexAdapterV2.ts"; |
| 48 | import * as EventSink from "../orchestration-v2/EventSink.ts"; |
| 49 | import * as Orchestrator from "../orchestration-v2/Orchestrator.ts"; |
| 50 | import * as ProjectionStore from "../orchestration-v2/ProjectionStore.ts"; |
| 51 | import type * as ProviderAdapter from "@t3tools/provider-core/server/ProviderAdapter"; |
| 52 | import * as ProviderAdapterRegistry from "../orchestration-v2/ProviderAdapterRegistry.ts"; |
| 53 | import * as ProviderContinuationRequests from "@t3tools/provider-core/server/ProviderContinuationRequests"; |
| 54 | import * as ThreadLaunch from "../orchestration-v2/ThreadLaunchService.ts"; |
| 55 | import * as ThreadManagement from "../orchestration-v2/ThreadManagementService.ts"; |
| 56 | import * as ProviderReplayHarness from "../orchestration-v2/testkit/ProviderReplayHarness.ts"; |
| 57 | import * as SqlitePersistence from "../persistence/Sqlite.ts"; |
| 58 | import * as ManagedProjectFolders from "../project/ManagedProjectFolders.ts"; |
| 59 | import * as ProjectCloneTracker from "../project/ProjectCloneTracker.ts"; |
| 60 | import * as ProjectService from "../project/ProjectService.ts"; |
| 61 | import * as RepositoryIdentityResolver from "../project/RepositoryIdentityResolver.ts"; |
| 62 | import * as ProviderRegistry from "../provider/ProviderRegistry.ts"; |
| 63 | import * as ScheduledTaskService from "../scheduledTasks/ScheduledTaskService.ts"; |
| 64 | import * as SecretRequests from "../secrets/SecretRequests.ts"; |
| 65 | import * as ServerSettings from "../serverSettings.ts"; |
| 66 | import * as CommandReceiptStore from "../orchestration-v2/CommandReceiptStore.ts"; |
| 67 | import * as SourceControlRepositoryService from "../sourceControl/SourceControlRepositoryService.ts"; |
| 68 | import * as GitVcsDriver from "../vcs/GitVcsDriver.ts"; |
| 69 | import * as PeerForwarding from "./PeerForwarding.ts"; |
| 70 | import * as PeerLinks from "./PeerLinks.ts"; |
| 71 | import { descriptorOf, layerLinkingEnvironment, linkTo, servePeer } from "./PeerLinks.testkit.ts"; |
| 72 | import * as RemoteDelegation from "./RemoteDelegation.ts"; |
| 73 | import * as PeerLinkRequests from "./PeerLinkRequests.ts"; |
| 74 | |
| 75 | // The laptop's agent delegates a task to the box through a link. The box is |
| 76 | // its real /mcp behind real OAuth, with one thread the test finishes; the |
| 77 | // laptop is its real orchestrator and toolkits. The parent here wakes as it |
| 78 | // would for a local child. |
| 79 | |
| 80 | const laptop = descriptorOf("environment-laptop", "Laptop"); |
| 81 | const box = descriptorOf("environment-box", "Box"); |
| 82 | const boxProject = ProjectId.make("project:box"); |
| 83 | const instanceId = ProviderInstanceId.make("codex"); |
| 84 | const driver = ProviderDriverKind.make("codex"); |
| 85 | const modelSelection = { instanceId, model: "gpt-5.4" }; |
| 86 | |
| 87 | type LaunchedWith = Pick< |
| 88 | ThreadLaunch.ThreadLaunchInput, |
| 89 | "modelSelection" | "runtimeMode" | "interactionMode" |
| 90 | >; |
| 91 | |
| 92 | /** The thread the box launches for the task, which the test finishes. */ |
| 93 | interface TimelineEntry { |
| 94 | readonly id: TurnItemId; |
| 95 | readonly runId: RunId; |
| 96 | readonly text: string; |
| 97 | readonly type: "assistant_message" | "error"; |
| 98 | } |
| 99 | |
| 100 | interface BoxThread { |
| 101 | readonly timeline: Ref.Ref<ReadonlyArray<TimelineEntry> | null>; |
| 102 | readonly pageSize: Ref.Ref<number>; |
| 103 | readonly holdRefill: Ref.Ref<boolean>; |
| 104 | readonly beforeTail: Ref.Ref<Effect.Effect<void>>; |
| 105 | readonly launched: Ref.Ref<ThreadId | null>; |
| 106 | /** The link session that launched it, as the box stamps it. */ |
| 107 | readonly linkOrigin: Ref.Ref<{ readonly sessionId: string; readonly label: string } | null>; |
| 108 | /** The parent the launch names, so the thread there can link back to it. */ |
| 109 | readonly delegatedFrom: Ref.Ref<ThreadLaunch.ThreadLaunchInput["delegatedFrom"] | null>; |
| 110 | /** The model and modes it was launched with. */ |
| 111 | readonly launchedWith: Ref.Ref<LaunchedWith | null>; |
| 112 | /** The first message the thread there was launched with. */ |
| 113 | readonly message: Ref.Ref<string | null>; |
| 114 | readonly status: Ref.Ref<OrchestrationV2Run["status"]>; |
| 115 | readonly reply: Ref.Ref<string | null>; |
| 116 | readonly error: Ref.Ref<string | null>; |
| 117 | readonly laterError: Ref.Ref<string | null>; |
| 118 | /** A later run's reply, after the task's own, as a follow-up message would leave. */ |
| 119 | readonly laterReply: Ref.Ref<string | null>; |
| 120 | /** The reply of a run queued there behind the later one; once set, the thread settles on it. */ |
| 121 | readonly queuedReply: Ref.Ref<string | null>; |
| 122 | /** A run there restarted, which its continuation finished with this reply. */ |
| 123 | readonly restart: Ref.Ref<{ readonly of: RunId; readonly reply: string } | null>; |
| 124 | /** The model the provider there reported running the task's run. */ |
| 125 | readonly executedModel: Ref.Ref<string | null>; |
| 126 | /** The run each wait asked for, in order. */ |
| 127 | readonly waitedFor: Ref.Ref<ReadonlyArray<RunId | undefined>>; |
| 128 | /** The thread's run ends with a task of its own still running, whose result wakes it again. */ |
| 129 | readonly nested: Ref.Ref<boolean>; |
| 130 | /** Completed once a wait there blocks on that task. */ |
| 131 | readonly nestedWaiting: Deferred.Deferred<void>; |
| 132 | /** That task settled and the run it woke answered, with `laterReply`. */ |
| 133 | readonly nestedSettled: Deferred.Deferred<void>; |
| 134 | /** Waiting there fails with this, as a thread deleted there does. */ |
| 135 | readonly gone: Ref.Ref<boolean>; |
| 136 | readonly interrupted: Ref.Ref<number>; |
| 137 | /** What each interrupt or stop there did, once per command id. */ |
| 138 | readonly stops: Ref.Ref<ReadonlyArray<"interrupt" | "thread.stop" | "delegated-tasks">>; |
| 139 | /** The launch starts no run, as one recorded before its run was kept. */ |
| 140 | readonly runless: Ref.Ref<boolean>; |
| 141 | /** Completed on the first interrupt there. */ |
| 142 | readonly interruptedOnce: Deferred.Deferred<void>; |
| 143 | readonly finished: Deferred.Deferred<void>; |
| 144 | /** Ends the later run a message sent there started, for a wait pinned to it. */ |
| 145 | readonly laterFinished: Deferred.Deferred<void>; |
| 146 | /** While set, reading the thread there waits: completed once a read starts, released by `releaseRead`. */ |
| 147 | readonly holdRead: Ref.Ref<boolean>; |
| 148 | readonly readStarted: Deferred.Deferred<void>; |
| 149 | readonly releaseRead: Deferred.Deferred<void>; |
| 150 | readonly holdFirstStop: Ref.Ref<boolean>; |
| 151 | readonly stopStarted: Deferred.Deferred<void>; |
| 152 | readonly releaseStop: Deferred.Deferred<void>; |
| 153 | /** Messages sent to the thread there after its launch, each starting a later run. */ |
| 154 | readonly sent: Ref.Ref<ReadonlyArray<string>>; |
| 155 | /** Runs as the launch there is answered, before the laptop hears back. */ |
| 156 | readonly afterLaunch: Ref.Ref<Effect.Effect<void>>; |
| 157 | } |
| 158 | |
| 159 | const makeBoxThread = Effect.gen(function* () { |
| 160 | return { |
| 161 | timeline: yield* Ref.make<ReadonlyArray<TimelineEntry> | null>(null), |
| 162 | pageSize: yield* Ref.make(1), |
| 163 | holdRefill: yield* Ref.make(false), |
| 164 | beforeTail: yield* Ref.make<Effect.Effect<void>>(Effect.void), |
| 165 | launched: yield* Ref.make<ThreadId | null>(null), |
| 166 | linkOrigin: yield* Ref.make<{ readonly sessionId: string; readonly label: string } | null>( |
| 167 | null, |
| 168 | ), |
| 169 | delegatedFrom: yield* Ref.make<ThreadLaunch.ThreadLaunchInput["delegatedFrom"] | null>(null), |
| 170 | launchedWith: yield* Ref.make<LaunchedWith | null>(null), |
| 171 | message: yield* Ref.make<string | null>(null), |
| 172 | status: yield* Ref.make<OrchestrationV2Run["status"]>("running"), |
| 173 | reply: yield* Ref.make<string | null>("Working…"), |
| 174 | error: yield* Ref.make<string | null>(null), |
| 175 | laterError: yield* Ref.make<string | null>(null), |
| 176 | laterReply: yield* Ref.make<string | null>(null), |
| 177 | queuedReply: yield* Ref.make<string | null>(null), |
| 178 | restart: yield* Ref.make<{ readonly of: RunId; readonly reply: string } | null>(null), |
| 179 | executedModel: yield* Ref.make<string | null>(null), |
| 180 | waitedFor: yield* Ref.make<ReadonlyArray<RunId | undefined>>([]), |
| 181 | nested: yield* Ref.make(false), |
| 182 | nestedWaiting: yield* Deferred.make<void>(), |
| 183 | nestedSettled: yield* Deferred.make<void>(), |
| 184 | gone: yield* Ref.make(false), |
| 185 | interrupted: yield* Ref.make(0), |
| 186 | stops: yield* Ref.make<ReadonlyArray<"interrupt" | "thread.stop" | "delegated-tasks">>([]), |
| 187 | runless: yield* Ref.make(false), |
| 188 | interruptedOnce: yield* Deferred.make<void>(), |
| 189 | finished: yield* Deferred.make<void>(), |
| 190 | laterFinished: yield* Deferred.make<void>(), |
| 191 | holdRead: yield* Ref.make(false), |
| 192 | readStarted: yield* Deferred.make<void>(), |
| 193 | releaseRead: yield* Deferred.make<void>(), |
| 194 | holdFirstStop: yield* Ref.make(false), |
| 195 | stopStarted: yield* Deferred.make<void>(), |
| 196 | releaseStop: yield* Deferred.make<void>(), |
| 197 | sent: yield* Ref.make<ReadonlyArray<string>>([]), |
| 198 | afterLaunch: yield* Ref.make<Effect.Effect<void>>(Effect.void), |
| 199 | } satisfies BoxThread; |
| 200 | }); |
| 201 | |
| 202 | /** The box's projects: more than a page, with this repository's last. */ |
| 203 | const boxProjects: ReadonlyArray<Project> = [ |
| 204 | ...Array.from({ length: 120 }, (_, index) => ({ |
| 205 | id: ProjectId.make(`project:box-other-${index}`), |
| 206 | title: `other ${index}`, |
| 207 | workspaceRoot: `/srv/other-${index}`, |
| 208 | repositoryIdentity: null, |
| 209 | defaultModelSelection: null, |
| 210 | scripts: [], |
| 211 | createdAt: "2026-10-01T00:00:00.000Z", |
| 212 | updatedAt: "2026-10-01T00:00:00.000Z", |
| 213 | deletedAt: null, |
| 214 | })), |
| 215 | { |
| 216 | id: boxProject, |
| 217 | title: "app", |
| 218 | workspaceRoot: "/srv/app", |
| 219 | repositoryIdentity: { |
| 220 | canonicalKey: "github.com/acme/app", |
| 221 | locator: { |
| 222 | source: "git-remote", |
| 223 | remoteName: "origin", |
| 224 | remoteUrl: "git@github.com:acme/app.git", |
| 225 | }, |
| 226 | }, |
| 227 | defaultModelSelection: null, |
| 228 | scripts: [], |
| 229 | createdAt: "2026-10-01T00:00:00.000Z", |
| 230 | updatedAt: "2026-10-01T00:00:00.000Z", |
| 231 | deletedAt: null, |
| 232 | }, |
| 233 | ]; |
| 234 | |
| 235 | const serveBox = (thread: BoxThread) => { |
| 236 | return servePeer(box, boxToolkitLayer(thread)); |
| 237 | }; |
| 238 | |
| 239 | const boxRunId = RunId.make("run:box"); |
| 240 | const boxLaterRunId = RunId.make("run:box-later"); |
| 241 | const boxQueuedRunId = RunId.make("run:box-queued"); |
| 242 | const continuationOf = (runId: RunId) => RunId.make(`${runId}:restarted`); |
| 243 | |
| 244 | const boxToolkitLayer = (thread: BoxThread) => { |
| 245 | const runId = boxRunId; |
| 246 | const interrupts = new Set<string>(); |
| 247 | const shellOf = (threadId: ThreadId) => |
| 248 | Effect.gen(function* () { |
| 249 | if ((yield* Ref.get(thread.launched)) !== threadId) return null; |
| 250 | const linkOrigin = yield* Ref.get(thread.linkOrigin); |
| 251 | return { |
| 252 | ...liveThreadShell(threadId, { runtimeMode: "full-access" }), |
| 253 | projectId: boxProject, |
| 254 | ...(linkOrigin === null ? {} : { linkOrigin }), |
| 255 | }; |
| 256 | }); |
| 257 | const runOf = (status: OrchestrationV2Run["status"]) => |
| 258 | ({ |
| 259 | id: runId, |
| 260 | threadId: ThreadId.make("thread:box"), |
| 261 | ordinal: 1, |
| 262 | status, |
| 263 | modelSelection, |
| 264 | activeAttemptId: "attempt:box", |
| 265 | requestedAt: DateTime.makeUnsafe(0), |
| 266 | startedAt: DateTime.makeUnsafe(0), |
| 267 | completedAt: null, |
| 268 | }) as unknown as OrchestrationV2Run; |
| 269 | const projectionOf = (threadId: ThreadId) => |
| 270 | Effect.gen(function* () { |
| 271 | const shell = yield* shellOf(threadId); |
| 272 | const status = yield* Ref.get(thread.status); |
| 273 | const runless = yield* Ref.get(thread.runless); |
| 274 | const model = yield* Ref.get(thread.executedModel); |
| 275 | return shell === null |
| 276 | ? (null as never) |
| 277 | : { |
| 278 | ...idleThreadProjection(shell), |
| 279 | runs: runless ? [] : [runOf(status)], |
| 280 | providerTurns: (model === null |
| 281 | ? [] |
| 282 | : [{ runAttemptId: "attempt:box", ordinal: 1, model }]) as never, |
| 283 | }; |
| 284 | }); |
| 285 | // One record per command id, as its receipt replays a repeat. |
| 286 | const recordStop = (commandId: string, kind: "interrupt" | "thread.stop" | "delegated-tasks") => |
| 287 | Effect.gen(function* () { |
| 288 | if (interrupts.has(`${kind}:${commandId}`)) return; |
| 289 | interrupts.add(`${kind}:${commandId}`); |
| 290 | yield* Ref.update(thread.stops, (seen) => [...seen, kind]); |
| 291 | if (kind !== "delegated-tasks") { |
| 292 | yield* Ref.update(thread.interrupted, (count) => count + 1); |
| 293 | yield* Deferred.succeed(thread.interruptedOnce, undefined); |
| 294 | } |
| 295 | }); |
| 296 | const layerBoxThreads = Layer.mock(ThreadManagement.ThreadManagementService)({ |
| 297 | getThreadShell: shellOf, |
| 298 | getProjectThreadRecords: (input) => projectionOf(input.threadId), |
| 299 | getThreadRecords: (threadId) => projectionOf(threadId), |
| 300 | getRunCount: (threadId) => |
| 301 | projectionOf(threadId).pipe(Effect.map((projection) => projection.runs.length)), |
| 302 | waitForThread: (input) => |
| 303 | Effect.gen(function* () { |
| 304 | yield* Ref.update(thread.waitedFor, (seen) => [...seen, input.runId]); |
| 305 | // A thread with no run has nothing to wait for. |
| 306 | if (yield* Ref.get(thread.runless)) { |
| 307 | return { threadId: input.threadId, run: null, timedOut: false }; |
| 308 | } |
| 309 | if (yield* Ref.get(thread.gone)) { |
| 310 | return yield* new ThreadManagement.ThreadManagementThreadNotFoundError({ |
| 311 | projectId: boxProject, |
| 312 | threadId: input.threadId, |
| 313 | }); |
| 314 | } |
| 315 | // The queued run's own task stays open while the follow-up ahead of it settles. |
| 316 | if (input.runId === boxQueuedRunId) return yield* Effect.never; |
| 317 | if (input.runId === boxLaterRunId) { |
| 318 | const laterDone = yield* Deferred.await(thread.laterFinished).pipe( |
| 319 | Effect.timeoutOption(input.timeoutMs), |
| 320 | ); |
| 321 | // Settled, the thread's answer is the run queued behind it, once there is one. |
| 322 | const queued = |
| 323 | Option.isSome(laterDone) && |
| 324 | input.untilDelegatedWorkSettled === true && |
| 325 | (yield* Ref.get(thread.queuedReply)) !== null; |
| 326 | // A restarted run is reported as its continuation, as the box follows the chain. |
| 327 | const restarted = (yield* Ref.get(thread.restart))?.of; |
| 328 | const latest = (runId: RunId) => (runId === restarted ? continuationOf(runId) : runId); |
| 329 | return { |
| 330 | threadId: input.threadId, |
| 331 | run: { |
| 332 | id: latest(boxLaterRunId), |
| 333 | status: Option.isSome(laterDone) ? "completed" : "running", |
| 334 | } as OrchestrationV2Run, |
| 335 | timedOut: Option.isNone(laterDone), |
| 336 | ...(queued |
| 337 | ? { |
| 338 | resultRun: { |
| 339 | id: latest(boxQueuedRunId), |
| 340 | status: "completed", |
| 341 | } as OrchestrationV2Run, |
| 342 | } |
| 343 | : {}), |
| 344 | }; |
| 345 | } |
| 346 | const done = yield* Deferred.await(thread.finished).pipe( |
| 347 | Effect.timeoutOption(input.timeoutMs), |
| 348 | ); |
| 349 | // Only a waiter that asks waits past the run for the thread's own task. |
| 350 | if ( |
| 351 | Option.isSome(done) && |
| 352 | input.untilDelegatedWorkSettled === true && |
| 353 | (yield* Ref.get(thread.nested)) |
| 354 | ) { |
| 355 | yield* Deferred.succeed(thread.nestedWaiting, undefined); |
| 356 | const settled = yield* Deferred.await(thread.nestedSettled).pipe( |
| 357 | Effect.timeoutOption(input.timeoutMs), |
| 358 | ); |
| 359 | return { |
| 360 | threadId: input.threadId, |
| 361 | run: { id: runId, status: yield* Ref.get(thread.status) } as OrchestrationV2Run, |
| 362 | timedOut: Option.isNone(settled), |
| 363 | ...(Option.isSome(settled) |
| 364 | ? { resultRun: { id: boxLaterRunId, status: "completed" } as OrchestrationV2Run } |
| 365 | : {}), |
| 366 | }; |
| 367 | } |
| 368 | // Unpinned, a wait reports the latest run: a later one once there is one. |
| 369 | const later = input.runId === undefined && (yield* Ref.get(thread.laterReply)) !== null; |
| 370 | return { |
| 371 | threadId: input.threadId, |
| 372 | run: { |
| 373 | id: later ? boxLaterRunId : runId, |
| 374 | status: later ? "completed" : yield* Ref.get(thread.status), |
| 375 | } as OrchestrationV2Run, |
| 376 | timedOut: Option.isNone(done), |
| 377 | }; |
| 378 | }), |
| 379 | getTimelinePage: (threadId, options) => |
| 380 | Effect.gen(function* () { |
| 381 | if (options.afterPosition === 299 && options.itemId === undefined) { |
| 382 | yield* Ref.getAndSet(thread.beforeTail, Effect.void).pipe(Effect.flatten); |
| 383 | } |
| 384 | if ( |
| 385 | (yield* Ref.get(thread.holdRead)) || |
| 386 | (options.itemId !== undefined && (yield* Ref.get(thread.holdRefill))) |
| 387 | ) { |
| 388 | yield* Deferred.succeed(thread.readStarted, undefined); |
| 389 | yield* Deferred.await(thread.releaseRead); |
| 390 | } |
| 391 | const reply = yield* Ref.get(thread.reply); |
| 392 | const later = yield* Ref.get(thread.laterReply); |
| 393 | const queued = yield* Ref.get(thread.queuedReply); |
| 394 | const error = yield* Ref.get(thread.error); |
| 395 | const laterError = yield* Ref.get(thread.laterError); |
| 396 | const restart = yield* Ref.get(thread.restart); |
| 397 | // One item per page, with errors available only in the activity view. |
| 398 | const replies = ( |
| 399 | (yield* Ref.get(thread.timeline)) ?? [ |
| 400 | ...(reply === null |
| 401 | ? [] |
| 402 | : [ |
| 403 | { |
| 404 | id: TurnItemId.make("item:reply"), |
| 405 | runId, |
| 406 | text: reply, |
| 407 | type: "assistant_message", |
| 408 | }, |
| 409 | ]), |
| 410 | ...(error === null |
| 411 | ? [] |
| 412 | : [{ id: TurnItemId.make("item:error"), runId, text: error, type: "error" }]), |
| 413 | ...(later === null |
| 414 | ? [] |
| 415 | : [ |
| 416 | { |
| 417 | id: TurnItemId.make("item:later-reply"), |
| 418 | runId: boxLaterRunId, |
| 419 | text: later, |
| 420 | type: "assistant_message", |
| 421 | }, |
| 422 | ]), |
| 423 | ...(laterError === null |
| 424 | ? [] |
| 425 | : [ |
| 426 | { |
| 427 | id: TurnItemId.make("item:later-error"), |
| 428 | runId: boxLaterRunId, |
| 429 | text: laterError, |
| 430 | type: "error", |
| 431 | }, |
| 432 | ]), |
| 433 | ...(queued === null |
| 434 | ? [] |
| 435 | : [ |
| 436 | { |
| 437 | id: TurnItemId.make("item:queued-reply"), |
| 438 | runId: boxQueuedRunId, |
| 439 | text: queued, |
| 440 | type: "assistant_message", |
| 441 | }, |
| 442 | ]), |
| 443 | ...(restart === null |
| 444 | ? [] |
| 445 | : [ |
| 446 | { |
| 447 | id: TurnItemId.make("item:restarted-reply"), |
| 448 | runId: continuationOf(restart.of), |
| 449 | text: restart.reply, |
| 450 | type: "assistant_message", |
| 451 | }, |
| 452 | ]), |
| 453 | ] |
| 454 | ).map((entry, position) => ({ ...entry, position })); |
| 455 | const visible = |
| 456 | options.itemId === undefined |
| 457 | ? replies.filter( |
| 458 | (entry) => options.view === "activity" || entry.type === "assistant_message", |
| 459 | ) |
| 460 | : replies.filter((entry) => entry.id === options.itemId); |
| 461 | const matching = visible |
| 462 | .filter((entry) => entry.position > (options.afterPosition ?? -1)) |
| 463 | .slice(0, yield* Ref.get(thread.pageSize)); |
| 464 | return { |
| 465 | items: matching.map(({ id, runId: itemRunId, text, position, type }) => ({ |
| 466 | position, |
| 467 | visibility: "local" as const, |
| 468 | sourceThreadId: threadId, |
| 469 | sourceItemId: id, |
| 470 | item: { |
| 471 | id, |
| 472 | threadId, |
| 473 | runId: itemRunId, |
| 474 | nodeId: null, |
| 475 | providerThreadId: null, |
| 476 | providerTurnId: null, |
| 477 | nativeItemRef: null, |
| 478 | parentItemId: null, |
| 479 | ordinal: 1, |
| 480 | status: "completed" as const, |
| 481 | title: null, |
| 482 | startedAt: DateTime.makeUnsafe(0), |
| 483 | completedAt: DateTime.makeUnsafe(0), |
| 484 | updatedAt: DateTime.makeUnsafe(0), |
| 485 | ...(type === "error" |
| 486 | ? { type, failure: { message: text } } |
| 487 | : { |
| 488 | type: "assistant_message" as const, |
| 489 | messageId: MessageId.make(`message:${id}`), |
| 490 | text, |
| 491 | streaming: false, |
| 492 | }), |
| 493 | } as never, |
| 494 | })), |
| 495 | totalItems: replies.length, |
| 496 | hasMore: |
| 497 | options.itemId === undefined && |
| 498 | matching.length > 0 && |
| 499 | visible.some((entry) => entry.position > matching.at(-1)!.position), |
| 500 | hasEarlier: false, |
| 501 | }; |
| 502 | }), |
| 503 | interruptThread: (input) => |
| 504 | recordStop(input.commandId, "interrupt").pipe( |
| 505 | Effect.as({ type: "interrupt_requested", run: runOf("running") } as never), |
| 506 | ), |
| 507 | dispatch: (command) => |
| 508 | command.type === "thread.stop" |
| 509 | ? recordStop(command.commandId, "thread.stop").pipe( |
| 510 | Effect.tap(() => |
| 511 | Effect.gen(function* () { |
| 512 | if (!(yield* Ref.getAndSet(thread.holdFirstStop, false))) return; |
| 513 | yield* Deferred.succeed(thread.stopStarted, undefined); |
| 514 | yield* Deferred.await(thread.releaseStop); |
| 515 | }), |
| 516 | ), |
| 517 | Effect.as({ sequence: 0, storedEvents: [] } as never), |
| 518 | ) |
| 519 | : Effect.die(`The box does not expect ${command.type}`), |
| 520 | stopDelegatedTasks: (input) => recordStop(input.commandId, "delegated-tasks"), |
| 521 | sendToThread: (input) => |
| 522 | Ref.update(thread.sent, (seen) => [...seen, input.text]).pipe( |
| 523 | Effect.as( |
| 524 | (input.mode === "queue" |
| 525 | ? { run: { id: boxQueuedRunId, status: "queued" }, delivery: "queued" } |
| 526 | : { run: { id: boxLaterRunId, status: "running" }, delivery: "started" }) as never, |
| 527 | ), |
| 528 | ), |
| 529 | }); |
| 530 | const layerBoxLaunches = Layer.mock(ThreadLaunch.ThreadLaunchService)({ |
| 531 | launch: (input) => |
| 532 | Ref.get(thread.runless).pipe( |
| 533 | Effect.flatMap((runless) => |
| 534 | Ref.set(thread.launched, input.threadId!).pipe( |
| 535 | Effect.andThen(Ref.set(thread.linkOrigin, input.linkOrigin ?? null)), |
| 536 | Effect.andThen(Ref.set(thread.delegatedFrom, input.delegatedFrom ?? null)), |
| 537 | Effect.andThen(Ref.set(thread.message, input.initialMessage?.text ?? null)), |
| 538 | Effect.andThen( |
| 539 | Ref.set(thread.launchedWith, { |
| 540 | modelSelection: input.modelSelection, |
| 541 | runtimeMode: input.runtimeMode, |
| 542 | interactionMode: input.interactionMode, |
| 543 | }), |
| 544 | ), |
| 545 | Effect.andThen(Effect.flatten(Ref.get(thread.afterLaunch))), |
| 546 | Effect.as({ |
| 547 | threadId: input.threadId, |
| 548 | projection: { |
| 549 | thread: { |
| 550 | id: input.threadId, |
| 551 | projectId: input.projectId, |
| 552 | title: input.title, |
| 553 | modelSelection: input.modelSelection, |
| 554 | }, |
| 555 | runs: |
| 556 | input.initialMessage?.messageId && !runless |
| 557 | ? [ |
| 558 | { |
| 559 | id: runId, |
| 560 | status: "running", |
| 561 | userMessageId: input.initialMessage.messageId, |
| 562 | }, |
| 563 | ] |
| 564 | : [], |
| 565 | }, |
| 566 | resumed: false, |
| 567 | } as unknown as ThreadLaunch.ThreadLaunchResult), |
| 568 | ), |
| 569 | ), |
| 570 | ), |
| 571 | }); |
| 572 | return Layer.merge( |
| 573 | McpHttpServer.layerOrchestratorToolkit, |
| 574 | McpHttpServer.layerProjectRegistration, |
| 575 | ).pipe( |
| 576 | Layer.provide(DocumentText.layer.pipe(Layer.provideMerge(NodeServices.layer))), |
| 577 | Layer.provide(NodeCrypto.layer), |
| 578 | Layer.provide(layerBoxThreads), |
| 579 | Layer.provide(layerBoxLaunches), |
| 580 | Layer.provide(Layer.mock(RemoteDelegation.RemoteDelegation)({})), |
| 581 | Layer.provide(Layer.mock(PeerForwarding.PeerForwarding)({})), |
| 582 | Layer.provide(Layer.mock(PeerLinkRequests.PeerLinkRequests)({})), |
| 583 | Layer.provide(Layer.mock(PeerLinks.PeerLinks)({})), |
| 584 | Layer.provide( |
| 585 | Layer.mock(ManagedProjectFolders.ManagedProjectFolders)({ namedProjectsRoot: "/p" }), |
| 586 | ), |
| 587 | Layer.provide(Layer.mock(GitVcsDriver.GitVcsDriver)({})), |
| 588 | Layer.provide(Layer.mock(SourceControlRepositoryService.SourceControlRepositoryService)({})), |
| 589 | Layer.provide( |
| 590 | ServerConfig.layerTest(process.cwd(), { prefix: "t3-remote-delegation-box-" }).pipe( |
| 591 | Layer.provide(NodeServices.layer), |
| 592 | ), |
| 593 | ), |
| 594 | Layer.provide( |
| 595 | Layer.mock(ProviderRegistry.ProviderRegistry)({ |
| 596 | getProviders: Effect.succeed([boxClaude]), |
| 597 | }), |
| 598 | ), |
| 599 | Layer.provide( |
| 600 | Layer.mock(ProviderAdapterRegistry.ProviderAdapterRegistryV2)({ |
| 601 | list: () => Effect.succeed([boxClaude.instanceId]), |
| 602 | }), |
| 603 | ), |
| 604 | Layer.provide(Layer.mock(ScheduledTaskService.ScheduledTaskService)({})), |
| 605 | Layer.provide( |
| 606 | Layer.mock(ProjectService.ProjectService)({ |
| 607 | getById: (id) => |
| 608 | Effect.succeed(Option.fromNullishOr(boxProjects.find((project) => project.id === id))), |
| 609 | // More than a page, so finding the project with this repository pages. |
| 610 | listPage: ({ cursor, limit }) => |
| 611 | Effect.succeed({ |
| 612 | projects: boxProjects.slice(cursor, cursor + limit), |
| 613 | nextCursor: cursor + limit < boxProjects.length ? cursor + limit : null, |
| 614 | }), |
| 615 | }), |
| 616 | ), |
| 617 | Layer.provide(Layer.mock(ProjectCloneTracker.ProjectCloneTracker)({})), |
| 618 | Layer.provide(Layer.mock(SecretRequests.SecretRequests)({})), |
| 619 | Layer.provide(ServerSettings.layerTest()), |
| 620 | Layer.provide(NodeServices.layer), |
| 621 | ); |
| 622 | }; |
| 623 | |
| 624 | const mcpClient = McpSchema.McpServerClient.of({ |
| 625 | clientId: 1, |
| 626 | protocolVersion: "2025-06-18", |
| 627 | clientCapabilities: {}, |
| 628 | clientInfo: { name: "remote-delegation", version: "1" }, |
| 629 | initializePayload: { |
| 630 | protocolVersion: "2025-06-18", |
| 631 | capabilities: {}, |
| 632 | clientInfo: { name: "remote-delegation", version: "1" }, |
| 633 | }, |
| 634 | getClient: Effect.die("unused"), |
| 635 | }); |
| 636 | |
| 637 | /** The box's only provider: a Claude instance, under its own id. */ |
| 638 | const boxClaude: ServerProvider = { |
| 639 | instanceId: ProviderInstanceId.make("claude-box"), |
| 640 | driver: ProviderDriverKind.make("claudeAgent"), |
| 641 | enabled: true, |
| 642 | installed: true, |
| 643 | version: "test", |
| 644 | status: "ready", |
| 645 | auth: { status: "authenticated" }, |
| 646 | checkedAt: "2026-09-13T00:00:00.000Z", |
| 647 | models: [ |
| 648 | { slug: "claude-sonnet-4-6", name: "Claude Sonnet", isCustom: false, capabilities: null }, |
| 649 | ], |
| 650 | slashCommands: [], |
| 651 | skills: [], |
| 652 | }; |
| 653 | |
| 654 | const adapter = { |
| 655 | instanceId, |
| 656 | driver, |
| 657 | getCapabilities: () => Effect.succeed(CodexProviderCapabilitiesV2), |
| 658 | planSelectionTransition: () => Effect.succeed({ type: "apply_on_next_turn" as const }), |
| 659 | openSession: () => Effect.die("No provider process needed"), |
| 660 | } as ProviderAdapter.ProviderAdapterV2["Service"]; |
| 661 | |
| 662 | /** |
| 663 | * The laptop: a real orchestrator (in-memory SQLite) whose parent thread has |
| 664 | * a live run, the real orchestrator toolkit, and RemoteDelegation. Built in |
| 665 | * its own scope so a test can tear it down and build it again on the same |
| 666 | * database, as a restart does. |
| 667 | */ |
| 668 | const makeLaptop = ( |
| 669 | database: Context.Context<SqlClient.SqlClient>, |
| 670 | offers: SubscriptionRef.SubscriptionRef< |
| 671 | ReadonlyArray<ProviderContinuationRequests.ProviderContinuationRequest> |
| 672 | >, |
| 673 | /** The laptop's links from before a restart, which keep their sessions there. */ |
| 674 | kept?: Context.Context<Layer.Success<ReturnType<typeof layerLinkingEnvironment>>>, |
| 675 | /** Runs as each follow attempt here starts, on its follower's fiber. */ |
| 676 | beforeFollow: Effect.Effect<void, Orchestrator.OrchestratorV2Error> = Effect.void, |
| 677 | ) => |
| 678 | Effect.gen(function* () { |
| 679 | const linking = kept ?? (yield* layerLinkingEnvironment(laptop).pipe(Layer.build)); |
| 680 | const layerDatabase = Layer.succeedContext(database); |
| 681 | const layerOrchestrator = Layer.mergeAll( |
| 682 | layerDatabase, |
| 683 | ProjectionStore.layer.pipe(Layer.provide(layerDatabase)), |
| 684 | ProviderReplayHarness.layerWithRegistry( |
| 685 | { name: "remote-delegation-laptop" }, |
| 686 | ProviderAdapterRegistry.layerFromAdapters([adapter]), |
| 687 | { databaseLayer: layerDatabase, runEffectWorker: false }, |
| 688 | ), |
| 689 | ).pipe( |
| 690 | Layer.provide( |
| 691 | Layer.succeed(ProviderContinuationRequests.ProviderContinuationRequests, { |
| 692 | offer: (request) => SubscriptionRef.update(offers, (seen) => [...seen, request]), |
| 693 | take: Effect.never, |
| 694 | }), |
| 695 | ), |
| 696 | ); |
| 697 | const layerThreads = ThreadManagement.layer.pipe( |
| 698 | Layer.provide(CommandReceiptStore.layer.pipe(Layer.provide(layerDatabase))), |
| 699 | Layer.provideMerge(Layer.merge(layerOrchestrator, ServerSettings.layerTest())), |
| 700 | ); |
| 701 | const layerHere = McpHttpServer.layerOrchestratorToolkit.pipe( |
| 702 | Layer.provideMerge(McpHttpServer.layerInProcessMcpServer), |
| 703 | Layer.provideMerge( |
| 704 | RemoteDelegation.layer.pipe( |
| 705 | Layer.provide( |
| 706 | Layer.effect( |
| 707 | ThreadManagement.ThreadManagementService, |
| 708 | Effect.gen(function* () { |
| 709 | const threads = yield* ThreadManagement.ThreadManagementService; |
| 710 | return { |
| 711 | ...threads, |
| 712 | getThreadEventSequence: (threadId: ThreadId) => |
| 713 | beforeFollow.pipe(Effect.andThen(threads.getThreadEventSequence(threadId))), |
| 714 | }; |
| 715 | }), |
| 716 | ), |
| 717 | ), |
| 718 | Layer.provideMerge(PeerForwarding.layer), |
| 719 | Layer.provide( |
| 720 | Layer.mock(ProjectService.ProjectService)({ |
| 721 | // A cold enrichment cache: the project itself reports no identity yet. |
| 722 | getById: () => |
| 723 | Effect.succeedSome({ |
| 724 | workspaceRoot: "/home/me/app", |
| 725 | repositoryIdentity: null, |
| 726 | } as never), |
| 727 | }), |
| 728 | ), |
| 729 | Layer.provide( |
| 730 | Layer.mock(RepositoryIdentityResolver.RepositoryIdentityResolver)({ |
| 731 | resolve: (cwd) => |
| 732 | Effect.succeed( |
| 733 | cwd === "/home/me/app" |
| 734 | ? ({ canonicalKey: "github.com/acme/app" } as RepositoryIdentity) |
| 735 | : null, |
| 736 | ), |
| 737 | }), |
| 738 | ), |
| 739 | ), |
| 740 | ), |
| 741 | Layer.provide(Layer.succeedContext(linking)), |
| 742 | Layer.provide(Layer.mock(PeerLinkRequests.PeerLinkRequests)({})), |
| 743 | Layer.provide(NodeCrypto.layer), |
| 744 | Layer.provideMerge(layerThreads), |
| 745 | Layer.provide(Layer.mock(ProviderRegistry.ProviderRegistry)({})), |
| 746 | Layer.provide(Layer.mock(ProviderAdapterRegistry.ProviderAdapterRegistryV2)({})), |
| 747 | Layer.provide(Layer.mock(ScheduledTaskService.ScheduledTaskService)({})), |
| 748 | Layer.provide(Layer.mock(ProjectService.ProjectService)({})), |
| 749 | Layer.provide(Layer.mock(ProjectCloneTracker.ProjectCloneTracker)({})), |
| 750 | Layer.provide(Layer.mock(SecretRequests.SecretRequests)({})), |
| 751 | Layer.provide(Layer.mock(ThreadLaunch.ThreadLaunchService)({})), |
| 752 | Layer.provide(Layer.mock(GitVcsDriver.GitVcsDriver)({})), |
| 753 | Layer.fresh, |
| 754 | ); |
| 755 | const here = yield* Layer.build(layerHere); |
| 756 | const server = Context.get(here, McpServer.McpServer); |
| 757 | return { |
| 758 | links: Context.get(linking, PeerLinks.PeerLinks), |
| 759 | settings: Context.get(linking, ServerSettings.ServerSettingsService), |
| 760 | orchestrator: Context.get(here, Orchestrator.OrchestratorV2), |
| 761 | remote: Context.get(here, RemoteDelegation.RemoteDelegation), |
| 762 | threads: Context.get(here, ThreadManagement.ThreadManagementService), |
| 763 | projections: Context.get(here, ProjectionStore.ProjectionStoreV2), |
| 764 | sink: Context.get(here, EventSink.EventSinkV2), |
| 765 | call: (name: string, args: Record<string, unknown>, scope = parentScope) => |
| 766 | server |
| 767 | .callTool({ name, arguments: args }) |
| 768 | .pipe( |
| 769 | Effect.provideService(McpInvocationContext.McpInvocationContext, scope), |
| 770 | Effect.provideService(McpSchema.McpServerClient, mcpClient), |
| 771 | ), |
| 772 | }; |
| 773 | }); |
| 774 | |
| 775 | const parentThreadId = ThreadId.make("thread:laptop-parent"); |
| 776 | const parentRunId = RunId.make("run:laptop-parent"); |
| 777 | const parentRootNode = NodeId.make("node:laptop-parent:root"); |
| 778 | const parentScope: McpInvocationContext.McpInvocationScope = { |
| 779 | environmentId: laptop.environmentId, |
| 780 | requestNamespace: "provider-session:laptop-parent", |
| 781 | thread: { |
| 782 | threadId: parentThreadId, |
| 783 | providerSessionId: "provider-session:laptop-parent", |
| 784 | providerInstanceId: instanceId, |
| 785 | }, |
| 786 | client: undefined, |
| 787 | capabilities: new Set(["orchestration"]), |
| 788 | issuedAt: 1, |
| 789 | }; |
| 790 | |
| 791 | /** The laptop's parent thread, mid-turn when its agent delegates. */ |
| 792 | const seedParent = ( |
| 793 | laptopEnv: Effect.Success<ReturnType<typeof makeLaptop>>, |
| 794 | parentModelSelection: ModelSelection = modelSelection, |
| 795 | ) => |
| 796 | Effect.gen(function* () { |
| 797 | const now = yield* DateTime.now; |
| 798 | const providerThreadId = ProviderThreadId.make("provider-thread:laptop-parent"); |
| 799 | yield* laptopEnv.orchestrator.dispatch({ |
| 800 | type: "thread.create", |
| 801 | createdBy: "user", |
| 802 | creationSource: "web", |
| 803 | commandId: CommandId.make("command:create:laptop-parent"), |
| 804 | threadId: parentThreadId, |
| 805 | projectId: ProjectId.make("project:laptop"), |
| 806 | title: "Laptop parent", |
| 807 | modelSelection: parentModelSelection, |
| 808 | runtimeMode: "full-access", |
| 809 | interactionMode: "default", |
| 810 | branch: null, |
| 811 | worktreePath: null, |
| 812 | }); |
| 813 | yield* laptopEnv.sink.write({ |
| 814 | commandId: CommandId.make("command:seed:laptop-parent"), |
| 815 | events: [ |
| 816 | { |
| 817 | id: EventId.make("event:seed-provider-thread:laptop-parent"), |
| 818 | type: "provider-thread.updated", |
| 819 | threadId: parentThreadId, |
| 820 | driver, |
| 821 | providerInstanceId: instanceId, |
| 822 | occurredAt: now, |
| 823 | payload: { |
| 824 | id: providerThreadId, |
| 825 | driver, |
| 826 | providerInstanceId: instanceId, |
| 827 | providerSessionId: null, |
| 828 | appThreadId: parentThreadId, |
| 829 | ownerNodeId: parentRootNode, |
| 830 | nativeThreadRef: { driver, nativeId: "native:laptop-parent", strength: "strong" }, |
| 831 | nativeConversationHeadRef: null, |
| 832 | status: "active", |
| 833 | firstRunOrdinal: 1, |
| 834 | lastRunOrdinal: 1, |
| 835 | handoffIds: [], |
| 836 | forkedFrom: null, |
| 837 | createdAt: now, |
| 838 | updatedAt: now, |
| 839 | }, |
| 840 | }, |
| 841 | { |
| 842 | id: EventId.make("event:seed-node:laptop-parent"), |
| 843 | type: "node.updated", |
| 844 | threadId: parentThreadId, |
| 845 | runId: parentRunId, |
| 846 | nodeId: parentRootNode, |
| 847 | driver, |
| 848 | providerInstanceId: instanceId, |
| 849 | occurredAt: now, |
| 850 | payload: { |
| 851 | id: parentRootNode, |
| 852 | threadId: parentThreadId, |
| 853 | runId: parentRunId, |
| 854 | parentNodeId: null, |
| 855 | rootNodeId: parentRootNode, |
| 856 | kind: "root_turn", |
| 857 | status: "running", |
| 858 | countsForRun: true, |
| 859 | providerThreadId, |
| 860 | providerTurnId: null, |
| 861 | nativeItemRef: null, |
| 862 | runtimeRequestId: null, |
| 863 | checkpointScopeId: null, |
| 864 | startedAt: now, |
| 865 | completedAt: null, |
| 866 | }, |
| 867 | }, |
| 868 | { |
| 869 | id: EventId.make("event:seed-run:laptop-parent"), |
| 870 | type: "run.updated", |
| 871 | threadId: parentThreadId, |
| 872 | runId: parentRunId, |
| 873 | nodeId: parentRootNode, |
| 874 | providerInstanceId: instanceId, |
| 875 | occurredAt: now, |
| 876 | payload: { |
| 877 | id: parentRunId, |
| 878 | threadId: parentThreadId, |
| 879 | ordinal: 1, |
| 880 | providerInstanceId: instanceId, |
| 881 | modelSelection: parentModelSelection, |
| 882 | providerThreadId, |
| 883 | userMessageId: MessageId.make("message:seed-user:laptop-parent"), |
| 884 | rootNodeId: parentRootNode, |
| 885 | activeAttemptId: null, |
| 886 | status: "running", |
| 887 | requestedAt: now, |
| 888 | startedAt: now, |
| 889 | completedAt: null, |
| 890 | checkpointId: null, |
| 891 | contextHandoffId: null, |
| 892 | }, |
| 893 | }, |
| 894 | ], |
| 895 | }); |
| 896 | }); |
| 897 | |
| 898 | const delegateToBox = (laptopEnv: Effect.Success<ReturnType<typeof makeLaptop>>) => |
| 899 | laptopEnv |
| 900 | .call("delegate_task", { |
| 901 | task: "Run the full suite on the box.", |
| 902 | target: { |
| 903 | environmentId: box.environmentId, |
| 904 | providerInstanceId: instanceId, |
| 905 | model: "gpt-5.4", |
| 906 | }, |
| 907 | mode: "async", |
| 908 | clientRequestId: "suite-on-box", |
| 909 | }) |
| 910 | .pipe( |
| 911 | Effect.map((result) => { |
| 912 | expect(result.isError, JSON.stringify(result.content)).toBe(false); |
| 913 | return result.structuredContent as { |
| 914 | taskId: NodeId; |
| 915 | status: string; |
| 916 | childThreadId: null; |
| 917 | remoteChild: { environmentId: string; threadId: ThreadId; label: string }; |
| 918 | link?: string; |
| 919 | }; |
| 920 | }), |
| 921 | ); |
| 922 | |
| 923 | /** Waits until the task here has the result its thread there ended with. */ |
| 924 | const taskResult = (laptopEnv: Effect.Success<ReturnType<typeof makeLaptop>>, taskId: NodeId) => |
| 925 | laptopEnv.orchestrator |
| 926 | .streamStoredEventsFrom({ |
| 927 | threadId: parentThreadId, |
| 928 | afterSequence: 0, |
| 929 | eventType: "subagent.updated", |
| 930 | }) |
| 931 | .pipe( |
| 932 | Stream.filter( |
| 933 | (stored) => |
| 934 | stored.event.type === "subagent.updated" && |
| 935 | stored.event.payload.id === taskId && |
| 936 | stored.event.payload.result !== null, |
| 937 | ), |
| 938 | Stream.runHead, |
| 939 | Effect.map((stored) => |
| 940 | Option.isSome(stored) && stored.value.event.type === "subagent.updated" |
| 941 | ? stored.value.event.payload |
| 942 | : undefined, |
| 943 | ), |
| 944 | ); |
| 945 | |
| 946 | const captureThreadReads = (nativeFetch: typeof globalThis.fetch) => { |
| 947 | const reads: Array<OrchestratorMcpThreadReadInput> = []; |
| 948 | const fetch: typeof globalThis.fetch = (input, init) => { |
| 949 | const body = init?.body instanceof Uint8Array ? new TextDecoder().decode(init.body) : null; |
| 950 | if (body?.startsWith("{")) { |
| 951 | const rpc: { |
| 952 | method?: string; |
| 953 | params?: { name?: string; arguments?: OrchestratorMcpThreadReadInput }; |
| 954 | } = JSON.parse(body); |
| 955 | if ( |
| 956 | rpc.method === "tools/call" && |
| 957 | rpc.params?.name === "t3_thread_read" && |
| 958 | rpc.params.arguments !== undefined |
| 959 | ) { |
| 960 | reads.push(rpc.params.arguments); |
| 961 | } |
| 962 | } |
| 963 | return nativeFetch(input, init); |
| 964 | }; |
| 965 | return { reads, fetch }; |
| 966 | }; |
| 967 | |
| 968 | it.effect("a task delegated to the box wakes the laptop's parent when it ends there", () => |
| 969 | Effect.scoped( |
| 970 | Effect.gen(function* () { |
| 971 | const thread = yield* makeBoxThread; |
| 972 | const b = yield* serveBox(thread); |
| 973 | const offers = yield* SubscriptionRef.make< |
| 974 | ReadonlyArray<ProviderContinuationRequests.ProviderContinuationRequest> |
| 975 | >([]); |
| 976 | const database = yield* SqlitePersistence.layerMemory.pipe(Layer.build); |
| 977 | const a = yield* makeLaptop(database, offers); |
| 978 | yield* linkTo(a.links, b, "full-access"); |
| 979 | yield* seedParent(a); |
| 980 | |
| 981 | const delegated = yield* delegateToBox(a); |
| 982 | expect(delegated).toMatchObject({ |
| 983 | status: "running", |
| 984 | childThreadId: null, |
| 985 | remoteChild: { environmentId: box.environmentId, label: "Box" }, |
| 986 | }); |
| 987 | // The box launched it in the project with the same repository. |
| 988 | expect(yield* Ref.get(thread.launched)).toBe(delegated.remoteChild.threadId); |
| 989 | // Its link opens the thread there, so the parent can mention it like any other. |
| 990 | expect(delegated.link).toBe( |
| 991 | `[Laptop parent](t3-thread://v2/${box.environmentId}/${delegated.remoteChild.threadId})`, |
| 992 | ); |
| 993 | // ...naming the parent here, so the thread there links back to it. |
| 994 | expect(yield* Ref.get(thread.delegatedFrom)).toEqual({ |
| 995 | environmentId: laptop.environmentId, |
| 996 | threadId: parentThreadId, |
| 997 | title: "Laptop parent", |
| 998 | }); |
| 999 | // A retry with the same key launches nothing new and records nothing new. |
| 1000 | const retried = yield* delegateToBox(a); |
| 1001 | expect(retried.taskId).toBe(delegated.taskId); |
| 1002 | expect((yield* a.orchestrator.getThreadProjection(parentThreadId)).subagents).toHaveLength(1); |
| 1003 | |
| 1004 | // Its provider there has not started the turn yet. |
| 1005 | yield* Ref.set(thread.status, "starting"); |
| 1006 | const starting = yield* a.call("task_status", { taskId: delegated.taskId }); |
| 1007 | expect(starting.structuredContent).toMatchObject({ |
| 1008 | status: "starting", |
| 1009 | workState: "working", |
| 1010 | }); |
| 1011 | yield* Ref.set(thread.status, "running"); |
| 1012 | |
| 1013 | const historyReads: Array<string> = []; |
| 1014 | const historyTracer = Tracer.make({ |
| 1015 | span(options) { |
| 1016 | const span = new Tracer.NativeSpan(options); |
| 1017 | const end = span.end.bind(span); |
| 1018 | span.end = (endTime, exit) => { |
| 1019 | end(endTime, exit); |
| 1020 | const query = span.attributes.get("db.query.text"); |
| 1021 | if ( |
| 1022 | typeof query === "string" && |
| 1023 | /\bSELECT\s+payload_json\s+FROM orchestration_v2_projection_(?:runs|context_transfers)\b/i.test( |
| 1024 | query, |
| 1025 | ) |
| 1026 | ) { |
| 1027 | historyReads.push(query); |
| 1028 | } |
| 1029 | }; |
| 1030 | return span; |
| 1031 | }, |
| 1032 | }); |
| 1033 | const running = yield* a |
| 1034 | .call("task_status", { taskId: delegated.taskId }) |
| 1035 | .pipe(Effect.withTracer(historyTracer)); |
| 1036 | expect(running.structuredContent).toMatchObject({ |
| 1037 | taskId: delegated.taskId, |
| 1038 | childThreadId: null, |
| 1039 | remoteChild: delegated.remoteChild, |
| 1040 | status: "running", |
| 1041 | workState: "working", |
| 1042 | summary: null, |
| 1043 | hasPendingQuestion: false, |
| 1044 | pendingApproval: null, |
| 1045 | }); |
| 1046 | // A running task there needs its local row, not the parent's run or transfer history. |
| 1047 | expect(historyReads).toEqual([]); |
| 1048 | |
| 1049 | yield* Ref.set(thread.reply, "All 412 tests pass on the box."); |
| 1050 | yield* Ref.set(thread.status, "completed"); |
| 1051 | yield* Deferred.succeed(thread.finished, undefined); |
| 1052 | const task = yield* taskResult(a, delegated.taskId); |
| 1053 | expect(task).toMatchObject({ status: "completed", result: "All 412 tests pass on the box." }); |
| 1054 | // The parent's live run claims the result, as for a child here. |
| 1055 | const parentRun = (yield* a.orchestrator.getThreadProjection(parentThreadId)).runs.find( |
| 1056 | (run) => run.id === parentRunId, |
| 1057 | ); |
| 1058 | expect(parentRun?.delegatedCompletion?.delivery?.taskIds).toEqual([delegated.taskId]); |
| 1059 | // The wake is offered after the result commits, so the result can be seen first. |
| 1060 | const offered = yield* SubscriptionRef.changes(offers).pipe( |
| 1061 | Stream.filter((seen) => seen.some((offer) => offer.threadId === parentThreadId)), |
| 1062 | Stream.runHead, |
| 1063 | ); |
| 1064 | expect(Option.isSome(offered)).toBe(true); |
| 1065 | |
| 1066 | const messageReads: Array<string> = []; |
| 1067 | const tracer = Tracer.make({ |
| 1068 | span(options) { |
| 1069 | const span = new Tracer.NativeSpan(options); |
| 1070 | const end = span.end.bind(span); |
| 1071 | span.end = (endTime, exit) => { |
| 1072 | end(endTime, exit); |
| 1073 | const query = span.attributes.get("db.query.text"); |
| 1074 | if ( |
| 1075 | typeof query === "string" && |
| 1076 | /SELECT[\s\S]*FROM orchestration_v2_projection_messages/i.test(query) |
| 1077 | ) { |
| 1078 | messageReads.push(query); |
| 1079 | } |
| 1080 | }; |
| 1081 | return span; |
| 1082 | }, |
| 1083 | }); |
| 1084 | const status = yield* a |
| 1085 | .call("task_status", { taskId: delegated.taskId }) |
| 1086 | .pipe(Effect.withTracer(tracer)); |
| 1087 | expect(status.structuredContent).toMatchObject({ |
| 1088 | status: "completed", |
| 1089 | summary: "All 412 tests pass on the box.", |
| 1090 | }); |
| 1091 | // Delivery validation still reads messages; choosing the observer does not need another read. |
| 1092 | expect(messageReads).toHaveLength(2); |
| 1093 | expect(messageReads.filter((query) => !/\bmessage_id\s+IN\b/i.test(query))).toHaveLength(1); |
| 1094 | const acknowledged = yield* a.projections.getThreadRecords(parentThreadId, ["subagents"], { |
| 1095 | subagentIds: [delegated.taskId], |
| 1096 | }); |
| 1097 | expect(acknowledged.subagents[0]?.completionDelivery).toEqual({ |
| 1098 | state: "acknowledged", |
| 1099 | observedByRunId: parentRunId, |
| 1100 | }); |
| 1101 | }), |
| 1102 | ).pipe(Effect.provide(NodeHttpServer.layerTest)), |
| 1103 | ); |
| 1104 | |
| 1105 | it.effect.each(["no active parent run", "a newer run owned by another provider"] as const)( |
| 1106 | "preserves remote observation guards with %s", |
| 1107 | (observer) => |
| 1108 | Effect.scoped( |
| 1109 | Effect.gen(function* () { |
| 1110 | const thread = yield* makeBoxThread; |
| 1111 | const b = yield* serveBox(thread); |
| 1112 | const offers = yield* SubscriptionRef.make< |
| 1113 | ReadonlyArray<ProviderContinuationRequests.ProviderContinuationRequest> |
| 1114 | >([]); |
| 1115 | const database = yield* SqlitePersistence.layerMemory.pipe(Layer.build); |
| 1116 | const a = yield* makeLaptop(database, offers); |
| 1117 | yield* linkTo(a.links, b, "full-access"); |
| 1118 | yield* seedParent(a); |
| 1119 | const delegated = yield* delegateToBox(a); |
| 1120 | |
| 1121 | yield* Ref.set(thread.reply, "Remote result remains available."); |
| 1122 | yield* Ref.set(thread.status, "completed"); |
| 1123 | yield* Deferred.succeed(thread.finished, undefined); |
| 1124 | yield* taskResult(a, delegated.taskId); |
| 1125 | yield* SubscriptionRef.changes(offers).pipe( |
| 1126 | Stream.filter((seen) => seen.some((offer) => offer.threadId === parentThreadId)), |
| 1127 | Stream.runHead, |
| 1128 | ); |
| 1129 | |
| 1130 | const parent = yield* a.projections.getThreadRecords( |
| 1131 | parentThreadId, |
| 1132 | ["runs", "subagents"], |
| 1133 | { |
| 1134 | subagentIds: [delegated.taskId], |
| 1135 | }, |
| 1136 | ); |
| 1137 | expect(parent.subagents[0]?.completionDelivery?.state).not.toBe("acknowledged"); |
| 1138 | const parentRun = parent.runs.find((run) => run.id === parentRunId); |
| 1139 | if (parentRun === undefined) return yield* Effect.die("The parent run was not seeded."); |
| 1140 | const now = yield* DateTime.now; |
| 1141 | const otherInstanceId = ProviderInstanceId.make("codex-other"); |
| 1142 | const observingRun: OrchestrationV2Run = |
| 1143 | observer === "no active parent run" |
| 1144 | ? { ...parentRun, status: "completed", completedAt: now } |
| 1145 | : { |
| 1146 | id: RunId.make("run:laptop-other-provider"), |
| 1147 | threadId: parentThreadId, |
| 1148 | ordinal: parentRun.ordinal + 1, |
| 1149 | providerInstanceId: otherInstanceId, |
| 1150 | modelSelection: { instanceId: otherInstanceId, model: "gpt-5.4" }, |
| 1151 | providerThreadId: null, |
| 1152 | userMessageId: MessageId.make("message:laptop-other-provider"), |
| 1153 | rootNodeId: null, |
| 1154 | activeAttemptId: null, |
| 1155 | status: "preparing", |
| 1156 | requestedAt: now, |
| 1157 | startedAt: null, |
| 1158 | completedAt: null, |
| 1159 | checkpointId: null, |
| 1160 | contextHandoffId: null, |
| 1161 | }; |
| 1162 | yield* a.sink.write({ |
| 1163 | commandId: CommandId.make("command:change-remote-result-observer"), |
| 1164 | events: [ |
| 1165 | { |
| 1166 | id: EventId.make("event:change-remote-result-observer"), |
| 1167 | type: "run.updated", |
| 1168 | threadId: parentThreadId, |
| 1169 | runId: observingRun.id, |
| 1170 | providerInstanceId: observingRun.providerInstanceId, |
| 1171 | occurredAt: now, |
| 1172 | payload: observingRun, |
| 1173 | }, |
| 1174 | ], |
| 1175 | }); |
| 1176 | |
| 1177 | const status = yield* a.call("task_status", { taskId: delegated.taskId }); |
| 1178 | if (observer === "no active parent run") { |
| 1179 | expect(status.isError).toBe(true); |
| 1180 | expect(status.content).toEqual([ |
| 1181 | { |
| 1182 | type: "text", |
| 1183 | text: JSON.stringify({ |
| 1184 | _tag: "OrchestratorMcpFailure", |
| 1185 | code: "parent_not_active", |
| 1186 | message: "The calling provider no longer owns an active thread run.", |
| 1187 | }), |
| 1188 | }, |
| 1189 | ]); |
| 1190 | const unacknowledged = yield* a.projections.getThreadRecords( |
| 1191 | parentThreadId, |
| 1192 | ["subagents"], |
| 1193 | { subagentIds: [delegated.taskId] }, |
| 1194 | ); |
| 1195 | expect(unacknowledged.subagents[0]?.result).toBe("Remote result remains available."); |
| 1196 | expect(unacknowledged.subagents[0]?.completionDelivery?.state).not.toBe("acknowledged"); |
| 1197 | return; |
| 1198 | } |
| 1199 | expect(status.isError, JSON.stringify(status.content)).not.toBe(true); |
| 1200 | expect(status.structuredContent).toMatchObject({ |
| 1201 | status: "completed", |
| 1202 | summary: "Remote result remains available.", |
| 1203 | }); |
| 1204 | const acknowledged = yield* a.projections.getThreadRecords(parentThreadId, ["subagents"], { |
| 1205 | subagentIds: [delegated.taskId], |
| 1206 | }); |
| 1207 | expect(acknowledged.subagents[0]?.completionDelivery).toEqual({ |
| 1208 | state: "acknowledged", |
| 1209 | observedByRunId: parentRunId, |
| 1210 | }); |
| 1211 | }), |
| 1212 | ).pipe(Effect.provide(NodeHttpServer.layerTest)), |
| 1213 | ); |
| 1214 | |
| 1215 | it.effect( |
| 1216 | "with the beta off here, a task already running there still ends, and no new one starts", |
| 1217 | () => |
| 1218 | Effect.scoped( |
| 1219 | Effect.gen(function* () { |
| 1220 | const thread = yield* makeBoxThread; |
| 1221 | const b = yield* serveBox(thread); |
| 1222 | const offers = yield* SubscriptionRef.make< |
| 1223 | ReadonlyArray<ProviderContinuationRequests.ProviderContinuationRequest> |
| 1224 | >([]); |
| 1225 | const database = yield* SqlitePersistence.layerMemory.pipe(Layer.build); |
| 1226 | const a = yield* makeLaptop(database, offers); |
| 1227 | yield* linkTo(a.links, b, "full-access"); |
| 1228 | yield* seedParent(a); |
| 1229 | const delegated = yield* delegateToBox(a); |
| 1230 | |
| 1231 | yield* a.settings.updateSettings({ linkedEnvironmentsEnabled: false }); |
| 1232 | const refused = yield* a.call("delegate_task", { |
| 1233 | task: "Run the linter on the box.", |
| 1234 | target: { |
| 1235 | environmentId: box.environmentId, |
| 1236 | providerInstanceId: instanceId, |
| 1237 | model: "gpt-5.4", |
| 1238 | }, |
| 1239 | mode: "async", |
| 1240 | clientRequestId: "lint-on-box", |
| 1241 | }); |
| 1242 | expect(refused.isError).toBe(true); |
| 1243 | expect(JSON.stringify(refused.content)).toContain( |
| 1244 | PeerForwarding.linkedEnvironmentsOff.message, |
| 1245 | ); |
| 1246 | expect((yield* a.orchestrator.getThreadProjection(parentThreadId)).subagents).toHaveLength( |
| 1247 | 1, |
| 1248 | ); |
| 1249 | |
| 1250 | yield* Ref.set(thread.reply, "All 412 tests pass on the box."); |
| 1251 | yield* Ref.set(thread.status, "completed"); |
| 1252 | yield* Deferred.succeed(thread.finished, undefined); |
| 1253 | expect(yield* taskResult(a, delegated.taskId)).toMatchObject({ |
| 1254 | status: "completed", |
| 1255 | result: "All 412 tests pass on the box.", |
| 1256 | }); |
| 1257 | }), |
| 1258 | ).pipe(Effect.provide(NodeHttpServer.layerTest)), |
| 1259 | ); |
| 1260 | |
| 1261 | it.effect("a task named only by environment inherits the parent's model, within the link", () => |
| 1262 | Effect.scoped( |
| 1263 | Effect.gen(function* () { |
| 1264 | const thread = yield* makeBoxThread; |
| 1265 | const b = yield* serveBox(thread); |
| 1266 | const offers = yield* SubscriptionRef.make< |
| 1267 | ReadonlyArray<ProviderContinuationRequests.ProviderContinuationRequest> |
| 1268 | >([]); |
| 1269 | const database = yield* SqlitePersistence.layerMemory.pipe(Layer.build); |
| 1270 | const a = yield* makeLaptop(database, offers); |
| 1271 | // The parent runs with full access; the link allows less. |
| 1272 | yield* linkTo(a.links, b, "auto-accept-edits"); |
| 1273 | // An explicit Standard must survive, or a Daybreak default there would replace it. |
| 1274 | const parentModelSelection = { |
| 1275 | ...modelSelection, |
| 1276 | options: [ |
| 1277 | { id: "reasoningEffort", value: "high" }, |
| 1278 | { id: "cyberAccessProgram", value: "standard" }, |
| 1279 | ], |
| 1280 | }; |
| 1281 | yield* seedParent(a, parentModelSelection); |
| 1282 | |
| 1283 | const result = yield* a.call("delegate_task", { |
| 1284 | task: "Run the suite on the box.", |
| 1285 | target: { environmentId: box.environmentId }, |
| 1286 | }); |
| 1287 | expect(result.isError, JSON.stringify(result.content)).toBe(false); |
| 1288 | // The parent's model and options, and the link's modes rather than the parent's. |
| 1289 | expect(yield* Ref.get(thread.launchedWith)).toEqual({ |
| 1290 | modelSelection: parentModelSelection, |
| 1291 | runtimeMode: "auto-accept-edits", |
| 1292 | interactionMode: "default", |
| 1293 | }); |
| 1294 | |
| 1295 | // Asking for more than the link allows is still refused there. |
| 1296 | const broader = yield* a.call("delegate_task", { |
| 1297 | task: "Run the suite on the box.", |
| 1298 | target: { environmentId: box.environmentId }, |
| 1299 | runtimeMode: "full-access", |
| 1300 | clientRequestId: "broader", |
| 1301 | }); |
| 1302 | expect(broader.isError).toBe(true); |
| 1303 | }), |
| 1304 | ).pipe(Effect.provide(NodeHttpServer.layerTest)), |
| 1305 | ); |
| 1306 | |
| 1307 | it.effect("a task naming another driver there never runs its model on the parent's instance", () => |
| 1308 | Effect.scoped( |
| 1309 | Effect.gen(function* () { |
| 1310 | const thread = yield* makeBoxThread; |
| 1311 | const b = yield* serveBox(thread); |
| 1312 | const offers = yield* SubscriptionRef.make< |
| 1313 | ReadonlyArray<ProviderContinuationRequests.ProviderContinuationRequest> |
| 1314 | >([]); |
| 1315 | const database = yield* SqlitePersistence.layerMemory.pipe(Layer.build); |
| 1316 | const a = yield* makeLaptop(database, offers); |
| 1317 | yield* linkTo(a.links, b, "full-access"); |
| 1318 | yield* seedParent(a); |
| 1319 | |
| 1320 | const result = yield* a.call("delegate_task", { |
| 1321 | task: "Run the suite on the box.", |
| 1322 | target: { |
| 1323 | environmentId: box.environmentId, |
| 1324 | driverKind: "claudeAgent", |
| 1325 | model: "claude-sonnet-4-6", |
| 1326 | }, |
| 1327 | }); |
| 1328 | expect(result.isError).toBe(false); |
| 1329 | expect((yield* Ref.get(thread.launchedWith))?.modelSelection.instanceId).toBe( |
| 1330 | boxClaude.instanceId, |
| 1331 | ); |
| 1332 | }), |
| 1333 | ).pipe(Effect.provide(NodeHttpServer.layerTest)), |
| 1334 | ); |
| 1335 | |
| 1336 | it.effect("refuses worktree isolation for a task there before launching anything", () => |
| 1337 | Effect.scoped( |
| 1338 | Effect.gen(function* () { |
| 1339 | const thread = yield* makeBoxThread; |
| 1340 | const b = yield* serveBox(thread); |
| 1341 | const offers = yield* SubscriptionRef.make< |
| 1342 | ReadonlyArray<ProviderContinuationRequests.ProviderContinuationRequest> |
| 1343 | >([]); |
| 1344 | const database = yield* SqlitePersistence.layerMemory.pipe(Layer.build); |
| 1345 | const a = yield* makeLaptop(database, offers); |
| 1346 | yield* linkTo(a.links, b, "auto"); |
| 1347 | yield* seedParent(a); |
| 1348 | |
| 1349 | const result = yield* a.call("delegate_task", { |
| 1350 | task: "Run the suite on the box.", |
| 1351 | target: { environmentId: box.environmentId }, |
| 1352 | isolation: "worktree", |
| 1353 | }); |
| 1354 | expect(result.isError).toBe(true); |
| 1355 | expect(JSON.stringify(result.content)).toContain("isolation"); |
| 1356 | expect(yield* Ref.get(thread.launched)).toBeNull(); |
| 1357 | }), |
| 1358 | ).pipe(Effect.provide(NodeHttpServer.layerTest)), |
| 1359 | ); |
| 1360 | |
| 1361 | it.effect("follows an open task again after the laptop restarts", () => |
| 1362 | Effect.scoped( |
| 1363 | Effect.gen(function* () { |
| 1364 | const thread = yield* makeBoxThread; |
| 1365 | const b = yield* serveBox(thread); |
| 1366 | const offers = yield* SubscriptionRef.make< |
| 1367 | ReadonlyArray<ProviderContinuationRequests.ProviderContinuationRequest> |
| 1368 | >([]); |
| 1369 | const database = yield* SqlitePersistence.layerMemory.pipe(Layer.build); |
| 1370 | // The laptop before the restart delegates, then goes away. |
| 1371 | const delegated = yield* Effect.scoped( |
| 1372 | Effect.gen(function* () { |
| 1373 | const before = yield* makeLaptop(database, offers); |
| 1374 | yield* linkTo(before.links, b, "full-access"); |
| 1375 | yield* seedParent(before); |
| 1376 | return yield* delegateToBox(before); |
| 1377 | }), |
| 1378 | ); |
| 1379 | // The task finishes on the box while the laptop is down. |
| 1380 | yield* Ref.set(thread.reply, "Finished while you were away."); |
| 1381 | yield* Ref.set(thread.status, "completed"); |
| 1382 | yield* Deferred.succeed(thread.finished, undefined); |
| 1383 | |
| 1384 | const after = yield* makeLaptop(database, offers); |
| 1385 | // The link lives in the laptop's own store, so it needs linking again here. |
| 1386 | yield* linkTo(after.links, b, "full-access"); |
| 1387 | yield* after.remote.start(); |
| 1388 | const task = yield* taskResult(after, delegated.taskId); |
| 1389 | expect(task).toMatchObject({ status: "completed", result: "Finished while you were away." }); |
| 1390 | }), |
| 1391 | ).pipe(Effect.provide(NodeHttpServer.layerTest)), |
| 1392 | ); |
| 1393 | |
| 1394 | it.effect("fails the task when the box revokes the link, and cancel interrupts it there", () => |
| 1395 | Effect.scoped( |
| 1396 | Effect.gen(function* () { |
| 1397 | const thread = yield* makeBoxThread; |
| 1398 | const b = yield* serveBox(thread); |
| 1399 | const offers = yield* SubscriptionRef.make< |
| 1400 | ReadonlyArray<ProviderContinuationRequests.ProviderContinuationRequest> |
| 1401 | >([]); |
| 1402 | const database = yield* SqlitePersistence.layerMemory.pipe(Layer.build); |
| 1403 | const a = yield* makeLaptop(database, offers); |
| 1404 | yield* linkTo(a.links, b, "full-access"); |
| 1405 | yield* seedParent(a); |
| 1406 | |
| 1407 | const cancelled = yield* delegateToBox(a); |
| 1408 | const cancel = yield* a.call("task_cancel", { |
| 1409 | taskId: cancelled.taskId, |
| 1410 | reason: "Not needed", |
| 1411 | }); |
| 1412 | expect(cancel.structuredContent).toEqual({ |
| 1413 | taskId: cancelled.taskId, |
| 1414 | status: "cancel_requested", |
| 1415 | }); |
| 1416 | // Cancelling stops the thread there as Stop does: its queue and its own tasks too. |
| 1417 | expect(yield* Ref.get(thread.stops)).toEqual(["thread.stop", "delegated-tasks"]); |
| 1418 | expect(yield* taskResult(a, cancelled.taskId)).toMatchObject({ status: "cancelled" }); |
| 1419 | |
| 1420 | // Cancelling again, once the result is in, still stops what came after it there. |
| 1421 | const again = yield* a.call("task_cancel", { |
| 1422 | taskId: cancelled.taskId, |
| 1423 | clientRequestId: "cancel-again", |
| 1424 | }); |
| 1425 | expect(again.structuredContent).toEqual({ taskId: cancelled.taskId, status: "cancelled" }); |
| 1426 | expect(yield* Ref.get(thread.interrupted)).toBe(2); |
| 1427 | // ...and a retry of that cancel stops nothing new. |
| 1428 | yield* a.call("task_cancel", { taskId: cancelled.taskId, clientRequestId: "cancel-again" }); |
| 1429 | expect(yield* Ref.get(thread.interrupted)).toBe(2); |
| 1430 | |
| 1431 | // A second task, then the box revokes the link before it ends. |
| 1432 | const revoked = yield* a |
| 1433 | .call("delegate_task", { |
| 1434 | task: "Another one.", |
| 1435 | target: { |
| 1436 | environmentId: box.environmentId, |
| 1437 | providerInstanceId: instanceId, |
| 1438 | model: "gpt-5.4", |
| 1439 | }, |
| 1440 | clientRequestId: "second", |
| 1441 | }) |
| 1442 | .pipe(Effect.map((result) => result.structuredContent as { taskId: NodeId })); |
| 1443 | const [session] = yield* b.linkedSessions; |
| 1444 | yield* b.auth.revokeSession(session!.sessionId); |
| 1445 | yield* Deferred.succeed(thread.finished, undefined); |
| 1446 | const failed = yield* taskResult(a, revoked.taskId); |
| 1447 | expect(failed?.status).toBe("failed"); |
| 1448 | expect(failed?.result).toBe( |
| 1449 | "The linked environment no longer accepts this link. It may have been revoked there; link it again.", |
| 1450 | ); |
| 1451 | }), |
| 1452 | ).pipe(Effect.provide(NodeHttpServer.layerTest)), |
| 1453 | ); |
| 1454 | |
| 1455 | it.effect("a task's result is its own run's whole last reply, not a later run's", () => |
| 1456 | Effect.scoped( |
| 1457 | Effect.gen(function* () { |
| 1458 | const thread = yield* makeBoxThread; |
| 1459 | const b = yield* serveBox(thread); |
| 1460 | const offers = yield* SubscriptionRef.make< |
| 1461 | ReadonlyArray<ProviderContinuationRequests.ProviderContinuationRequest> |
| 1462 | >([]); |
| 1463 | const database = yield* SqlitePersistence.layerMemory.pipe(Layer.build); |
| 1464 | const a = yield* makeLaptop(database, offers); |
| 1465 | yield* linkTo(a.links, b, "full-access"); |
| 1466 | yield* seedParent(a); |
| 1467 | |
| 1468 | const result = yield* a.call("delegate_task", { |
| 1469 | task: "Review the diff.", |
| 1470 | role: "review", |
| 1471 | target: { environmentId: box.environmentId }, |
| 1472 | clientRequestId: "review-on-box", |
| 1473 | }); |
| 1474 | expect(result.isError, JSON.stringify(result.content)).toBe(false); |
| 1475 | const { taskId } = result.structuredContent as { taskId: NodeId }; |
| 1476 | // The role reaches the thread there, as it does a child here. |
| 1477 | expect(yield* Ref.get(thread.message)).toBe( |
| 1478 | "Act as the review sub-agent for this task.\n\nReview the diff.", |
| 1479 | ); |
| 1480 | |
| 1481 | // A reply longer than one read, then a later run's reply on the same thread. |
| 1482 | const long = "x".repeat(45_000); |
| 1483 | yield* Ref.set(thread.reply, long); |
| 1484 | yield* Ref.set(thread.laterReply, "Someone else asked a follow-up."); |
| 1485 | yield* Ref.set(thread.executedModel, "gpt-5.4-mini"); |
| 1486 | yield* Ref.set(thread.status, "completed"); |
| 1487 | yield* Deferred.succeed(thread.finished, undefined); |
| 1488 | const task = yield* taskResult(a, taskId); |
| 1489 | expect(task?.status).toBe("completed"); |
| 1490 | expect(task?.result).toBe( |
| 1491 | `Requested model gpt-5.4; the provider reported running gpt-5.4-mini.\n\n${long}`, |
| 1492 | ); |
| 1493 | const status = yield* a.call("task_status", { taskId }); |
| 1494 | expect(status.structuredContent).toMatchObject({ |
| 1495 | status: "completed", |
| 1496 | executedModel: "gpt-5.4-mini", |
| 1497 | }); |
| 1498 | // Every wait was for the run the launch started. |
| 1499 | const waits = yield* Ref.get(thread.waitedFor); |
| 1500 | expect(waits.length).toBeGreaterThan(0); |
| 1501 | expect(waits.every((runId) => runId === boxRunId)).toBe(true); |
| 1502 | }), |
| 1503 | ).pipe(Effect.provide(NodeHttpServer.layerTest)), |
| 1504 | ); |
| 1505 | |
| 1506 | it.effect("a task whose thread there waits on its own task ends with the answer after it", () => |
| 1507 | Effect.scoped( |
| 1508 | Effect.gen(function* () { |
| 1509 | const thread = yield* makeBoxThread; |
| 1510 | const b = yield* serveBox(thread); |
| 1511 | const offers = yield* SubscriptionRef.make< |
| 1512 | ReadonlyArray<ProviderContinuationRequests.ProviderContinuationRequest> |
| 1513 | >([]); |
| 1514 | const database = yield* SqlitePersistence.layerMemory.pipe(Layer.build); |
| 1515 | const a = yield* makeLaptop(database, offers); |
| 1516 | yield* linkTo(a.links, b, "full-access"); |
| 1517 | yield* seedParent(a); |
| 1518 | const delegated = yield* delegateToBox(a); |
| 1519 | |
| 1520 | // The task's run there ends after delegating a review of its own. |
| 1521 | yield* Ref.set(thread.executedModel, "gpt-5.4-mini"); |
| 1522 | yield* Ref.set(thread.nested, true); |
| 1523 | yield* Ref.set(thread.reply, "Started a review; I will answer when it reports."); |
| 1524 | yield* Ref.set(thread.status, "completed"); |
| 1525 | yield* Deferred.succeed(thread.finished, undefined); |
| 1526 | const first = yield* Effect.raceFirst( |
| 1527 | Deferred.await(thread.nestedWaiting).pipe(Effect.as("waiting there" as const)), |
| 1528 | taskResult(a, delegated.taskId).pipe(Effect.as("completed here" as const)), |
| 1529 | ); |
| 1530 | expect(first).toBe("waiting there"); |
| 1531 | |
| 1532 | // The review reports and wakes the thread there, whose next run answers. |
| 1533 | yield* Ref.set(thread.laterReply, "The review found two issues; both are fixed."); |
| 1534 | yield* Deferred.succeed(thread.nestedSettled, undefined); |
| 1535 | expect(yield* taskResult(a, delegated.taskId)).toMatchObject({ |
| 1536 | status: "completed", |
| 1537 | result: "The review found two issues; both are fixed.", |
| 1538 | }); |
| 1539 | const status = yield* a.call("task_status", { taskId: delegated.taskId }); |
| 1540 | expect(status.structuredContent).toMatchObject({ |
| 1541 | status: "completed", |
| 1542 | executedModel: null, |
| 1543 | }); |
| 1544 | }), |
| 1545 | ).pipe(Effect.provide(NodeHttpServer.layerTest)), |
| 1546 | ); |
| 1547 | |
| 1548 | it.effect("the same key from another provider session is another task", () => |
| 1549 | Effect.scoped( |
| 1550 | Effect.gen(function* () { |
| 1551 | const thread = yield* makeBoxThread; |
| 1552 | const b = yield* serveBox(thread); |
| 1553 | const offers = yield* SubscriptionRef.make< |
| 1554 | ReadonlyArray<ProviderContinuationRequests.ProviderContinuationRequest> |
| 1555 | >([]); |
| 1556 | const database = yield* SqlitePersistence.layerMemory.pipe(Layer.build); |
| 1557 | const a = yield* makeLaptop(database, offers); |
| 1558 | yield* linkTo(a.links, b, "full-access"); |
| 1559 | yield* seedParent(a); |
| 1560 | |
| 1561 | const first = yield* delegateToBox(a); |
| 1562 | const later = yield* a.call( |
| 1563 | "delegate_task", |
| 1564 | { |
| 1565 | task: "Run the full suite on the box.", |
| 1566 | target: { environmentId: box.environmentId }, |
| 1567 | clientRequestId: "suite-on-box", |
| 1568 | }, |
| 1569 | { ...parentScope, requestNamespace: "provider-session:laptop-parent-2" }, |
| 1570 | ); |
| 1571 | expect(later.isError, JSON.stringify(later.content)).toBe(false); |
| 1572 | const second = later.structuredContent as { |
| 1573 | taskId: NodeId; |
| 1574 | remoteChild: { threadId: ThreadId }; |
| 1575 | }; |
| 1576 | expect(second.taskId).not.toBe(first.taskId); |
| 1577 | expect(second.remoteChild.threadId).not.toBe(first.remoteChild.threadId); |
| 1578 | expect((yield* a.orchestrator.getThreadProjection(parentThreadId)).subagents).toHaveLength(2); |
| 1579 | }), |
| 1580 | ).pipe(Effect.provide(NodeHttpServer.layerTest)), |
| 1581 | ); |
| 1582 | |
| 1583 | it.effect.each([null, "Working before the provider failed."])( |
| 1584 | "reports the failed remote run's error instead of its earlier reply (%s)", |
| 1585 | (reply) => |
| 1586 | Effect.scoped( |
| 1587 | Effect.gen(function* () { |
| 1588 | const thread = yield* makeBoxThread; |
| 1589 | const b = yield* serveBox(thread); |
| 1590 | const offers = yield* SubscriptionRef.make< |
| 1591 | ReadonlyArray<ProviderContinuationRequests.ProviderContinuationRequest> |
| 1592 | >([]); |
| 1593 | const database = yield* SqlitePersistence.layerMemory.pipe(Layer.build); |
| 1594 | const a = yield* makeLaptop(database, offers); |
| 1595 | yield* linkTo(a.links, b, "full-access"); |
| 1596 | yield* seedParent(a); |
| 1597 | const delegated = yield* delegateToBox(a); |
| 1598 | yield* Ref.set(thread.reply, reply); |
| 1599 | const error = "Provider failed: authentication expired on the destination."; |
| 1600 | yield* Ref.set(thread.error, error); |
| 1601 | yield* Ref.set(thread.status, "failed"); |
| 1602 | yield* Deferred.succeed(thread.finished, undefined); |
| 1603 | expect(yield* taskResult(a, delegated.taskId)).toMatchObject({ |
| 1604 | status: "failed", |
| 1605 | result: error, |
| 1606 | }); |
| 1607 | const status = yield* a.call("task_status", { taskId: delegated.taskId }); |
| 1608 | expect(status.structuredContent).toMatchObject({ |
| 1609 | status: "failed", |
| 1610 | summary: error, |
| 1611 | latestTerminalSummary: error, |
| 1612 | }); |
| 1613 | const offered = yield* SubscriptionRef.changes(offers).pipe( |
| 1614 | Stream.filter((seen) => seen.some((offer) => offer.threadId === parentThreadId)), |
| 1615 | Stream.runHead, |
| 1616 | ); |
| 1617 | expect(Option.isSome(offered)).toBe(true); |
| 1618 | }), |
| 1619 | ).pipe(Effect.provide(NodeHttpServer.layerTest)), |
| 1620 | ); |
| 1621 | |
| 1622 | it.effect.each([false, true])( |
| 1623 | "ignores later-run errors and reads the failed run's full error (ownError=%s)", |
| 1624 | (ownError) => |
| 1625 | Effect.scoped( |
| 1626 | Effect.gen(function* () { |
| 1627 | const thread = yield* makeBoxThread; |
| 1628 | const b = yield* serveBox(thread); |
| 1629 | const offers = yield* SubscriptionRef.make< |
| 1630 | ReadonlyArray<ProviderContinuationRequests.ProviderContinuationRequest> |
| 1631 | >([]); |
| 1632 | const database = yield* SqlitePersistence.layerMemory.pipe(Layer.build); |
| 1633 | const a = yield* makeLaptop(database, offers); |
| 1634 | yield* linkTo(a.links, b, "full-access"); |
| 1635 | yield* seedParent(a); |
| 1636 | const delegated = yield* delegateToBox(a); |
| 1637 | yield* Ref.set(thread.reply, null); |
| 1638 | const error = `Provider failed: ${"detail ".repeat(10_000)}end of error.`; |
| 1639 | if (ownError) yield* Ref.set(thread.error, error); |
| 1640 | yield* Ref.set(thread.laterReply, "A later user run answered."); |
| 1641 | yield* Ref.set(thread.laterError, "Another run's provider error."); |
| 1642 | yield* Ref.set(thread.status, "failed"); |
| 1643 | yield* Deferred.succeed(thread.finished, undefined); |
| 1644 | expect(yield* taskResult(a, delegated.taskId)).toMatchObject({ |
| 1645 | status: "failed", |
| 1646 | result: ownError |
| 1647 | ? error |
| 1648 | : "(The thread in the linked environment ended without a reply.)", |
| 1649 | }); |
| 1650 | expect(yield* Ref.get(thread.waitedFor)).toEqual([boxRunId]); |
| 1651 | }), |
| 1652 | ).pipe(Effect.provide(NodeHttpServer.layerTest)), |
| 1653 | ); |
| 1654 | |
| 1655 | it.effect.each(["tail-error", "middle-error", "reply-fallback", "cancel-refill"] as const)( |
| 1656 | "reads a failed task through full activity pages (%s)", |
| 1657 | (scenario) => |
| 1658 | Effect.gen(function* () { |
| 1659 | const { reads, fetch } = captureThreadReads(yield* FetchHttpClient.Fetch); |
| 1660 | return yield* Effect.scoped( |
| 1661 | Effect.gen(function* () { |
| 1662 | const thread = yield* makeBoxThread; |
| 1663 | const b = yield* serveBox(thread); |
| 1664 | const offers = yield* SubscriptionRef.make< |
| 1665 | ReadonlyArray<ProviderContinuationRequests.ProviderContinuationRequest> |
| 1666 | >([]); |
| 1667 | const database = yield* SqlitePersistence.layerMemory.pipe(Layer.build); |
| 1668 | const a = yield* makeLaptop(database, offers); |
| 1669 | yield* linkTo(a.links, b, "full-access"); |
| 1670 | yield* seedParent(a); |
| 1671 | const delegated = yield* delegateToBox(a); |
| 1672 | const result = `Provider details: ${"detail ".repeat(8_000)}end.`; |
| 1673 | const middle = scenario !== "tail-error"; |
| 1674 | const ownType = scenario === "reply-fallback" ? "assistant_message" : "error"; |
| 1675 | const history = (count: number, runId: RunId, prefix: string): Array<TimelineEntry> => |
| 1676 | Array.from({ length: count }, (_, index) => ({ |
| 1677 | id: TurnItemId.make(`${prefix}:${index}`), |
| 1678 | runId, |
| 1679 | text: "discarded history ".repeat(120), |
| 1680 | type: "assistant_message", |
| 1681 | })); |
| 1682 | const timeline: Array<TimelineEntry> = [ |
| 1683 | ...history(middle ? 150 : 300, boxRunId, "before"), |
| 1684 | { id: TurnItemId.make("selected"), runId: boxRunId, text: result, type: ownType }, |
| 1685 | ...(middle ? history(200, boxLaterRunId, "later") : []), |
| 1686 | { |
| 1687 | id: TurnItemId.make("other-error"), |
| 1688 | runId: boxLaterRunId, |
| 1689 | text: "Wrong run", |
| 1690 | type: "error", |
| 1691 | }, |
| 1692 | ]; |
| 1693 | yield* Ref.set(thread.timeline, timeline); |
| 1694 | yield* Ref.set(thread.pageSize, 100); |
| 1695 | yield* Ref.set(thread.status, "failed"); |
| 1696 | if (scenario === "cancel-refill") yield* Ref.set(thread.holdRefill, true); |
| 1697 | yield* Deferred.succeed(thread.finished, undefined); |
| 1698 | if (scenario === "cancel-refill") { |
| 1699 | yield* Deferred.await(thread.readStarted); |
| 1700 | yield* a.threads.stopDelegatedTasks({ |
| 1701 | threadId: parentThreadId, |
| 1702 | commandId: CommandId.make("command:stop:mid-refill"), |
| 1703 | }); |
| 1704 | yield* Deferred.succeed(thread.releaseRead, undefined); |
| 1705 | expect(yield* taskResult(a, delegated.taskId)).toMatchObject({ status: "interrupted" }); |
| 1706 | yield* Deferred.await(thread.interruptedOnce); |
| 1707 | expect(yield* Ref.get(thread.stops)).toContain("thread.stop"); |
| 1708 | } else { |
| 1709 | expect(yield* taskResult(a, delegated.taskId)).toMatchObject({ |
| 1710 | status: "failed", |
| 1711 | result, |
| 1712 | }); |
| 1713 | } |
| 1714 | const pages = reads.filter((read) => read.itemId === undefined && read.limit === 100); |
| 1715 | // A page moves on by however many items the response budget left it. |
| 1716 | const remaining = (read: OrchestratorMcpThreadReadInput) => |
| 1717 | timeline.length - 1 - (read.afterPosition ?? -1); |
| 1718 | expect(pages[0]).toMatchObject({ |
| 1719 | view: "activity", |
| 1720 | limit: 100, |
| 1721 | maxCharsPerItem: 20_000, |
| 1722 | }); |
| 1723 | expect(pages.slice(1).some((read) => remaining(read) > 100)).toBe(true); |
| 1724 | for (const read of pages.slice(1)) |
| 1725 | expect(read.maxCharsPerItem).toBe(remaining(read) > 100 ? 1 : 20_000); |
| 1726 | const forwards = pages.slice(1).map((read) => read.afterPosition ?? -1); |
| 1727 | expect(forwards).toEqual([...forwards].sort((left, right) => left - right)); |
| 1728 | expect(new Set(forwards).size).toBe(forwards.length); |
| 1729 | const refills = reads.filter((read) => read.itemId !== undefined); |
| 1730 | expect(refills[0]).toMatchObject({ itemId: "selected", textOffset: middle ? 1 : 20_000 }); |
| 1731 | if (scenario !== "cancel-refill") expect(refills).toHaveLength(middle ? 3 : 2); |
| 1732 | }), |
| 1733 | ).pipe( |
| 1734 | Effect.provide(NodeHttpServer.layerTest), |
| 1735 | Effect.provideService(FetchHttpClient.Fetch, fetch), |
| 1736 | ); |
| 1737 | }), |
| 1738 | ); |
| 1739 | |
| 1740 | it.effect.each([ |
| 1741 | "terminal", |
| 1742 | "long", |
| 1743 | "later-run", |
| 1744 | "sparse", |
| 1745 | "growing", |
| 1746 | "shrinking", |
| 1747 | "cancel", |
| 1748 | "no-run", |
| 1749 | "cancelled", |
| 1750 | "interrupted", |
| 1751 | ] as const)("reads a successful task with an aligned tail probe (%s)", (scenario) => |
| 1752 | Effect.gen(function* () { |
| 1753 | const { reads, fetch } = captureThreadReads(yield* FetchHttpClient.Fetch); |
| 1754 | return yield* Effect.scoped( |
| 1755 | Effect.gen(function* () { |
| 1756 | const thread = yield* makeBoxThread; |
| 1757 | const b = yield* serveBox(thread); |
| 1758 | const offers = yield* SubscriptionRef.make< |
| 1759 | ReadonlyArray<ProviderContinuationRequests.ProviderContinuationRequest> |
| 1760 | >([]); |
| 1761 | const database = yield* SqlitePersistence.layerMemory.pipe(Layer.build); |
| 1762 | const a = yield* makeLaptop(database, offers); |
| 1763 | yield* linkTo(a.links, b, "full-access"); |
| 1764 | yield* seedParent(a); |
| 1765 | const result = |
| 1766 | scenario === "long" |
| 1767 | ? `Complete ${"detail ".repeat(12_000)}end.` |
| 1768 | : "Authoritative result."; |
| 1769 | const middle = scenario === "later-run" || scenario === "shrinking"; |
| 1770 | const history = (count: number, runId: RunId, prefix: string): Array<TimelineEntry> => |
| 1771 | Array.from({ length: count }, (_, index) => ({ |
| 1772 | id: TurnItemId.make(`${prefix}:${index}`), |
| 1773 | runId, |
| 1774 | text: "History", |
| 1775 | type: scenario === "sparse" && index % 2 === 0 ? "error" : "assistant_message", |
| 1776 | })); |
| 1777 | const timeline: Array<TimelineEntry> = [ |
| 1778 | ...history(middle ? 150 : 300, boxRunId, "before"), |
| 1779 | { |
| 1780 | id: TurnItemId.make("selected"), |
| 1781 | runId: scenario === "no-run" ? boxLaterRunId : boxRunId, |
| 1782 | text: result, |
| 1783 | type: "assistant_message", |
| 1784 | }, |
| 1785 | ...(middle ? history(200, boxLaterRunId, "later") : []), |
| 1786 | ]; |
| 1787 | yield* Ref.set(thread.timeline, timeline); |
| 1788 | yield* Ref.set(thread.pageSize, 100); |
| 1789 | if (scenario === "growing") { |
| 1790 | yield* Ref.set( |
| 1791 | thread.beforeTail, |
| 1792 | Ref.set(thread.timeline, [...timeline, ...history(101, boxLaterRunId, "growth")]), |
| 1793 | ); |
| 1794 | } |
| 1795 | if (scenario === "shrinking") { |
| 1796 | yield* Ref.set(thread.beforeTail, Ref.set(thread.timeline, timeline.slice(0, 151))); |
| 1797 | } |
| 1798 | if (scenario === "cancel") { |
| 1799 | yield* Ref.set( |
| 1800 | thread.beforeTail, |
| 1801 | Effect.gen(function* () { |
| 1802 | yield* Deferred.succeed(thread.readStarted, undefined); |
| 1803 | yield* Deferred.await(thread.releaseRead); |
| 1804 | }), |
| 1805 | ); |
| 1806 | } |
| 1807 | if (scenario === "no-run") yield* Ref.set(thread.runless, true); |
| 1808 | const delegated = yield* delegateToBox(a); |
| 1809 | const status = |
| 1810 | scenario === "cancelled" || scenario === "interrupted" ? scenario : "completed"; |
| 1811 | const modelNote = scenario === "terminal" || scenario === "long"; |
| 1812 | if (modelNote) yield* Ref.set(thread.executedModel, "gpt-5.4-mini"); |
| 1813 | yield* Ref.set(thread.status, status); |
| 1814 | yield* Deferred.succeed(thread.finished, undefined); |
| 1815 | if (scenario === "cancel") { |
| 1816 | yield* Deferred.await(thread.readStarted); |
| 1817 | yield* a.threads.stopDelegatedTasks({ |
| 1818 | threadId: parentThreadId, |
| 1819 | commandId: CommandId.make("command:stop:tail-probe"), |
| 1820 | }); |
| 1821 | yield* Deferred.succeed(thread.releaseRead, undefined); |
| 1822 | expect(yield* taskResult(a, delegated.taskId)).toMatchObject({ status: "interrupted" }); |
| 1823 | yield* Deferred.await(thread.interruptedOnce); |
| 1824 | expect(yield* Ref.get(thread.stops)).toContain("thread.stop"); |
| 1825 | } else { |
| 1826 | expect(yield* taskResult(a, delegated.taskId)).toMatchObject({ |
| 1827 | status, |
| 1828 | result: modelNote |
| 1829 | ? `Requested model gpt-5.4; the provider reported running gpt-5.4-mini.\n\n${result}` |
| 1830 | : result, |
| 1831 | }); |
| 1832 | } |
| 1833 | const pages = reads.filter((read) => read.itemId === undefined && read.limit === 100); |
| 1834 | // The thread's final page starts at its canonical 100-item boundary. |
| 1835 | const tailPosition = Math.floor((timeline.length - 1) / 100) * 100 - 1; |
| 1836 | expect(pages.at(0)?.afterPosition).toBeUndefined(); |
| 1837 | expect( |
| 1838 | pages.every( |
| 1839 | (read) => |
| 1840 | read.view === "messages" && read.limit === 100 && read.maxCharsPerItem === 20_000, |
| 1841 | ), |
| 1842 | ).toBe(true); |
| 1843 | if (scenario === "cancelled" || scenario === "interrupted") { |
| 1844 | // A run that did not complete keeps the established scan, without the probe. |
| 1845 | expect(pages.map((read) => read.afterPosition)).not.toContain(tailPosition); |
| 1846 | const forwards = pages.slice(1).map((read) => read.afterPosition ?? -1); |
| 1847 | expect(forwards.length).toBeGreaterThan(0); |
| 1848 | expect(forwards).toEqual([...forwards].sort((left, right) => left - right)); |
| 1849 | } else { |
| 1850 | expect(pages.at(1)?.afterPosition).toBe(tailPosition); |
| 1851 | const rest = pages.slice(2); |
| 1852 | if (scenario === "later-run" || scenario === "shrinking" || scenario === "growing") { |
| 1853 | // An inconclusive tail falls back to scanning from where the first page ended. |
| 1854 | expect(rest.length).toBeGreaterThan(0); |
| 1855 | const resumed = rest.map((read) => read.afterPosition ?? -1); |
| 1856 | expect(resumed[0]).toBeLessThan(tailPosition); |
| 1857 | expect(resumed).toEqual([...resumed].sort((left, right) => left - right)); |
| 1858 | } else { |
| 1859 | // A terminal page holding the run's last reply ends the read there. |
| 1860 | expect(rest).toHaveLength(0); |
| 1861 | } |
| 1862 | } |
| 1863 | const refills = reads.filter((read) => read.itemId !== undefined); |
| 1864 | expect(refills).toHaveLength(scenario === "long" ? 4 : 0); |
| 1865 | if (scenario === "long") |
| 1866 | expect(refills.map((read) => read.textOffset)).toEqual([20_000, 40_000, 60_000, 80_000]); |
| 1867 | }), |
| 1868 | ).pipe( |
| 1869 | Effect.provide(NodeHttpServer.layerTest), |
| 1870 | Effect.provideService(FetchHttpClient.Fetch, fetch), |
| 1871 | ); |
| 1872 | }), |
| 1873 | ); |
| 1874 | |
| 1875 | it.effect("keeps a successful remote reply despite an error item in the run's activity", () => |
| 1876 | Effect.scoped( |
| 1877 | Effect.gen(function* () { |
| 1878 | const thread = yield* makeBoxThread; |
| 1879 | const b = yield* serveBox(thread); |
| 1880 | const offers = yield* SubscriptionRef.make< |
| 1881 | ReadonlyArray<ProviderContinuationRequests.ProviderContinuationRequest> |
| 1882 | >([]); |
| 1883 | const database = yield* SqlitePersistence.layerMemory.pipe(Layer.build); |
| 1884 | const a = yield* makeLaptop(database, offers); |
| 1885 | yield* linkTo(a.links, b, "full-access"); |
| 1886 | yield* seedParent(a); |
| 1887 | const delegated = yield* delegateToBox(a); |
| 1888 | yield* Ref.set(thread.reply, "Recovered and completed the task."); |
| 1889 | yield* Ref.set(thread.error, "A recoverable tool error."); |
| 1890 | yield* Ref.set(thread.status, "completed"); |
| 1891 | yield* Deferred.succeed(thread.finished, undefined); |
| 1892 | expect(yield* taskResult(a, delegated.taskId)).toMatchObject({ |
| 1893 | status: "completed", |
| 1894 | result: "Recovered and completed the task.", |
| 1895 | }); |
| 1896 | }), |
| 1897 | ).pipe(Effect.provide(NodeHttpServer.layerTest)), |
| 1898 | ); |
| 1899 | |
| 1900 | it.effect("fails the task when its thread there is gone, or the link here is", () => |
| 1901 | Effect.scoped( |
| 1902 | Effect.gen(function* () { |
| 1903 | const thread = yield* makeBoxThread; |
| 1904 | const b = yield* serveBox(thread); |
| 1905 | const offers = yield* SubscriptionRef.make< |
| 1906 | ReadonlyArray<ProviderContinuationRequests.ProviderContinuationRequest> |
| 1907 | >([]); |
| 1908 | const database = yield* SqlitePersistence.layerMemory.pipe(Layer.build); |
| 1909 | const a = yield* makeLaptop(database, offers); |
| 1910 | yield* linkTo(a.links, b, "full-access"); |
| 1911 | yield* seedParent(a); |
| 1912 | |
| 1913 | yield* Ref.set(thread.gone, true); |
| 1914 | const gone = yield* delegateToBox(a); |
| 1915 | const failed = yield* taskResult(a, gone.taskId); |
| 1916 | expect(failed?.status).toBe("failed"); |
| 1917 | expect(failed?.result).toContain("was not found"); |
| 1918 | |
| 1919 | // The link is forgotten here while the task runs there. |
| 1920 | yield* Ref.set(thread.gone, false); |
| 1921 | const forgotten = yield* a |
| 1922 | .call("delegate_task", { |
| 1923 | task: "Another one.", |
| 1924 | target: { environmentId: box.environmentId }, |
| 1925 | clientRequestId: "forgotten", |
| 1926 | }) |
| 1927 | .pipe(Effect.map((result) => result.structuredContent as { taskId: NodeId })); |
| 1928 | yield* a.links.unlink(box.environmentId); |
| 1929 | yield* Ref.set(thread.status, "completed"); |
| 1930 | yield* Deferred.succeed(thread.finished, undefined); |
| 1931 | const unlinked = yield* taskResult(a, forgotten.taskId); |
| 1932 | expect(unlinked?.status).toBe("failed"); |
| 1933 | expect(unlinked?.result).toContain("not linked to"); |
| 1934 | }), |
| 1935 | ).pipe(Effect.provide(NodeHttpServer.layerTest)), |
| 1936 | ); |
| 1937 | |
| 1938 | it.effect("cancel ends the task here even when the link is gone", () => |
| 1939 | Effect.scoped( |
| 1940 | Effect.gen(function* () { |
| 1941 | const thread = yield* makeBoxThread; |
| 1942 | const b = yield* serveBox(thread); |
| 1943 | const offers = yield* SubscriptionRef.make< |
| 1944 | ReadonlyArray<ProviderContinuationRequests.ProviderContinuationRequest> |
| 1945 | >([]); |
| 1946 | const database = yield* SqlitePersistence.layerMemory.pipe(Layer.build); |
| 1947 | const a = yield* makeLaptop(database, offers); |
| 1948 | yield* linkTo(a.links, b, "full-access"); |
| 1949 | yield* seedParent(a); |
| 1950 | |
| 1951 | const delegated = yield* delegateToBox(a); |
| 1952 | yield* a.links.unlink(box.environmentId); |
| 1953 | const cancel = yield* a.call("task_cancel", { taskId: delegated.taskId }); |
| 1954 | expect(cancel.structuredContent).toEqual({ |
| 1955 | taskId: delegated.taskId, |
| 1956 | status: "cancel_requested", |
| 1957 | }); |
| 1958 | const task = yield* taskResult(a, delegated.taskId); |
| 1959 | expect(task?.status).toBe("cancelled"); |
| 1960 | expect(task?.result).toContain("could not be stopped in Box"); |
| 1961 | expect(yield* Ref.get(thread.interrupted)).toBe(0); |
| 1962 | }), |
| 1963 | ).pipe(Effect.provide(NodeHttpServer.layerTest)), |
| 1964 | ); |
| 1965 | |
| 1966 | it.effect("a parent stopped while the task's result is read still stops its thread there", () => |
| 1967 | Effect.scoped( |
| 1968 | Effect.gen(function* () { |
| 1969 | const thread = yield* makeBoxThread; |
| 1970 | const b = yield* serveBox(thread); |
| 1971 | const offers = yield* SubscriptionRef.make< |
| 1972 | ReadonlyArray<ProviderContinuationRequests.ProviderContinuationRequest> |
| 1973 | >([]); |
| 1974 | const database = yield* SqlitePersistence.layerMemory.pipe(Layer.build); |
| 1975 | const a = yield* makeLaptop(database, offers); |
| 1976 | yield* linkTo(a.links, b, "full-access"); |
| 1977 | yield* seedParent(a); |
| 1978 | const delegated = yield* delegateToBox(a); |
| 1979 | |
| 1980 | // The task's run ends there, and while the follower reads its reply, |
| 1981 | // the parent here is stopped: more work may follow that run there. |
| 1982 | yield* Ref.set(thread.holdRead, true); |
| 1983 | yield* Ref.set(thread.reply, "Done there."); |
| 1984 | yield* Ref.set(thread.status, "completed"); |
| 1985 | yield* Deferred.succeed(thread.finished, undefined); |
| 1986 | yield* Deferred.await(thread.readStarted); |
| 1987 | yield* a.threads.stopDelegatedTasks({ |
| 1988 | threadId: parentThreadId, |
| 1989 | commandId: CommandId.make("command:stop:laptop-parent-mid-read"), |
| 1990 | }); |
| 1991 | yield* Deferred.succeed(thread.releaseRead, undefined); |
| 1992 | |
| 1993 | // The Stop's result stands, and the thread there is still stopped. |
| 1994 | expect(yield* taskResult(a, delegated.taskId)).toMatchObject({ status: "interrupted" }); |
| 1995 | yield* Deferred.await(thread.interruptedOnce); |
| 1996 | expect(yield* Ref.get(thread.stops)).toContain("thread.stop"); |
| 1997 | }), |
| 1998 | ).pipe(Effect.provide(NodeHttpServer.layerTest)), |
| 1999 | ); |
| 2000 | |
| 2001 | it.effect("a stop the box refuses because the parent was lowered ends instead of retrying", () => |
| 2002 | Effect.scoped( |
| 2003 | Effect.gen(function* () { |
| 2004 | const thread = yield* makeBoxThread; |
| 2005 | const b = yield* serveBox(thread); |
| 2006 | const offers = yield* SubscriptionRef.make< |
| 2007 | ReadonlyArray<ProviderContinuationRequests.ProviderContinuationRequest> |
| 2008 | >([]); |
| 2009 | const database = yield* SqlitePersistence.layerMemory.pipe(Layer.build); |
| 2010 | const a = yield* makeLaptop(database, offers); |
| 2011 | yield* linkTo(a.links, b, "full-access"); |
| 2012 | yield* seedParent(a); |
| 2013 | const delegated = yield* delegateToBox(a); |
| 2014 | |
| 2015 | // The parent is lowered below the full-access thread there, then stopped. |
| 2016 | yield* a.orchestrator.dispatch({ |
| 2017 | type: "thread.runtime-mode.set", |
| 2018 | commandId: CommandId.make("command:lower:laptop-parent"), |
| 2019 | threadId: parentThreadId, |
| 2020 | runtimeMode: "approval-required", |
| 2021 | }); |
| 2022 | yield* a.threads.stopDelegatedTasks({ |
| 2023 | threadId: parentThreadId, |
| 2024 | commandId: CommandId.make("command:stop:laptop-parent-lowered"), |
| 2025 | }); |
| 2026 | |
| 2027 | // The box refuses that stop; the follower records it as settled and |
| 2028 | // stops, so nothing retries it, now or after a restart. |
| 2029 | const settled = yield* a.orchestrator |
| 2030 | .streamStoredEventsFrom({ |
| 2031 | threadId: parentThreadId, |
| 2032 | afterSequence: 0, |
| 2033 | eventType: "subagent.updated", |
| 2034 | }) |
| 2035 | .pipe( |
| 2036 | Stream.filter( |
| 2037 | (stored) => |
| 2038 | stored.event.type === "subagent.updated" && |
| 2039 | stored.event.payload.id === delegated.taskId && |
| 2040 | stored.event.payload.remoteChild?.stoppedThere === true, |
| 2041 | ), |
| 2042 | Stream.runHead, |
| 2043 | ); |
| 2044 | expect(Option.isSome(settled)).toBe(true); |
| 2045 | expect(yield* Ref.get(thread.stops)).toEqual([]); |
| 2046 | expect(yield* a.projections.getOpenRemoteDelegatedTasks).toEqual([]); |
| 2047 | }), |
| 2048 | ).pipe(Effect.provide(NodeHttpServer.layerTest)), |
| 2049 | ); |
| 2050 | |
| 2051 | it.effect("stopping the parent stops its task there", () => |
| 2052 | Effect.scoped( |
| 2053 | Effect.gen(function* () { |
| 2054 | const thread = yield* makeBoxThread; |
| 2055 | const b = yield* serveBox(thread); |
| 2056 | const offers = yield* SubscriptionRef.make< |
| 2057 | ReadonlyArray<ProviderContinuationRequests.ProviderContinuationRequest> |
| 2058 | >([]); |
| 2059 | const database = yield* SqlitePersistence.layerMemory.pipe(Layer.build); |
| 2060 | const a = yield* makeLaptop(database, offers); |
| 2061 | yield* linkTo(a.links, b, "full-access"); |
| 2062 | yield* seedParent(a); |
| 2063 | |
| 2064 | const delegated = yield* delegateToBox(a); |
| 2065 | // What the delegated-tasks.stop effect runs once the parent's Stop commits. |
| 2066 | yield* a.threads.stopDelegatedTasks({ |
| 2067 | threadId: parentThreadId, |
| 2068 | commandId: CommandId.make("command:stop:laptop-parent"), |
| 2069 | reason: "User stopped", |
| 2070 | }); |
| 2071 | expect(yield* taskResult(a, delegated.taskId)).toMatchObject({ |
| 2072 | status: "interrupted", |
| 2073 | result: "Stopped with its parent: User stopped", |
| 2074 | }); |
| 2075 | // The follower sees it ended here and stops it there. |
| 2076 | yield* Deferred.await(thread.interruptedOnce); |
| 2077 | expect(yield* Ref.get(thread.interrupted)).toBe(1); |
| 2078 | }), |
| 2079 | ).pipe(Effect.provide(NodeHttpServer.layerTest)), |
| 2080 | ); |
| 2081 | |
| 2082 | /** A follower retry schedule the test steps: each failed attempt is offered, then waits for `retry`. */ |
| 2083 | const steppedRetries = Effect.gen(function* () { |
| 2084 | const failed = yield* Queue.unbounded<void>(); |
| 2085 | const retry = yield* Queue.unbounded<void>(); |
| 2086 | const schedule = Schedule.forever.pipe( |
| 2087 | Schedule.tap(() => Queue.offer(failed, undefined).pipe(Effect.andThen(Queue.take(retry)))), |
| 2088 | ); |
| 2089 | return { failed, retry, schedule }; |
| 2090 | }); |
| 2091 | |
| 2092 | it.effect("a follower that keeps hitting defects fails the task and wakes the parent", () => |
| 2093 | Effect.scoped( |
| 2094 | Effect.gen(function* () { |
| 2095 | const thread = yield* makeBoxThread; |
| 2096 | const b = yield* serveBox(thread); |
| 2097 | const offers = yield* SubscriptionRef.make< |
| 2098 | ReadonlyArray<ProviderContinuationRequests.ProviderContinuationRequest> |
| 2099 | >([]); |
| 2100 | const database = yield* SqlitePersistence.layerMemory.pipe(Layer.build); |
| 2101 | const retries = yield* steppedRetries; |
| 2102 | const attempts = yield* Ref.make(0); |
| 2103 | // The defect carries internal detail that the task's result must not. |
| 2104 | const beforeFollow = Ref.update(attempts, (count) => count + 1).pipe( |
| 2105 | Effect.andThen(Effect.die(new Error("A follow attempt hit a bug: defect-detail-4f1c9e"))), |
| 2106 | ); |
| 2107 | const a = yield* makeLaptop(database, offers, undefined, beforeFollow).pipe( |
| 2108 | Effect.provideService(RemoteDelegation.FollowRetrySchedule, retries.schedule), |
| 2109 | ); |
| 2110 | yield* linkTo(a.links, b, "full-access"); |
| 2111 | yield* seedParent(a); |
| 2112 | yield* Queue.take(retries.failed).pipe( |
| 2113 | Effect.andThen(Queue.offer(retries.retry, undefined)), |
| 2114 | Effect.forever, |
| 2115 | Effect.forkScoped, |
| 2116 | ); |
| 2117 | |
| 2118 | const delegated = yield* delegateToBox(a); |
| 2119 | const task = yield* taskResult(a, delegated.taskId); |
| 2120 | expect(yield* Ref.get(attempts)).toBe(3); |
| 2121 | expect(task?.status).toBe("failed"); |
| 2122 | expect(task?.result).toContain(delegated.remoteChild.threadId); |
| 2123 | expect(task?.result).not.toContain("defect-detail-4f1c9e"); |
| 2124 | const offered = yield* SubscriptionRef.changes(offers).pipe( |
| 2125 | Stream.filter((seen) => seen.some((offer) => offer.threadId === parentThreadId)), |
| 2126 | Stream.runHead, |
| 2127 | ); |
| 2128 | expect(Option.isSome(offered)).toBe(true); |
| 2129 | }), |
| 2130 | ).pipe(Effect.provide(NodeHttpServer.layerTest)), |
| 2131 | ); |
| 2132 | |
| 2133 | it.effect.each(["a still-running wait", "a failed attempt"] as const)( |
| 2134 | "defects separated by %s do not add up to giving up", |
| 2135 | (between) => |
| 2136 | Effect.scoped( |
| 2137 | Effect.gen(function* () { |
| 2138 | const thread = yield* makeBoxThread; |
| 2139 | const b = yield* serveBox(thread); |
| 2140 | const offers = yield* SubscriptionRef.make< |
| 2141 | ReadonlyArray<ProviderContinuationRequests.ProviderContinuationRequest> |
| 2142 | >([]); |
| 2143 | const database = yield* SqlitePersistence.layerMemory.pipe(Layer.build); |
| 2144 | const retries = yield* steppedRetries; |
| 2145 | const attempts = yield* Ref.make(0); |
| 2146 | const defect = Effect.die(new Error("A follow attempt hit a bug.")); |
| 2147 | const script = [ |
| 2148 | defect, |
| 2149 | between === "a still-running wait" |
| 2150 | ? Effect.void |
| 2151 | : Effect.fail( |
| 2152 | new Orchestrator.OrchestratorProjectionError({ threadId: parentThreadId }), |
| 2153 | ), |
| 2154 | defect, |
| 2155 | between === "a still-running wait" |
| 2156 | ? Effect.void |
| 2157 | : Effect.fail( |
| 2158 | new Orchestrator.OrchestratorProjectionError({ threadId: parentThreadId }), |
| 2159 | ), |
| 2160 | defect, |
| 2161 | ]; |
| 2162 | // Past the script, the run there has ended, so the next attempt completes the task. |
| 2163 | const beforeFollow = Ref.getAndUpdate(attempts, (count) => count + 1).pipe( |
| 2164 | Effect.flatMap( |
| 2165 | (count) => |
| 2166 | script[count] ?? |
| 2167 | Ref.set(thread.reply, "All green.").pipe( |
| 2168 | Effect.andThen(Ref.set(thread.status, "completed")), |
| 2169 | ), |
| 2170 | ), |
| 2171 | ); |
| 2172 | // Each wait there answers at once that its run is still running. |
| 2173 | yield* Deferred.succeed(thread.finished, undefined); |
| 2174 | const a = yield* makeLaptop(database, offers, undefined, beforeFollow).pipe( |
| 2175 | Effect.provideService(RemoteDelegation.FollowRetrySchedule, retries.schedule), |
| 2176 | ); |
| 2177 | yield* linkTo(a.links, b, "full-access"); |
| 2178 | yield* seedParent(a); |
| 2179 | yield* Queue.take(retries.failed).pipe( |
| 2180 | Effect.andThen(Queue.offer(retries.retry, undefined)), |
| 2181 | Effect.forever, |
| 2182 | Effect.forkScoped, |
| 2183 | ); |
| 2184 | |
| 2185 | const delegated = yield* delegateToBox(a); |
| 2186 | expect(yield* taskResult(a, delegated.taskId)).toMatchObject({ |
| 2187 | status: "completed", |
| 2188 | result: "All green.", |
| 2189 | }); |
| 2190 | }), |
| 2191 | ).pipe(Effect.provide(NodeHttpServer.layerTest)), |
| 2192 | ); |
| 2193 | |
| 2194 | /** Waits until the task here records that the `stopCount`th Stop after its result reached there. */ |
| 2195 | const stoppedThere = ( |
| 2196 | laptopEnv: Effect.Success<ReturnType<typeof makeLaptop>>, |
| 2197 | taskId: NodeId, |
| 2198 | // `null` for a cancel's stop, which carries no count. |
| 2199 | stopCount: number | null = 1, |
| 2200 | ) => |
| 2201 | laptopEnv.orchestrator |
| 2202 | .streamStoredEventsFrom({ |
| 2203 | threadId: parentThreadId, |
| 2204 | afterSequence: 0, |
| 2205 | eventType: "subagent.updated", |
| 2206 | }) |
| 2207 | .pipe( |
| 2208 | Stream.filter( |
| 2209 | (stored) => |
| 2210 | stored.event.type === "subagent.updated" && |
| 2211 | stored.event.payload.id === taskId && |
| 2212 | stored.event.payload.remoteChild?.stoppedThere === true && |
| 2213 | stored.event.payload.remoteChild.stopCount === (stopCount ?? undefined), |
| 2214 | ), |
| 2215 | Stream.runHead, |
| 2216 | Effect.map((stored) => |
| 2217 | Option.isSome(stored) && stored.value.event.type === "subagent.updated" |
| 2218 | ? stored.value.event.payload |
| 2219 | : undefined, |
| 2220 | ), |
| 2221 | ); |
| 2222 | |
| 2223 | it.effect("a follower that keeps hitting defects leaves a stop it owes owed", () => |
| 2224 | Effect.scoped( |
| 2225 | Effect.gen(function* () { |
| 2226 | const thread = yield* makeBoxThread; |
| 2227 | const b = yield* serveBox(thread); |
| 2228 | const offers = yield* SubscriptionRef.make< |
| 2229 | ReadonlyArray<ProviderContinuationRequests.ProviderContinuationRequest> |
| 2230 | >([]); |
| 2231 | const database = yield* SqlitePersistence.layerMemory.pipe(Layer.build); |
| 2232 | const retries = yield* steppedRetries; |
| 2233 | // Follow attempts hit a defect only once the task has its result. |
| 2234 | const broken = yield* Ref.make(false); |
| 2235 | const follower = yield* Deferred.make<Fiber.Fiber<unknown, unknown>>(); |
| 2236 | const beforeFollow = Ref.get(broken).pipe( |
| 2237 | Effect.flatMap((isBroken) => |
| 2238 | isBroken |
| 2239 | ? Effect.withFiber((fiber) => Deferred.succeed(follower, fiber)).pipe( |
| 2240 | Effect.andThen(Effect.die(new Error("A follow attempt hit a bug."))), |
| 2241 | ) |
| 2242 | : Effect.void, |
| 2243 | ), |
| 2244 | ); |
| 2245 | const a = yield* makeLaptop(database, offers, undefined, beforeFollow).pipe( |
| 2246 | Effect.provideService(RemoteDelegation.FollowRetrySchedule, retries.schedule), |
| 2247 | ); |
| 2248 | yield* a.remote.start(); |
| 2249 | yield* linkTo(a.links, b, "full-access"); |
| 2250 | yield* seedParent(a); |
| 2251 | const delegated = yield* delegateToBox(a); |
| 2252 | yield* Ref.set(thread.reply, "All green."); |
| 2253 | yield* Ref.set(thread.status, "completed"); |
| 2254 | yield* Deferred.succeed(thread.finished, undefined); |
| 2255 | yield* taskResult(a, delegated.taskId); |
| 2256 | yield* Ref.set(broken, true); |
| 2257 | yield* Queue.take(retries.failed).pipe( |
| 2258 | Effect.andThen(Queue.offer(retries.retry, undefined)), |
| 2259 | Effect.forever, |
| 2260 | Effect.forkScoped, |
| 2261 | ); |
| 2262 | |
| 2263 | // The Stop owes one stop there, which every attempt to send fails on. |
| 2264 | yield* a.threads.stopDelegatedTasks({ |
| 2265 | threadId: parentThreadId, |
| 2266 | commandId: CommandId.make("command:stop:laptop-parent-defective-follower"), |
| 2267 | }); |
| 2268 | const owedAgain = a.orchestrator |
| 2269 | .streamStoredEventsFrom({ |
| 2270 | threadId: parentThreadId, |
| 2271 | afterSequence: 0, |
| 2272 | eventType: "subagent.updated", |
| 2273 | }) |
| 2274 | .pipe( |
| 2275 | Stream.filter( |
| 2276 | (stored) => |
| 2277 | stored.event.type === "subagent.updated" && |
| 2278 | stored.event.payload.id === delegated.taskId && |
| 2279 | (stored.event.payload.remoteChild?.stopCount ?? 0) > 1, |
| 2280 | ), |
| 2281 | Stream.runHead, |
| 2282 | Effect.flatMap((owed) => |
| 2283 | Option.isSome(owed) ? Effect.succeed("owed another stop" as const) : Effect.never, |
| 2284 | ), |
| 2285 | ); |
| 2286 | const ended = yield* Deferred.await(follower).pipe( |
| 2287 | Effect.flatMap(Fiber.await), |
| 2288 | Effect.as("follower ended" as const), |
| 2289 | Effect.raceFirst(owedAgain), |
| 2290 | ); |
| 2291 | expect(ended).toBe("follower ended"); |
| 2292 | const { subagents } = yield* a.threads.getThreadRecords(parentThreadId, ["subagents"], { |
| 2293 | subagentIds: [delegated.taskId], |
| 2294 | }); |
| 2295 | expect(subagents[0]).toMatchObject({ |
| 2296 | status: "completed", |
| 2297 | result: "All green.", |
| 2298 | remoteChild: { stoppedThere: false, stopCount: 1 }, |
| 2299 | }); |
| 2300 | }), |
| 2301 | ).pipe(Effect.provide(NodeHttpServer.layerTest)), |
| 2302 | ); |
| 2303 | |
| 2304 | it.effect("stopping the parent stops follow-up work on a task there that has its result", () => |
| 2305 | Effect.scoped( |
| 2306 | Effect.gen(function* () { |
| 2307 | const thread = yield* makeBoxThread; |
| 2308 | const b = yield* serveBox(thread); |
| 2309 | const offers = yield* SubscriptionRef.make< |
| 2310 | ReadonlyArray<ProviderContinuationRequests.ProviderContinuationRequest> |
| 2311 | >([]); |
| 2312 | const database = yield* SqlitePersistence.layerMemory.pipe(Layer.build); |
| 2313 | const a = yield* makeLaptop(database, offers); |
| 2314 | yield* a.remote.start(); |
| 2315 | yield* linkTo(a.links, b, "full-access"); |
| 2316 | yield* seedParent(a); |
| 2317 | const delegated = yield* delegateToBox(a); |
| 2318 | yield* Ref.set(thread.reply, "All green."); |
| 2319 | yield* Ref.set(thread.status, "completed"); |
| 2320 | yield* Deferred.succeed(thread.finished, undefined); |
| 2321 | expect(yield* taskResult(a, delegated.taskId)).toMatchObject({ |
| 2322 | status: "completed", |
| 2323 | result: "All green.", |
| 2324 | }); |
| 2325 | |
| 2326 | // The parent sends the thread there a follow-up, which starts a run there as a new task. |
| 2327 | const sent = yield* a.call("t3_thread_send", { |
| 2328 | environmentId: box.environmentId, |
| 2329 | threadId: delegated.remoteChild.threadId, |
| 2330 | message: "Now run the slow suite too.", |
| 2331 | }); |
| 2332 | expect(sent.isError, JSON.stringify(sent.content)).toBe(false); |
| 2333 | const followUpId = (sent.structuredContent as { taskId: NodeId }).taskId; |
| 2334 | expect(yield* Ref.get(thread.sent)).toEqual(["Now run the slow suite too."]); |
| 2335 | expect(yield* Ref.get(thread.interrupted)).toBe(0); |
| 2336 | |
| 2337 | yield* a.threads.stopDelegatedTasks({ |
| 2338 | threadId: parentThreadId, |
| 2339 | commandId: CommandId.make("command:stop:laptop-parent-after-result"), |
| 2340 | }); |
| 2341 | // The thread there is stopped as Stop does, for each task, and the first keeps its result. |
| 2342 | const task = yield* stoppedThere(a, delegated.taskId); |
| 2343 | expect(yield* stoppedThere(a, followUpId, null)).toMatchObject({ status: "interrupted" }); |
| 2344 | expect((yield* Ref.get(thread.stops)).toSorted()).toEqual([ |
| 2345 | "delegated-tasks", |
| 2346 | "delegated-tasks", |
| 2347 | "thread.stop", |
| 2348 | "thread.stop", |
| 2349 | ]); |
| 2350 | expect(task).toMatchObject({ status: "completed", result: "All green." }); |
| 2351 | expect(yield* a.projections.getOpenRemoteDelegatedTasks).toEqual([]); |
| 2352 | |
| 2353 | // A second Stop owes a second stop there, not a replay of the first. |
| 2354 | yield* a.threads.stopDelegatedTasks({ |
| 2355 | threadId: parentThreadId, |
| 2356 | commandId: CommandId.make("command:stop:laptop-parent-again"), |
| 2357 | }); |
| 2358 | expect(yield* stoppedThere(a, delegated.taskId, 2)).toMatchObject({ status: "completed" }); |
| 2359 | expect(yield* stoppedThere(a, followUpId, 1)).toMatchObject({ status: "interrupted" }); |
| 2360 | expect(yield* Ref.get(thread.interrupted)).toBe(4); |
| 2361 | }), |
| 2362 | ).pipe(Effect.provide(NodeHttpServer.layerTest)), |
| 2363 | ); |
| 2364 | |
| 2365 | it.effect.each([ |
| 2366 | { order: "sent", queuedFirst: false }, |
| 2367 | // Concurrent sends may be recorded here in the opposite order they run there. |
| 2368 | { order: "recorded out of order", queuedFirst: true }, |
| 2369 | ])( |
| 2370 | "a follow-up there ends with its own reply, not one queued behind it ($order)", |
| 2371 | ({ queuedFirst }) => |
| 2372 | Effect.scoped( |
| 2373 | Effect.gen(function* () { |
| 2374 | const thread = yield* makeBoxThread; |
| 2375 | const b = yield* serveBox(thread); |
| 2376 | const offers = yield* SubscriptionRef.make< |
| 2377 | ReadonlyArray<ProviderContinuationRequests.ProviderContinuationRequest> |
| 2378 | >([]); |
| 2379 | const database = yield* SqlitePersistence.layerMemory.pipe(Layer.build); |
| 2380 | const a = yield* makeLaptop(database, offers); |
| 2381 | yield* a.remote.start(); |
| 2382 | yield* linkTo(a.links, b, "full-access"); |
| 2383 | yield* seedParent(a); |
| 2384 | const delegated = yield* delegateToBox(a); |
| 2385 | yield* Ref.set(thread.reply, "All green."); |
| 2386 | yield* Ref.set(thread.status, "completed"); |
| 2387 | yield* Deferred.succeed(thread.finished, undefined); |
| 2388 | yield* taskResult(a, delegated.taskId); |
| 2389 | |
| 2390 | const sendFollowUp = a.call("t3_thread_send", { |
| 2391 | environmentId: box.environmentId, |
| 2392 | threadId: delegated.remoteChild.threadId, |
| 2393 | message: "Now run the slow suite too.", |
| 2394 | }); |
| 2395 | const sendQueued = a.call("t3_thread_send", { |
| 2396 | environmentId: box.environmentId, |
| 2397 | threadId: delegated.remoteChild.threadId, |
| 2398 | message: "Then the flaky suite.", |
| 2399 | mode: "queue", |
| 2400 | }); |
| 2401 | const [first, second] = queuedFirst |
| 2402 | ? [sendQueued, sendFollowUp] |
| 2403 | : [sendFollowUp, sendQueued]; |
| 2404 | const firstSent = yield* first; |
| 2405 | yield* TestClock.adjust("1 second"); |
| 2406 | const secondSent = yield* second; |
| 2407 | const [followUp, queued] = queuedFirst ? [secondSent, firstSent] : [firstSent, secondSent]; |
| 2408 | const followUpId = (followUp.structuredContent as { taskId: NodeId }).taskId; |
| 2409 | expect((queued.structuredContent as { taskId?: NodeId }).taskId).toBeDefined(); |
| 2410 | |
| 2411 | // The thread there settles on the queued run, after the follow-up's own run ended. |
| 2412 | yield* Ref.set(thread.laterReply, "The slow suite passes too."); |
| 2413 | yield* Ref.set(thread.queuedReply, "The flaky suite passes."); |
| 2414 | yield* Deferred.succeed(thread.laterFinished, undefined); |
| 2415 | expect(yield* taskResult(a, followUpId)).toMatchObject({ |
| 2416 | status: "completed", |
| 2417 | result: "The slow suite passes too.", |
| 2418 | }); |
| 2419 | }), |
| 2420 | ).pipe(Effect.provide(NodeHttpServer.layerTest)), |
| 2421 | ); |
| 2422 | |
| 2423 | it.effect.each([ |
| 2424 | { |
| 2425 | restarted: "follow-up", |
| 2426 | of: boxLaterRunId, |
| 2427 | laterReply: "Running the slow suite…", |
| 2428 | restartReply: "The slow suite passes too.", |
| 2429 | queuedReply: "The flaky suite passes.", |
| 2430 | }, |
| 2431 | { |
| 2432 | restarted: "queued", |
| 2433 | of: boxQueuedRunId, |
| 2434 | laterReply: "The slow suite passes too.", |
| 2435 | restartReply: "The flaky suite passes.", |
| 2436 | queuedReply: "Running the flaky suite…", |
| 2437 | }, |
| 2438 | ])( |
| 2439 | "a follow-up there ends with its own reply when the $restarted run is restarted", |
| 2440 | ({ of, laterReply, restartReply, queuedReply }) => |
| 2441 | Effect.scoped( |
| 2442 | Effect.gen(function* () { |
| 2443 | const thread = yield* makeBoxThread; |
| 2444 | const b = yield* serveBox(thread); |
| 2445 | const offers = yield* SubscriptionRef.make< |
| 2446 | ReadonlyArray<ProviderContinuationRequests.ProviderContinuationRequest> |
| 2447 | >([]); |
| 2448 | const database = yield* SqlitePersistence.layerMemory.pipe(Layer.build); |
| 2449 | const a = yield* makeLaptop(database, offers); |
| 2450 | yield* a.remote.start(); |
| 2451 | yield* linkTo(a.links, b, "full-access"); |
| 2452 | yield* seedParent(a); |
| 2453 | const delegated = yield* delegateToBox(a); |
| 2454 | yield* Ref.set(thread.reply, "All green."); |
| 2455 | yield* Ref.set(thread.status, "completed"); |
| 2456 | yield* Deferred.succeed(thread.finished, undefined); |
| 2457 | yield* taskResult(a, delegated.taskId); |
| 2458 | |
| 2459 | const followUp = yield* a.call("t3_thread_send", { |
| 2460 | environmentId: box.environmentId, |
| 2461 | threadId: delegated.remoteChild.threadId, |
| 2462 | message: "Now run the slow suite too.", |
| 2463 | }); |
| 2464 | yield* a.call("t3_thread_send", { |
| 2465 | environmentId: box.environmentId, |
| 2466 | threadId: delegated.remoteChild.threadId, |
| 2467 | message: "Then the flaky suite.", |
| 2468 | mode: "queue", |
| 2469 | }); |
| 2470 | const followUpId = (followUp.structuredContent as { taskId: NodeId }).taskId; |
| 2471 | |
| 2472 | // The thread there settles on the queued run, after one of the two was restarted. |
| 2473 | yield* Ref.set(thread.laterReply, laterReply); |
| 2474 | yield* Ref.set(thread.queuedReply, queuedReply); |
| 2475 | yield* Ref.set(thread.restart, { of, reply: restartReply }); |
| 2476 | yield* Deferred.succeed(thread.laterFinished, undefined); |
| 2477 | expect(yield* taskResult(a, followUpId)).toMatchObject({ |
| 2478 | status: "completed", |
| 2479 | result: "The slow suite passes too.", |
| 2480 | }); |
| 2481 | }), |
| 2482 | ).pipe(Effect.provide(NodeHttpServer.layerTest)), |
| 2483 | ); |
| 2484 | |
| 2485 | it.effect("a follow-up sent after the parent's own run ended is still a task", () => |
| 2486 | Effect.scoped( |
| 2487 | Effect.gen(function* () { |
| 2488 | const thread = yield* makeBoxThread; |
| 2489 | const b = yield* serveBox(thread); |
| 2490 | const offers = yield* SubscriptionRef.make< |
| 2491 | ReadonlyArray<ProviderContinuationRequests.ProviderContinuationRequest> |
| 2492 | >([]); |
| 2493 | const database = yield* SqlitePersistence.layerMemory.pipe(Layer.build); |
| 2494 | const a = yield* makeLaptop(database, offers); |
| 2495 | yield* a.remote.start(); |
| 2496 | yield* linkTo(a.links, b, "full-access"); |
| 2497 | yield* seedParent(a); |
| 2498 | const delegated = yield* delegateToBox(a); |
| 2499 | yield* Ref.set(thread.reply, "All green."); |
| 2500 | yield* Ref.set(thread.status, "completed"); |
| 2501 | yield* Deferred.succeed(thread.finished, undefined); |
| 2502 | yield* taskResult(a, delegated.taskId); |
| 2503 | |
| 2504 | // The parent's turn ends, while a background agent of its own still holds its credential. |
| 2505 | const now = yield* DateTime.now; |
| 2506 | const projection = yield* a.orchestrator.getThreadProjection(parentThreadId); |
| 2507 | const run = projection.runs.find((candidate) => candidate.id === parentRunId)!; |
| 2508 | yield* a.sink.write({ |
| 2509 | commandId: CommandId.make("command:seed:laptop-parent-ended"), |
| 2510 | events: [ |
| 2511 | { |
| 2512 | id: EventId.make("event:seed-background-agent:laptop-parent"), |
| 2513 | type: "provider-thread.updated", |
| 2514 | threadId: parentThreadId, |
| 2515 | driver, |
| 2516 | providerInstanceId: instanceId, |
| 2517 | occurredAt: now, |
| 2518 | payload: { |
| 2519 | ...projection.providerThreads[0]!, |
| 2520 | pendingBackgroundTasks: [{ taskId: "native-agent", kind: "subagent" }], |
| 2521 | updatedAt: now, |
| 2522 | }, |
| 2523 | }, |
| 2524 | { |
| 2525 | id: EventId.make("event:seed-run-ended:laptop-parent"), |
| 2526 | type: "run.updated", |
| 2527 | threadId: parentThreadId, |
| 2528 | runId: parentRunId, |
| 2529 | nodeId: parentRootNode, |
| 2530 | providerInstanceId: instanceId, |
| 2531 | occurredAt: now, |
| 2532 | payload: { ...run, status: "completed", completedAt: now }, |
| 2533 | }, |
| 2534 | ], |
| 2535 | }); |
| 2536 | |
| 2537 | const sent = yield* a.call("t3_thread_send", { |
| 2538 | environmentId: box.environmentId, |
| 2539 | threadId: delegated.remoteChild.threadId, |
| 2540 | message: "Now run the slow suite too.", |
| 2541 | }); |
| 2542 | expect(sent.isError, JSON.stringify(sent.content)).toBe(false); |
| 2543 | const followUp = sent.structuredContent as { taskId?: NodeId }; |
| 2544 | expect(followUp.taskId, JSON.stringify(followUp)).toBeDefined(); |
| 2545 | const tasks = (yield* a.orchestrator.getThreadProjection(parentThreadId)).subagents; |
| 2546 | expect(tasks.find((task) => task.id === followUp.taskId)).toMatchObject({ |
| 2547 | runId: parentRunId, |
| 2548 | status: "running", |
| 2549 | }); |
| 2550 | |
| 2551 | // Its run there ends with its own reply, and the parent wakes for it. |
| 2552 | const wakes = ( |
| 2553 | seen: ReadonlyArray<ProviderContinuationRequests.ProviderContinuationRequest>, |
| 2554 | ) => seen.filter((offer) => offer.threadId === parentThreadId).length; |
| 2555 | const wokenBefore = wakes(yield* SubscriptionRef.get(offers)); |
| 2556 | yield* Ref.set(thread.laterReply, "The slow suite passes too."); |
| 2557 | yield* Deferred.succeed(thread.laterFinished, undefined); |
| 2558 | expect(yield* taskResult(a, followUp.taskId!)).toMatchObject({ |
| 2559 | status: "completed", |
| 2560 | result: "The slow suite passes too.", |
| 2561 | }); |
| 2562 | const offered = yield* SubscriptionRef.changes(offers).pipe( |
| 2563 | Stream.filter((seen) => wakes(seen) > wokenBefore), |
| 2564 | Stream.runHead, |
| 2565 | ); |
| 2566 | expect(Option.isSome(offered)).toBe(true); |
| 2567 | }), |
| 2568 | ).pipe(Effect.provide(NodeHttpServer.layerTest)), |
| 2569 | ); |
| 2570 | |
| 2571 | it.effect("a delayed Stop acknowledgement leaves a newer Stop owed", () => |
| 2572 | Effect.scoped( |
| 2573 | Effect.gen(function* () { |
| 2574 | const thread = yield* makeBoxThread; |
| 2575 | const b = yield* serveBox(thread); |
| 2576 | const offers = yield* SubscriptionRef.make< |
| 2577 | ReadonlyArray<ProviderContinuationRequests.ProviderContinuationRequest> |
| 2578 | >([]); |
| 2579 | const database = yield* SqlitePersistence.layerMemory.pipe(Layer.build); |
| 2580 | const a = yield* makeLaptop(database, offers); |
| 2581 | yield* a.remote.start(); |
| 2582 | yield* linkTo(a.links, b, "full-access"); |
| 2583 | yield* seedParent(a); |
| 2584 | const delegated = yield* delegateToBox(a); |
| 2585 | yield* Ref.set(thread.reply, "All green."); |
| 2586 | yield* Ref.set(thread.status, "completed"); |
| 2587 | yield* Deferred.succeed(thread.finished, undefined); |
| 2588 | yield* taskResult(a, delegated.taskId); |
| 2589 | |
| 2590 | yield* Ref.set(thread.holdFirstStop, true); |
| 2591 | yield* a.threads.stopDelegatedTasks({ |
| 2592 | threadId: parentThreadId, |
| 2593 | commandId: CommandId.make("command:stop:delayed-first"), |
| 2594 | }); |
| 2595 | yield* Deferred.await(thread.stopStarted); |
| 2596 | // Work starts after the first Stop landed there, but before its acknowledgement returns. |
| 2597 | const sent = yield* a.call("t3_thread_send", { |
| 2598 | environmentId: box.environmentId, |
| 2599 | threadId: delegated.remoteChild.threadId, |
| 2600 | message: "Run another check.", |
| 2601 | }); |
| 2602 | const followUpId = (sent.structuredContent as { taskId: NodeId }).taskId; |
| 2603 | yield* a.threads.stopDelegatedTasks({ |
| 2604 | threadId: parentThreadId, |
| 2605 | commandId: CommandId.make("command:stop:delayed-second"), |
| 2606 | }); |
| 2607 | const pending = yield* a.threads.getThreadRecords(parentThreadId, ["subagents"]); |
| 2608 | expect( |
| 2609 | pending.subagents.find((task) => task.id === delegated.taskId)?.remoteChild, |
| 2610 | ).toMatchObject({ stoppedThere: false, stopCount: 2 }); |
| 2611 | yield* Deferred.succeed(thread.releaseStop, undefined); |
| 2612 | expect(yield* stoppedThere(a, delegated.taskId, 2)).toMatchObject({ |
| 2613 | status: "completed", |
| 2614 | result: "All green.", |
| 2615 | }); |
| 2616 | // The follow-up the work became is stopped there too, as its own task. |
| 2617 | expect(yield* stoppedThere(a, followUpId, null)).toMatchObject({ status: "interrupted" }); |
| 2618 | expect(yield* Ref.get(thread.interrupted)).toBe(3); |
| 2619 | expect(yield* a.projections.getOpenRemoteDelegatedTasks).toEqual([]); |
| 2620 | }), |
| 2621 | ).pipe(Effect.provide(NodeHttpServer.layerTest)), |
| 2622 | ); |
| 2623 | |
| 2624 | it.effect("a Stop on a task that has its result reaches there after a restart", () => |
| 2625 | Effect.scoped( |
| 2626 | Effect.gen(function* () { |
| 2627 | const thread = yield* makeBoxThread; |
| 2628 | const b = yield* serveBox(thread); |
| 2629 | const offers = yield* SubscriptionRef.make< |
| 2630 | ReadonlyArray<ProviderContinuationRequests.ProviderContinuationRequest> |
| 2631 | >([]); |
| 2632 | const database = yield* SqlitePersistence.layerMemory.pipe(Layer.build); |
| 2633 | const linking = yield* layerLinkingEnvironment(laptop).pipe(Layer.build); |
| 2634 | // The laptop records the Stop, then goes away before its follower sends it. |
| 2635 | const delegated = yield* Effect.scoped( |
| 2636 | Effect.gen(function* () { |
| 2637 | const before = yield* makeLaptop(database, offers, linking); |
| 2638 | yield* linkTo(before.links, b, "full-access"); |
| 2639 | yield* seedParent(before); |
| 2640 | const task = yield* delegateToBox(before); |
| 2641 | yield* Ref.set(thread.status, "completed"); |
| 2642 | yield* Deferred.succeed(thread.finished, undefined); |
| 2643 | yield* taskResult(before, task.taskId); |
| 2644 | yield* before.threads.stopDelegatedTasks({ |
| 2645 | threadId: parentThreadId, |
| 2646 | commandId: CommandId.make("command:stop:laptop-parent-before-restart"), |
| 2647 | }); |
| 2648 | return task; |
| 2649 | }), |
| 2650 | ); |
| 2651 | expect(yield* Ref.get(thread.interrupted)).toBe(0); |
| 2652 | |
| 2653 | const after = yield* makeLaptop(database, offers, linking); |
| 2654 | expect(yield* after.projections.getOpenRemoteDelegatedTasks).toEqual([ |
| 2655 | { parentThreadId, taskId: delegated.taskId }, |
| 2656 | ]); |
| 2657 | yield* after.remote.start(); |
| 2658 | expect(yield* stoppedThere(after, delegated.taskId)).toMatchObject({ status: "completed" }); |
| 2659 | expect(yield* Ref.get(thread.stops)).toEqual(["thread.stop", "delegated-tasks"]); |
| 2660 | }), |
| 2661 | ).pipe(Effect.provide(NodeHttpServer.layerTest)), |
| 2662 | ); |
| 2663 | |
| 2664 | it.effect("a parent stopped while the box is unreachable still stops its task there", () => |
| 2665 | Effect.scoped( |
| 2666 | Effect.gen(function* () { |
| 2667 | const thread = yield* makeBoxThread; |
| 2668 | const b = yield* serveBox(thread); |
| 2669 | const offers = yield* SubscriptionRef.make< |
| 2670 | ReadonlyArray<ProviderContinuationRequests.ProviderContinuationRequest> |
| 2671 | >([]); |
| 2672 | const database = yield* SqlitePersistence.layerMemory.pipe(Layer.build); |
| 2673 | const retries = yield* steppedRetries; |
| 2674 | const a = yield* makeLaptop(database, offers).pipe( |
| 2675 | Effect.provideService(RemoteDelegation.FollowRetrySchedule, retries.schedule), |
| 2676 | ); |
| 2677 | yield* linkTo(a.links, b, "full-access"); |
| 2678 | yield* seedParent(a); |
| 2679 | // The box sleeps as soon as the task is there, so its follower's first attempt fails. |
| 2680 | yield* Ref.set( |
| 2681 | thread.afterLaunch, |
| 2682 | Ref.set(b.descriptor, descriptorOf("environment-elsewhere", "Elsewhere")), |
| 2683 | ); |
| 2684 | const delegated = yield* delegateToBox(a); |
| 2685 | yield* Queue.take(retries.failed); |
| 2686 | // The parent is stopped meanwhile. |
| 2687 | yield* a.threads.stopDelegatedTasks({ |
| 2688 | threadId: parentThreadId, |
| 2689 | commandId: CommandId.make("command:stop:laptop-parent-offline"), |
| 2690 | }); |
| 2691 | expect(yield* taskResult(a, delegated.taskId)).toMatchObject({ status: "interrupted" }); |
| 2692 | |
| 2693 | // Once the box is back, the follower's next attempt stops the task there. |
| 2694 | yield* Ref.set(b.descriptor, box); |
| 2695 | yield* Queue.offer(retries.retry, undefined); |
| 2696 | yield* Deferred.await(thread.interruptedOnce); |
| 2697 | expect(yield* Ref.get(thread.interrupted)).toBe(1); |
| 2698 | }), |
| 2699 | ).pipe(Effect.provide(NodeHttpServer.layerTest)), |
| 2700 | ); |
| 2701 | |
| 2702 | it.effect("a task cancelled while the box is unreachable is stopped there after a restart", () => |
| 2703 | Effect.scoped( |
| 2704 | Effect.gen(function* () { |
| 2705 | const thread = yield* makeBoxThread; |
| 2706 | const b = yield* serveBox(thread); |
| 2707 | const offers = yield* SubscriptionRef.make< |
| 2708 | ReadonlyArray<ProviderContinuationRequests.ProviderContinuationRequest> |
| 2709 | >([]); |
| 2710 | const database = yield* SqlitePersistence.layerMemory.pipe(Layer.build); |
| 2711 | // Its links outlive the restart, so the box knows it as the same link session. |
| 2712 | const linking = yield* layerLinkingEnvironment(laptop).pipe(Layer.build); |
| 2713 | // The laptop cancels while the box sleeps, then restarts before it wakes. |
| 2714 | const delegated = yield* Effect.scoped( |
| 2715 | Effect.gen(function* () { |
| 2716 | const before = yield* makeLaptop(database, offers, linking); |
| 2717 | yield* linkTo(before.links, b, "full-access"); |
| 2718 | yield* seedParent(before); |
| 2719 | const task = yield* delegateToBox(before); |
| 2720 | yield* Ref.set(b.descriptor, descriptorOf("environment-elsewhere", "Elsewhere")); |
| 2721 | yield* before.call("task_cancel", { taskId: task.taskId }); |
| 2722 | expect(yield* taskResult(before, task.taskId)).toMatchObject({ status: "cancelled" }); |
| 2723 | return task; |
| 2724 | }), |
| 2725 | ); |
| 2726 | expect(yield* Ref.get(thread.interrupted)).toBe(0); |
| 2727 | |
| 2728 | yield* Ref.set(b.descriptor, box); |
| 2729 | const after = yield* makeLaptop(database, offers, linking); |
| 2730 | yield* after.remote.start(); |
| 2731 | yield* Deferred.await(thread.interruptedOnce); |
| 2732 | // Once the stop is recorded, nothing is owed there any more. |
| 2733 | yield* stoppedThere(after, delegated.taskId, null); |
| 2734 | expect(yield* after.projections.getOpenRemoteDelegatedTasks).toEqual([]); |
| 2735 | expect(yield* Ref.get(thread.stops)).toEqual(["thread.stop", "delegated-tasks"]); |
| 2736 | // The result here is still the cancel's. |
| 2737 | expect(yield* taskResult(after, delegated.taskId)).toMatchObject({ status: "cancelled" }); |
| 2738 | }), |
| 2739 | ).pipe(Effect.provide(NodeHttpServer.layerTest)), |
| 2740 | ); |
| 2741 | |
| 2742 | it.effect("a task whose thread there has no run ends with its last reply", () => |
| 2743 | Effect.scoped( |
| 2744 | Effect.gen(function* () { |
| 2745 | const thread = yield* makeBoxThread; |
| 2746 | const b = yield* serveBox(thread); |
| 2747 | const offers = yield* SubscriptionRef.make< |
| 2748 | ReadonlyArray<ProviderContinuationRequests.ProviderContinuationRequest> |
| 2749 | >([]); |
| 2750 | const database = yield* SqlitePersistence.layerMemory.pipe(Layer.build); |
| 2751 | const a = yield* makeLaptop(database, offers); |
| 2752 | yield* linkTo(a.links, b, "full-access"); |
| 2753 | yield* seedParent(a); |
| 2754 | // As a task recorded before its run was kept, whose thread there is idle. |
| 2755 | yield* Ref.set(thread.runless, true); |
| 2756 | yield* Ref.set(thread.reply, "Done before it was followed."); |
| 2757 | |
| 2758 | const delegated = yield* delegateToBox(a); |
| 2759 | expect(yield* taskResult(a, delegated.taskId)).toMatchObject({ |
| 2760 | status: "completed", |
| 2761 | result: "Done before it was followed.", |
| 2762 | }); |
| 2763 | // One look, not a loop. |
| 2764 | expect(yield* Ref.get(thread.waitedFor)).toEqual([undefined]); |
| 2765 | }), |
| 2766 | ).pipe(Effect.provide(NodeHttpServer.layerTest)), |
| 2767 | ); |
| 2768 | |
| 2769 | it.effect("a local task runs in this thread's project", () => |
| 2770 | Effect.scoped( |
| 2771 | Effect.gen(function* () { |
| 2772 | const offers = yield* SubscriptionRef.make< |
| 2773 | ReadonlyArray<ProviderContinuationRequests.ProviderContinuationRequest> |
| 2774 | >([]); |
| 2775 | const database = yield* SqlitePersistence.layerMemory.pipe(Layer.build); |
| 2776 | const a = yield* makeLaptop(database, offers); |
| 2777 | yield* seedParent(a); |
| 2778 | |
| 2779 | const result = yield* a.call("delegate_task", { |
| 2780 | task: "Run it here.", |
| 2781 | target: { projectId: "project:elsewhere" }, |
| 2782 | }); |
| 2783 | expect(result.isError).toBe(true); |
| 2784 | expect(JSON.stringify(result.content)).toContain("invalid_request"); |
| 2785 | }), |
| 2786 | ).pipe(Effect.provide(NodeHttpServer.layerTest)), |
| 2787 | ); |