The best way to control Claude Code, Codex and more from any computer or phone
Mirrored from https://github.com/Camo67/t3code.git
Synced Nov 21, 03:37 UTC
| 1 | import { assert, it } from "@effect/vitest"; |
| 2 | import * as NodeCrypto from "@effect/platform-node/NodeCrypto"; |
| 3 | import * as NodeServices from "@effect/platform-node/NodeServices"; |
| 4 | import { |
| 5 | CheckpointId, |
| 6 | CheckpointRef, |
| 7 | CheckpointScopeId, |
| 8 | MessageId, |
| 9 | GitCommandError, |
| 10 | type ModelSelection, |
| 11 | NodeId, |
| 12 | type OrchestrationV2AppThread, |
| 13 | type OrchestrationV2DomainEvent, |
| 14 | type OrchestrationV2ExecutionNode, |
| 15 | type OrchestrationV2RuntimeRequest, |
| 16 | type OrchestrationV2ProviderThread, |
| 17 | type OrchestrationV2Run, |
| 18 | type OrchestrationV2TurnItem, |
| 19 | ProviderDriverKind, |
| 20 | ProviderInstanceId, |
| 21 | ProviderThreadId, |
| 22 | ProviderTurnId, |
| 23 | RunAttemptId, |
| 24 | RunId, |
| 25 | RuntimeRequestId, |
| 26 | TurnItemId, |
| 27 | } from "@t3tools/contracts"; |
| 28 | import * as Cause from "effect/Cause"; |
| 29 | import * as DateTime from "effect/DateTime"; |
| 30 | import * as Deferred from "effect/Deferred"; |
| 31 | import * as Fiber from "effect/Fiber"; |
| 32 | import * as Effect from "effect/Effect"; |
| 33 | import * as Exit from "effect/Exit"; |
| 34 | import * as FileSystem from "effect/FileSystem"; |
| 35 | import * as Layer from "effect/Layer"; |
| 36 | import * as Schema from "effect/Schema"; |
| 37 | import * as SqlClient from "effect/sql/SqlClient"; |
| 38 | import * as SqlError from "effect/sql/SqlError"; |
| 39 | import * as Stream from "effect/Stream"; |
| 40 | import * as Tracer from "effect/Tracer"; |
| 41 | |
| 42 | import { toolOutputImages } from "@t3tools/shared/toolOutput"; |
| 43 | import * as SqlitePersistence from "../persistence/Sqlite.ts"; |
| 44 | import * as ServerConfig from "../config.ts"; |
| 45 | import * as GitVcsDriver from "../vcs/GitVcsDriver.ts"; |
| 46 | import * as EventSink from "./EventSink.ts"; |
| 47 | import * as EventStore from "./EventStore.ts"; |
| 48 | import * as IdAllocator from "@t3tools/provider-core/server/IdAllocator"; |
| 49 | import * as ProjectionStore from "./ProjectionStore.ts"; |
| 50 | import * as ProviderEventIngestor from "./ProviderEventIngestor.ts"; |
| 51 | import * as ProviderUsageAnalytics from "../telemetry/ProviderUsageAnalytics.ts"; |
| 52 | import * as ThreadCommandExecutor from "./ThreadCommandExecutor.ts"; |
| 53 | import * as ProviderInstanceRegistry from "../provider/ProviderInstanceRegistry.ts"; |
| 54 | import { UsageLimitResetFallback } from "../usage/UsageLimitSources.ts"; |
| 55 | import * as ProviderAdapter from "@t3tools/provider-core/server/ProviderAdapter"; |
| 56 | import type { ProviderInstance } from "@t3tools/provider-core/server/driver"; |
| 57 | import type { ServerProvider } from "@t3tools/contracts"; |
| 58 | import { makeProviderFailure } from "@t3tools/provider-core/server/failure"; |
| 59 | import { |
| 60 | makeProviderEventRoutingState, |
| 61 | type ProviderEventRouteIdentity, |
| 62 | routeProviderEvent, |
| 63 | selectInheritedBackgroundTurnItems, |
| 64 | } from "./RunExecutionService.ts"; |
| 65 | |
| 66 | const layerTestDatabase = SqlitePersistence.layerMemory; |
| 67 | const layerTestStores = Layer.merge(EventStore.layer, ProjectionStore.layer).pipe( |
| 68 | Layer.provide(layerTestDatabase), |
| 69 | ); |
| 70 | |
| 71 | const layerTestEventSink = EventSink.layer.pipe( |
| 72 | Layer.provide(Layer.mergeAll(layerTestStores, layerTestDatabase)), |
| 73 | ); |
| 74 | |
| 75 | const layerGit = GitVcsDriver.layer.pipe( |
| 76 | Layer.provide(ServerConfig.layerTest(process.cwd(), { prefix: "ingestor-git-" })), |
| 77 | Layer.provideMerge(Layer.mergeAll(NodeServices.layer, NodeCrypto.layer)), |
| 78 | ); |
| 79 | |
| 80 | const layerEmptyInstances = Layer.mock(ProviderInstanceRegistry.ProviderInstanceRegistry)({ |
| 81 | getInstance: () => Effect.undefined, |
| 82 | }); |
| 83 | const makeLayerTest = <E>( |
| 84 | ingestorEventSink: Layer.Layer<EventSink.EventSinkV2, E>, |
| 85 | instances = layerEmptyInstances, |
| 86 | ) => |
| 87 | Layer.mergeAll( |
| 88 | instances, |
| 89 | layerTestDatabase, |
| 90 | layerTestStores, |
| 91 | layerTestEventSink, |
| 92 | IdAllocator.layer, |
| 93 | ThreadCommandExecutor.layer, |
| 94 | layerGit, |
| 95 | ProviderEventIngestor.layer.pipe( |
| 96 | Layer.provide( |
| 97 | Layer.mergeAll( |
| 98 | instances, |
| 99 | layerTestStores, |
| 100 | ingestorEventSink, |
| 101 | IdAllocator.layer, |
| 102 | ThreadCommandExecutor.layer, |
| 103 | layerGit, |
| 104 | ), |
| 105 | ), |
| 106 | ), |
| 107 | ); |
| 108 | const layerTest = makeLayerTest(layerTestEventSink); |
| 109 | |
| 110 | // EventSink reports a lock that outlasted busy_timeout at BEGIN IMMEDIATE this way. |
| 111 | const lockTimeout = new EventSink.EventSinkWriteError({ |
| 112 | eventCount: 1, |
| 113 | cause: new SqlError.SqlError({ reason: new SqlError.LockTimeoutError({ cause: null }) }), |
| 114 | }); |
| 115 | const constraintFailure = new EventSink.EventSinkWriteError({ |
| 116 | eventCount: 1, |
| 117 | cause: new SqlError.SqlError({ reason: new SqlError.ConstraintError({ cause: null }) }), |
| 118 | }); |
| 119 | |
| 120 | /** The ingestor's next sink writes of `events` fail with `errors`, in order; the rest go through. */ |
| 121 | const layerFailingWrites = ( |
| 122 | errors: ReadonlyArray<EventSink.EventSinkWriteError>, |
| 123 | matches: (events: ReadonlyArray<OrchestrationV2DomainEvent>) => boolean = () => true, |
| 124 | ) => |
| 125 | makeLayerTest( |
| 126 | Layer.effect( |
| 127 | EventSink.EventSinkV2, |
| 128 | Effect.gen(function* () { |
| 129 | const eventSink = yield* EventSink.EventSinkV2; |
| 130 | const pending = [...errors]; |
| 131 | const failNext = <A, R>( |
| 132 | events: ReadonlyArray<OrchestrationV2DomainEvent>, |
| 133 | write: Effect.Effect<A, EventSink.EventSinkV2Error, R>, |
| 134 | ) => Effect.suspend(() => (matches(events) ? (pending.shift() ?? write) : write)); |
| 135 | return EventSink.EventSinkV2.of({ |
| 136 | ...eventSink, |
| 137 | write: (input) => failNext(input.events, eventSink.write(input)), |
| 138 | writeIfRunCurrent: (input) => failNext(input.events, eventSink.writeIfRunCurrent(input)), |
| 139 | writeIfProviderThreadOwner: (input) => |
| 140 | failNext(input.events, eventSink.writeIfProviderThreadOwner(input)), |
| 141 | }); |
| 142 | }), |
| 143 | ).pipe(Layer.provide(layerTestEventSink)), |
| 144 | ); |
| 145 | |
| 146 | const makeSharedWorktree = Effect.gen(function* () { |
| 147 | const fs = yield* FileSystem.FileSystem; |
| 148 | const git = yield* GitVcsDriver.GitVcsDriver; |
| 149 | const directory = yield* fs.makeTempDirectoryScoped({ prefix: "ingestor-worktree-" }); |
| 150 | const repo = `${directory}/repo`; |
| 151 | const worktreePath = `${directory}/shared`; |
| 152 | yield* fs.makeDirectory(repo); |
| 153 | const command = (cwd: string, args: string[]) => |
| 154 | git.execute({ operation: "ingestor-test", cwd, args }); |
| 155 | yield* command(repo, ["init", "-b", "main"]); |
| 156 | yield* command(repo, [ |
| 157 | "-c", |
| 158 | "user.name=Test", |
| 159 | "-c", |
| 160 | "user.email=test@example.test", |
| 161 | "commit", |
| 162 | "--allow-empty", |
| 163 | "-m", |
| 164 | "initial", |
| 165 | ]); |
| 166 | yield* command(repo, ["worktree", "add", "-b", "t3/abcd1234", worktreePath]); |
| 167 | return { repo, worktreePath, command }; |
| 168 | }); |
| 169 | const modelSelection = { |
| 170 | instanceId: ProviderInstanceId.make("codex"), |
| 171 | model: "gpt-5.4", |
| 172 | } satisfies ModelSelection; |
| 173 | const CODEX_DRIVER = ProviderDriverKind.make("codex"); |
| 174 | const isProviderEvent = Schema.is(ProviderAdapter.ProviderAdapterV2Event); |
| 175 | |
| 176 | function threadCreatedEvent( |
| 177 | now: DateTime.Utc, |
| 178 | ): Effect.Effect< |
| 179 | OrchestrationV2DomainEvent, |
| 180 | IdAllocator.IdAllocatorV2Error, |
| 181 | IdAllocator.IdAllocatorV2 |
| 182 | > { |
| 183 | return Effect.gen(function* () { |
| 184 | const idAllocator = yield* IdAllocator.IdAllocatorV2; |
| 185 | const projectId = yield* idAllocator.allocate.project({ |
| 186 | fixtureName: "provider-event-ingestor", |
| 187 | }); |
| 188 | const threadId = yield* idAllocator.allocate.thread({ |
| 189 | fixtureName: "provider-event-ingestor", |
| 190 | projectId, |
| 191 | }); |
| 192 | const providerThreadId = idAllocator.derive.providerThread({ |
| 193 | driver: CODEX_DRIVER, |
| 194 | nativeThreadId: "native-thread", |
| 195 | }); |
| 196 | const thread: OrchestrationV2AppThread = { |
| 197 | createdBy: "user", |
| 198 | creationSource: "web", |
| 199 | id: threadId, |
| 200 | projectId, |
| 201 | title: "Provider event ingestor", |
| 202 | providerInstanceId: modelSelection.instanceId, |
| 203 | modelSelection: modelSelection, |
| 204 | runtimeMode: "full-access", |
| 205 | interactionMode: "default", |
| 206 | branch: null, |
| 207 | worktreePath: null, |
| 208 | branchPullRequest: null, |
| 209 | activeOrderKey: null, |
| 210 | activeProviderThreadId: providerThreadId, |
| 211 | lineage: { |
| 212 | parentThreadId: null, |
| 213 | relationshipToParent: null, |
| 214 | rootThreadId: threadId, |
| 215 | }, |
| 216 | forkedFrom: null, |
| 217 | createdAt: now, |
| 218 | updatedAt: now, |
| 219 | archivedAt: null, |
| 220 | settledOverride: null, |
| 221 | settledAt: null, |
| 222 | lastVisitedAt: null, |
| 223 | deletedAt: null, |
| 224 | }; |
| 225 | |
| 226 | return { |
| 227 | id: yield* idAllocator.allocate.event({ threadId }), |
| 228 | type: "thread.created", |
| 229 | threadId, |
| 230 | occurredAt: now, |
| 231 | payload: thread, |
| 232 | }; |
| 233 | }); |
| 234 | } |
| 235 | |
| 236 | const layer = it.layer(layerTest); |
| 237 | |
| 238 | it.effect("takes a hub's reset for an unreset usage_limit, then refreshes the snapshot", () => |
| 239 | Effect.gen(function* () { |
| 240 | const refreshed = yield* Deferred.make<void>(); |
| 241 | const adapter = yield* ProviderAdapter.ProviderAdapterV2.pipe( |
| 242 | Effect.provide( |
| 243 | Layer.mock(ProviderAdapter.ProviderAdapterV2)({ |
| 244 | instanceId: ProviderInstanceId.make("cursor-work"), |
| 245 | driver: ProviderDriverKind.make("cursor"), |
| 246 | }), |
| 247 | ), |
| 248 | ); |
| 249 | const instance: ProviderInstance = { |
| 250 | instanceId: ProviderInstanceId.make("cursor-work"), |
| 251 | driverKind: ProviderDriverKind.make("cursor"), |
| 252 | continuationIdentity: { |
| 253 | driverKind: ProviderDriverKind.make("cursor"), |
| 254 | continuationKey: "work", |
| 255 | }, |
| 256 | enabled: true, |
| 257 | displayName: undefined, |
| 258 | orchestrationAdapter: adapter, |
| 259 | textGeneration: { |
| 260 | generateBranchName: () => Effect.die("unused"), |
| 261 | generateCommitMessage: () => Effect.die("unused"), |
| 262 | generatePrContent: () => Effect.die("unused"), |
| 263 | generateThreadTitle: () => Effect.die("unused"), |
| 264 | generateReviewGuide: () => Effect.die("unused"), |
| 265 | generateTranscriptionPostProcessing: () => Effect.die("unused"), |
| 266 | }, |
| 267 | snapshot: { |
| 268 | refreshUsage: Deferred.succeed(refreshed, undefined).pipe(Effect.asVoid), |
| 269 | getSnapshot: Effect.succeed({ |
| 270 | auth: { status: "authenticated", endpoint: "http://hub.test:8317" }, |
| 271 | } as ServerProvider), |
| 272 | refresh: Effect.die("unused"), |
| 273 | resolveMaintenance: () => Effect.die("unused"), |
| 274 | applyUsageLimits: () => Effect.die("unused"), |
| 275 | streamChanges: Stream.empty, |
| 276 | }, |
| 277 | }; |
| 278 | const lookups: ProviderInstanceId[] = []; |
| 279 | const hubLookups: Array<[string, string, string | null]> = []; |
| 280 | const instances = Layer.mock(ProviderInstanceRegistry.ProviderInstanceRegistry)({ |
| 281 | getInstance: (id) => |
| 282 | Effect.sync(() => { |
| 283 | lookups.push(id); |
| 284 | return id === instance.instanceId ? instance : undefined; |
| 285 | }), |
| 286 | }); |
| 287 | yield* Effect.gen(function* () { |
| 288 | const now = yield* DateTime.now; |
| 289 | const eventSink = yield* EventSink.EventSinkV2; |
| 290 | const ingestor = yield* ProviderEventIngestor.ProviderEventIngestorV2; |
| 291 | const idAllocator = yield* IdAllocator.IdAllocatorV2; |
| 292 | const threadEvent = yield* threadCreatedEvent(now); |
| 293 | yield* eventSink.write({ events: [threadEvent] }); |
| 294 | const input = { |
| 295 | providerSessionId: yield* idAllocator.allocate.providerSession({ |
| 296 | providerInstanceId: instance.instanceId, |
| 297 | threadId: threadEvent.threadId, |
| 298 | }), |
| 299 | providerInstanceId: instance.instanceId, |
| 300 | threadId: threadEvent.threadId, |
| 301 | }; |
| 302 | for (const failureClass of ["validation_error", "usage_limit"] as const) { |
| 303 | const stored = yield* ingestor.ingestNormalized({ |
| 304 | ...input, |
| 305 | event: { |
| 306 | type: "turn.terminal", |
| 307 | driver: instance.driverKind, |
| 308 | providerThreadId: idAllocator.derive.providerThread({ |
| 309 | driver: instance.driverKind, |
| 310 | nativeThreadId: "limited", |
| 311 | }), |
| 312 | providerTurnId: idAllocator.derive.providerTurn({ |
| 313 | driver: instance.driverKind, |
| 314 | nativeTurnId: failureClass, |
| 315 | }), |
| 316 | runOrdinal: 1, |
| 317 | failureItemOrdinal: 1, |
| 318 | status: "failed", |
| 319 | failure: makeProviderFailure({ |
| 320 | class: failureClass, |
| 321 | message: "Provider rejected turn.", |
| 322 | }), |
| 323 | threadDisposition: "reusable", |
| 324 | }, |
| 325 | }); |
| 326 | assert.equal(stored.length, 1); |
| 327 | if (failureClass === "validation_error") assert.deepEqual(lookups, []); |
| 328 | else { |
| 329 | const event = stored[0]?.event; |
| 330 | assert.equal( |
| 331 | event?.type === "turn-item.updated" && event.payload.type === "error" |
| 332 | ? event.payload.failure.resetAt |
| 333 | : undefined, |
| 334 | "2026-11-13T18:00:00.000Z", |
| 335 | ); |
| 336 | } |
| 337 | } |
| 338 | assert.deepEqual(lookups, [instance.instanceId, instance.instanceId]); |
| 339 | assert.deepEqual(hubLookups, [["cursor", "http://hub.test:8317", "gpt-5.4"]]); |
| 340 | yield* Deferred.await(refreshed); |
| 341 | }).pipe( |
| 342 | Effect.provide(makeLayerTest(layerTestEventSink, instances)), |
| 343 | Effect.provideService(UsageLimitResetFallback, { |
| 344 | resetAt: (driver, endpoint, model) => |
| 345 | Effect.sync(() => { |
| 346 | hubLookups.push([driver, endpoint, model]); |
| 347 | return "2026-11-13T18:00:00.000Z"; |
| 348 | }), |
| 349 | }), |
| 350 | ); |
| 351 | }), |
| 352 | ); |
| 353 | |
| 354 | it.effect("records accepted billed turn usage once without billing the context window", () => { |
| 355 | const recorded: Array<ProviderUsageAnalytics.ProviderTurnReport> = []; |
| 356 | const layerAnalytics = Layer.succeed(ProviderUsageAnalytics.ProviderTurnAnalytics, { |
| 357 | runStarted: () => Effect.void, |
| 358 | turnCompleted: (report) => |
| 359 | Effect.sync(() => { |
| 360 | recorded.push(report); |
| 361 | }), |
| 362 | runSettled: () => Effect.void, |
| 363 | }); |
| 364 | return Effect.gen(function* () { |
| 365 | const now = yield* DateTime.now; |
| 366 | const eventSink = yield* EventSink.EventSinkV2; |
| 367 | const ingestor = yield* ProviderEventIngestor.ProviderEventIngestorV2; |
| 368 | const idAllocator = yield* IdAllocator.IdAllocatorV2; |
| 369 | const threadEvent = yield* threadCreatedEvent(now); |
| 370 | yield* eventSink.write({ events: [threadEvent] }); |
| 371 | const providerSessionId = yield* idAllocator.allocate.providerSession({ |
| 372 | providerInstanceId: modelSelection.instanceId, |
| 373 | threadId: threadEvent.threadId, |
| 374 | }); |
| 375 | const providerThreadId = idAllocator.derive.providerThread({ |
| 376 | driver: CODEX_DRIVER, |
| 377 | nativeThreadId: "billing-thread", |
| 378 | }); |
| 379 | const providerTurn = { |
| 380 | id: idAllocator.derive.providerTurn({ driver: CODEX_DRIVER, nativeTurnId: "billing-turn" }), |
| 381 | providerThreadId, |
| 382 | nodeId: NodeId.make("node:billing-turn"), |
| 383 | runAttemptId: null, |
| 384 | nativeTurnRef: null, |
| 385 | ordinal: 1, |
| 386 | status: "completed" as const, |
| 387 | startedAt: now, |
| 388 | completedAt: DateTime.makeUnsafe(DateTime.toEpochMillis(now) + 120), |
| 389 | tokenUsage: { |
| 390 | inputTokens: 4000, |
| 391 | cachedInputTokens: 3000, |
| 392 | outputTokens: 100, |
| 393 | usedTokens: 4100, |
| 394 | updatedAt: DateTime.formatIso(now), |
| 395 | }, |
| 396 | turnTokenUsage: { |
| 397 | usageStatus: "complete" as const, |
| 398 | usageScope: "main_agent" as const, |
| 399 | hasSubagents: false, |
| 400 | inputTokens: 40, |
| 401 | cachedInputTokens: 30, |
| 402 | outputTokens: 10, |
| 403 | }, |
| 404 | }; |
| 405 | const input = { |
| 406 | providerSessionId, |
| 407 | providerInstanceId: modelSelection.instanceId, |
| 408 | threadId: threadEvent.threadId, |
| 409 | analyticsContext: { |
| 410 | modelSelection, |
| 411 | runtimeMode: "full-access" as const, |
| 412 | interactionMode: "default" as const, |
| 413 | origin: { |
| 414 | createdBy: "agent" as const, |
| 415 | creationSource: "provider" as const, |
| 416 | scheduled: false, |
| 417 | subagent: true, |
| 418 | }, |
| 419 | }, |
| 420 | event: { type: "provider_turn.updated" as const, driver: CODEX_DRIVER, providerTurn }, |
| 421 | }; |
| 422 | yield* ingestor.ingestNormalized(input); |
| 423 | yield* ingestor.ingestNormalized(input); |
| 424 | const ignored = yield* ingestor.ingestNormalized({ |
| 425 | ...input, |
| 426 | event: { |
| 427 | ...input.event, |
| 428 | providerTurn: { |
| 429 | ...providerTurn, |
| 430 | id: idAllocator.derive.providerTurn({ |
| 431 | driver: CODEX_DRIVER, |
| 432 | nativeTurnId: "stale-billing-turn", |
| 433 | }), |
| 434 | }, |
| 435 | }, |
| 436 | writeIfRunCurrent: { |
| 437 | runId: RunId.make("missing-run"), |
| 438 | activeAttemptId: RunAttemptId.make("stale-attempt"), |
| 439 | expectedStatus: "running", |
| 440 | }, |
| 441 | }); |
| 442 | assert.isEmpty(ignored); |
| 443 | assert.lengthOf(recorded, 1); |
| 444 | assert.deepEqual(recorded[0], { |
| 445 | driver: CODEX_DRIVER, |
| 446 | providerTurn, |
| 447 | context: input.analyticsContext, |
| 448 | }); |
| 449 | }).pipe(Effect.provide(layerTest.pipe(Layer.provide(layerAnalytics)))); |
| 450 | }); |
| 451 | |
| 452 | it.effect( |
| 453 | "reports invalid provider questions as typed errors and still ingests valid questions", |
| 454 | () => |
| 455 | Effect.gen(function* () { |
| 456 | const now = yield* DateTime.now; |
| 457 | const eventSink = yield* EventSink.EventSinkV2; |
| 458 | const eventStore = yield* EventStore.EventStoreV2; |
| 459 | const projectionStore = yield* ProjectionStore.ProjectionStoreV2; |
| 460 | const ingestor = yield* ProviderEventIngestor.ProviderEventIngestorV2; |
| 461 | const idAllocator = yield* IdAllocator.IdAllocatorV2; |
| 462 | const threadEvent = yield* threadCreatedEvent(now); |
| 463 | const threadId = threadEvent.threadId; |
| 464 | yield* eventSink.write({ events: [threadEvent] }); |
| 465 | const providerInstanceId = ProviderInstanceId.make("pi"); |
| 466 | const providerSessionId = yield* idAllocator.allocate.providerSession({ |
| 467 | providerInstanceId, |
| 468 | threadId, |
| 469 | }); |
| 470 | const question = { id: "input", header: "Enter a value", question: "", options: [] }; |
| 471 | const item = { |
| 472 | id: TurnItemId.make(`${threadId}:question`), |
| 473 | threadId, |
| 474 | runId: null, |
| 475 | nodeId: null, |
| 476 | providerThreadId: null, |
| 477 | providerTurnId: null, |
| 478 | nativeItemRef: null, |
| 479 | parentItemId: null, |
| 480 | ordinal: 1, |
| 481 | status: "waiting", |
| 482 | title: "Enter a value", |
| 483 | startedAt: now, |
| 484 | completedAt: null, |
| 485 | updatedAt: now, |
| 486 | type: "user_input_request", |
| 487 | requestId: RuntimeRequestId.make(`${threadId}:question`), |
| 488 | questions: [question], |
| 489 | } satisfies OrchestrationV2TurnItem; |
| 490 | const event = { |
| 491 | type: "turn_item.updated", |
| 492 | driver: ProviderDriverKind.make("pi"), |
| 493 | turnItem: item, |
| 494 | } satisfies ProviderAdapter.ProviderAdapterV2Event; |
| 495 | const input = { providerSessionId, providerInstanceId, threadId, event }; |
| 496 | |
| 497 | assert.isFalse(isProviderEvent(event)); |
| 498 | const error = yield* ingestor.ingestNormalized(input).pipe(Effect.flip); |
| 499 | assert.instanceOf(error, ProviderEventIngestor.ProviderEventNormalizeError); |
| 500 | assert.equal( |
| 501 | error.message, |
| 502 | `Failed to normalize provider event turn_item.updated for thread ${threadId}.`, |
| 503 | ); |
| 504 | assert.instanceOf(error.cause, Schema.SchemaError); |
| 505 | assert.equal(yield* eventStore.latestSequence({ threadId }), 1); |
| 506 | assert.isEmpty((yield* projectionStore.getThreadProjection(threadId)).turnItems); |
| 507 | |
| 508 | const validEvent = { |
| 509 | ...event, |
| 510 | turnItem: { ...item, questions: [{ ...question, question: "Enter a value" }] }, |
| 511 | }; |
| 512 | assert.isTrue(isProviderEvent(validEvent)); |
| 513 | const stored = yield* ingestor.ingestNormalized({ ...input, event: validEvent }); |
| 514 | assert.lengthOf(stored, 1); |
| 515 | assert.equal(stored[0]?.event.type, "turn-item.updated"); |
| 516 | assert.deepEqual((yield* projectionStore.getThreadProjection(threadId)).turnItems, [ |
| 517 | validEvent.turnItem, |
| 518 | ]); |
| 519 | }).pipe(Effect.provide(layerTest)), |
| 520 | ); |
| 521 | |
| 522 | layer("ProviderEventIngestorV2", (it) => { |
| 523 | it.effect("normalizes provider events through the real event log and projection store", () => |
| 524 | Effect.gen(function* () { |
| 525 | const now = yield* DateTime.now; |
| 526 | const eventSink = yield* EventSink.EventSinkV2; |
| 527 | const eventStore = yield* EventStore.EventStoreV2; |
| 528 | const projectionStore = yield* ProjectionStore.ProjectionStoreV2; |
| 529 | const ingestor = yield* ProviderEventIngestor.ProviderEventIngestorV2; |
| 530 | const idAllocator = yield* IdAllocator.IdAllocatorV2; |
| 531 | const threadEvent = yield* threadCreatedEvent(now); |
| 532 | const providerSessionId = yield* idAllocator.allocate.providerSession({ |
| 533 | providerInstanceId: modelSelection.instanceId, |
| 534 | threadId: threadEvent.threadId, |
| 535 | }); |
| 536 | const providerThread: OrchestrationV2ProviderThread = { |
| 537 | id: idAllocator.derive.providerThread({ |
| 538 | driver: CODEX_DRIVER, |
| 539 | nativeThreadId: "native-thread", |
| 540 | }), |
| 541 | driver: CODEX_DRIVER, |
| 542 | providerInstanceId: modelSelection.instanceId, |
| 543 | providerSessionId, |
| 544 | appThreadId: threadEvent.threadId, |
| 545 | ownerNodeId: null, |
| 546 | nativeThreadRef: { |
| 547 | driver: CODEX_DRIVER, |
| 548 | nativeId: "native-thread", |
| 549 | strength: "strong", |
| 550 | }, |
| 551 | nativeConversationHeadRef: null, |
| 552 | status: "idle", |
| 553 | firstRunOrdinal: null, |
| 554 | lastRunOrdinal: null, |
| 555 | handoffIds: [], |
| 556 | forkedFrom: null, |
| 557 | createdAt: now, |
| 558 | updatedAt: now, |
| 559 | }; |
| 560 | |
| 561 | yield* eventSink.write({ events: [threadEvent] }); |
| 562 | const storedEvents = yield* ingestor.ingestNormalized({ |
| 563 | providerSessionId, |
| 564 | providerInstanceId: modelSelection.instanceId, |
| 565 | threadId: threadEvent.threadId, |
| 566 | event: { |
| 567 | type: "provider_thread.updated", |
| 568 | driver: CODEX_DRIVER, |
| 569 | providerThread, |
| 570 | }, |
| 571 | }); |
| 572 | |
| 573 | const projection = yield* projectionStore.getThreadProjection(threadEvent.threadId); |
| 574 | const storedDomainEvents = yield* eventStore.read({}).pipe(Stream.runCollect); |
| 575 | const afterFirstEvent = yield* eventStore |
| 576 | .read({ afterSequence: 1, threadId: threadEvent.threadId }) |
| 577 | .pipe(Stream.runCollect); |
| 578 | const latestThreadSequence = yield* eventStore.latestSequence({ |
| 579 | threadId: threadEvent.threadId, |
| 580 | }); |
| 581 | |
| 582 | assert.equal(storedEvents.length, 1); |
| 583 | assert.equal(storedEvents[0]?.event.type, "provider-thread.updated"); |
| 584 | assert.deepEqual( |
| 585 | projection.providerThreads.map((thread) => thread.id), |
| 586 | [providerThread.id], |
| 587 | ); |
| 588 | assert.deepEqual( |
| 589 | Array.from(storedDomainEvents).map((stored) => stored.event.type), |
| 590 | ["thread.created", "provider-thread.updated"], |
| 591 | ); |
| 592 | assert.deepEqual( |
| 593 | Array.from(storedDomainEvents).map((stored) => stored.sequence), |
| 594 | [1, 2], |
| 595 | ); |
| 596 | assert.deepEqual( |
| 597 | Array.from(afterFirstEvent).map((stored) => stored.event.type), |
| 598 | ["provider-thread.updated"], |
| 599 | ); |
| 600 | assert.equal(latestThreadSequence, 2); |
| 601 | }), |
| 602 | ); |
| 603 | |
| 604 | it.effect("skips a node update only when it matches the stored node", () => |
| 605 | Effect.gen(function* () { |
| 606 | const now = yield* DateTime.now; |
| 607 | const eventSink = yield* EventSink.EventSinkV2; |
| 608 | const projectionStore = yield* ProjectionStore.ProjectionStoreV2; |
| 609 | const ingestor = yield* ProviderEventIngestor.ProviderEventIngestorV2; |
| 610 | const idAllocator = yield* IdAllocator.IdAllocatorV2; |
| 611 | const threadEvent = yield* threadCreatedEvent(now); |
| 612 | const threadId = threadEvent.threadId; |
| 613 | yield* eventSink.write({ events: [threadEvent] }); |
| 614 | const providerSessionId = yield* idAllocator.allocate.providerSession({ |
| 615 | providerInstanceId: modelSelection.instanceId, |
| 616 | threadId, |
| 617 | }); |
| 618 | const nodeId = NodeId.make(`${threadId}:approval`); |
| 619 | const node: OrchestrationV2ExecutionNode = { |
| 620 | id: nodeId, |
| 621 | threadId, |
| 622 | runId: null, |
| 623 | parentNodeId: null, |
| 624 | rootNodeId: nodeId, |
| 625 | kind: "approval_request", |
| 626 | status: "waiting", |
| 627 | countsForRun: false, |
| 628 | providerThreadId: null, |
| 629 | providerTurnId: null, |
| 630 | nativeItemRef: null, |
| 631 | runtimeRequestId: null, |
| 632 | checkpointScopeId: null, |
| 633 | startedAt: now, |
| 634 | completedAt: null, |
| 635 | }; |
| 636 | const ingestNode = (sent: OrchestrationV2ExecutionNode) => |
| 637 | ingestor.ingestNormalized({ |
| 638 | providerSessionId, |
| 639 | providerInstanceId: modelSelection.instanceId, |
| 640 | threadId, |
| 641 | event: { type: "node.updated", driver: CODEX_DRIVER, node: sent }, |
| 642 | }); |
| 643 | |
| 644 | yield* eventSink.withProviderBatch( |
| 645 | Effect.gen(function* () { |
| 646 | assert.lengthOf(yield* ingestNode(node), 1); |
| 647 | assert.isEmpty(yield* ingestNode({ ...node })); |
| 648 | }), |
| 649 | ); |
| 650 | yield* eventSink.write({ |
| 651 | events: [ |
| 652 | { |
| 653 | id: yield* idAllocator.allocate.event({ threadId }), |
| 654 | type: "node.updated", |
| 655 | threadId, |
| 656 | occurredAt: now, |
| 657 | payload: { ...node, status: "completed", completedAt: now }, |
| 658 | }, |
| 659 | ], |
| 660 | }); |
| 661 | assert.lengthOf(yield* ingestNode(node), 1); |
| 662 | const projection = yield* projectionStore.getThreadProjection(threadId); |
| 663 | assert.equal(projection.nodes.find((entry) => entry.id === nodeId)?.status, "waiting"); |
| 664 | }), |
| 665 | ); |
| 666 | |
| 667 | it.effect("rolls back runs a provider rewound off its active branch", () => |
| 668 | Effect.gen(function* () { |
| 669 | const now = yield* DateTime.now; |
| 670 | const eventSink = yield* EventSink.EventSinkV2; |
| 671 | const projectionStore = yield* ProjectionStore.ProjectionStoreV2; |
| 672 | const ingestor = yield* ProviderEventIngestor.ProviderEventIngestorV2; |
| 673 | const idAllocator = yield* IdAllocator.IdAllocatorV2; |
| 674 | const threadEvent = yield* threadCreatedEvent(now); |
| 675 | const threadId = threadEvent.threadId; |
| 676 | const providerSessionId = yield* idAllocator.allocate.providerSession({ |
| 677 | providerInstanceId: modelSelection.instanceId, |
| 678 | threadId, |
| 679 | }); |
| 680 | const rewoundThreadId = idAllocator.derive.providerThread({ |
| 681 | driver: CODEX_DRIVER, |
| 682 | nativeThreadId: "rewound-thread", |
| 683 | }); |
| 684 | const otherThreadId = idAllocator.derive.providerThread({ |
| 685 | driver: CODEX_DRIVER, |
| 686 | nativeThreadId: "other-thread", |
| 687 | }); |
| 688 | const scopeId = CheckpointScopeId.make("checkpoint-scope:rewound"); |
| 689 | const seedRun = Effect.fnUntraced(function* (input: { |
| 690 | readonly ordinal: number; |
| 691 | readonly providerThreadId: ProviderThreadId; |
| 692 | readonly status: "completed" | "interrupted" | "running"; |
| 693 | readonly nativeTurnId: string | null; |
| 694 | }) { |
| 695 | const runId = RunId.make(`run:rewound:${input.ordinal}`); |
| 696 | const attemptId = RunAttemptId.make(`attempt:rewound:${input.ordinal}`); |
| 697 | const nodeId = NodeId.make(`node:rewound:${input.ordinal}`); |
| 698 | const providerTurnId = ProviderTurnId.make(`provider-turn:rewound:${input.ordinal}`); |
| 699 | const completedAt = input.status === "running" ? null : now; |
| 700 | const eventId = () => idAllocator.allocate.event({ threadId }); |
| 701 | const events: Array<OrchestrationV2DomainEvent> = [ |
| 702 | { |
| 703 | id: yield* eventId(), |
| 704 | type: "run.created", |
| 705 | threadId, |
| 706 | occurredAt: now, |
| 707 | payload: { |
| 708 | id: runId, |
| 709 | threadId, |
| 710 | ordinal: input.ordinal, |
| 711 | providerInstanceId: modelSelection.instanceId, |
| 712 | modelSelection, |
| 713 | providerThreadId: input.providerThreadId, |
| 714 | userMessageId: MessageId.make(`message:rewound:${input.ordinal}`), |
| 715 | rootNodeId: nodeId, |
| 716 | activeAttemptId: attemptId, |
| 717 | status: input.status, |
| 718 | requestedAt: now, |
| 719 | startedAt: now, |
| 720 | completedAt, |
| 721 | checkpointId: null, |
| 722 | contextHandoffId: null, |
| 723 | }, |
| 724 | }, |
| 725 | { |
| 726 | id: yield* eventId(), |
| 727 | type: "run-attempt.created", |
| 728 | threadId, |
| 729 | occurredAt: now, |
| 730 | payload: { |
| 731 | id: attemptId, |
| 732 | runId, |
| 733 | attemptOrdinal: 1, |
| 734 | rootNodeId: nodeId, |
| 735 | providerInstanceId: modelSelection.instanceId, |
| 736 | providerThreadId: input.providerThreadId, |
| 737 | providerTurnId, |
| 738 | reason: "initial", |
| 739 | status: input.status, |
| 740 | startedAt: now, |
| 741 | completedAt, |
| 742 | }, |
| 743 | }, |
| 744 | { |
| 745 | id: yield* eventId(), |
| 746 | type: "provider-turn.updated", |
| 747 | threadId, |
| 748 | occurredAt: now, |
| 749 | payload: { |
| 750 | id: providerTurnId, |
| 751 | providerThreadId: input.providerThreadId, |
| 752 | nodeId, |
| 753 | runAttemptId: attemptId, |
| 754 | nativeTurnRef: |
| 755 | input.nativeTurnId === null |
| 756 | ? { |
| 757 | driver: CODEX_DRIVER, |
| 758 | nativeId: `synthetic:${input.ordinal}`, |
| 759 | strength: "weak", |
| 760 | } |
| 761 | : { driver: CODEX_DRIVER, nativeId: input.nativeTurnId, strength: "strong" }, |
| 762 | ordinal: input.ordinal, |
| 763 | status: input.status, |
| 764 | startedAt: now, |
| 765 | completedAt, |
| 766 | }, |
| 767 | }, |
| 768 | { |
| 769 | id: yield* eventId(), |
| 770 | type: "node.updated", |
| 771 | threadId, |
| 772 | occurredAt: now, |
| 773 | payload: { |
| 774 | id: nodeId, |
| 775 | threadId, |
| 776 | runId, |
| 777 | parentNodeId: null, |
| 778 | rootNodeId: nodeId, |
| 779 | kind: "root_turn", |
| 780 | status: input.status, |
| 781 | countsForRun: true, |
| 782 | providerThreadId: input.providerThreadId, |
| 783 | providerTurnId, |
| 784 | nativeItemRef: null, |
| 785 | runtimeRequestId: null, |
| 786 | checkpointScopeId: scopeId, |
| 787 | startedAt: now, |
| 788 | completedAt, |
| 789 | }, |
| 790 | }, |
| 791 | ...(input.status === "completed" |
| 792 | ? [ |
| 793 | { |
| 794 | id: yield* eventId(), |
| 795 | type: "checkpoint.captured" as const, |
| 796 | threadId, |
| 797 | occurredAt: now, |
| 798 | payload: { |
| 799 | id: CheckpointId.make(`checkpoint:rewound:${input.ordinal}`), |
| 800 | threadId, |
| 801 | scopeId, |
| 802 | runId, |
| 803 | nodeId, |
| 804 | parentCheckpointId: null, |
| 805 | ordinalWithinScope: input.ordinal, |
| 806 | appRunOrdinal: input.ordinal, |
| 807 | ref: CheckpointRef.make(`refs/t3/checkpoints/rewound/${input.ordinal}`), |
| 808 | status: "ready" as const, |
| 809 | files: [], |
| 810 | capturedAt: now, |
| 811 | }, |
| 812 | }, |
| 813 | ] |
| 814 | : []), |
| 815 | ]; |
| 816 | yield* eventSink.write({ events }); |
| 817 | return runId; |
| 818 | }); |
| 819 | |
| 820 | yield* eventSink.write({ events: [threadEvent] }); |
| 821 | const otherProviderRun = yield* seedRun({ |
| 822 | ordinal: 1, |
| 823 | providerThreadId: otherThreadId, |
| 824 | status: "completed", |
| 825 | nativeTurnId: "other-turn", |
| 826 | }); |
| 827 | const keptRun = yield* seedRun({ |
| 828 | ordinal: 2, |
| 829 | providerThreadId: rewoundThreadId, |
| 830 | status: "completed", |
| 831 | nativeTurnId: "kept-turn", |
| 832 | }); |
| 833 | const abandonedRun = yield* seedRun({ |
| 834 | ordinal: 3, |
| 835 | providerThreadId: rewoundThreadId, |
| 836 | status: "completed", |
| 837 | nativeTurnId: "abandoned-turn", |
| 838 | }); |
| 839 | const unlocatedRun = yield* seedRun({ |
| 840 | ordinal: 4, |
| 841 | providerThreadId: rewoundThreadId, |
| 842 | status: "interrupted", |
| 843 | nativeTurnId: null, |
| 844 | }); |
| 845 | const rewindingRun = yield* seedRun({ |
| 846 | ordinal: 5, |
| 847 | providerThreadId: rewoundThreadId, |
| 848 | status: "running", |
| 849 | nativeTurnId: null, |
| 850 | }); |
| 851 | |
| 852 | yield* ingestor.ingestNormalized({ |
| 853 | providerSessionId, |
| 854 | providerInstanceId: modelSelection.instanceId, |
| 855 | threadId, |
| 856 | runId: rewindingRun, |
| 857 | event: { |
| 858 | type: "provider_thread.updated", |
| 859 | driver: CODEX_DRIVER, |
| 860 | providerThread: { |
| 861 | id: rewoundThreadId, |
| 862 | driver: CODEX_DRIVER, |
| 863 | providerInstanceId: modelSelection.instanceId, |
| 864 | providerSessionId, |
| 865 | appThreadId: threadId, |
| 866 | ownerNodeId: null, |
| 867 | nativeThreadRef: { |
| 868 | driver: CODEX_DRIVER, |
| 869 | nativeId: "rewound-thread", |
| 870 | strength: "strong", |
| 871 | }, |
| 872 | nativeConversationHeadRef: { |
| 873 | driver: CODEX_DRIVER, |
| 874 | nativeId: "kept-turn-reply", |
| 875 | strength: "strong", |
| 876 | }, |
| 877 | status: "idle", |
| 878 | firstRunOrdinal: 2, |
| 879 | lastRunOrdinal: 5, |
| 880 | handoffIds: [], |
| 881 | forkedFrom: null, |
| 882 | createdAt: now, |
| 883 | updatedAt: now, |
| 884 | }, |
| 885 | retainedNativeTurnIds: ["turn-started-outside-t3", "kept-turn"], |
| 886 | }, |
| 887 | }); |
| 888 | |
| 889 | const projection = yield* projectionStore.getThreadProjection(threadId); |
| 890 | const runIds = [otherProviderRun, keptRun, abandonedRun, unlocatedRun, rewindingRun]; |
| 891 | assert.deepEqual( |
| 892 | runIds.map((runId) => projection.runs.find((run) => run.id === runId)?.status), |
| 893 | ["completed", "completed", "rolled_back", "rolled_back", "running"], |
| 894 | ); |
| 895 | assert.deepEqual( |
| 896 | runIds.map((runId) => projection.nodes.find((node) => node.runId === runId)?.status), |
| 897 | ["completed", "completed", "rolled_back", "rolled_back", "running"], |
| 898 | ); |
| 899 | assert.deepEqual( |
| 900 | [otherProviderRun, keptRun, abandonedRun].map( |
| 901 | (runId) => |
| 902 | projection.checkpoints.find((checkpoint) => checkpoint.runId === runId)?.status, |
| 903 | ), |
| 904 | ["ready", "ready", "stale"], |
| 905 | ); |
| 906 | assert.equal( |
| 907 | projection.providerThreads.find((thread) => thread.id === rewoundThreadId)?.lastRunOrdinal, |
| 908 | 5, |
| 909 | ); |
| 910 | }), |
| 911 | ); |
| 912 | |
| 913 | it.effect( |
| 914 | "treats successful provider terminal markers as non-persisted orchestration control signals", |
| 915 | () => |
| 916 | Effect.gen(function* () { |
| 917 | const ingestor = yield* ProviderEventIngestor.ProviderEventIngestorV2; |
| 918 | const idAllocator = yield* IdAllocator.IdAllocatorV2; |
| 919 | const projectId = yield* idAllocator.allocate.project({ |
| 920 | fixtureName: "provider-event-terminal", |
| 921 | }); |
| 922 | const threadId = yield* idAllocator.allocate.thread({ |
| 923 | fixtureName: "provider-event-terminal", |
| 924 | projectId, |
| 925 | }); |
| 926 | const providerSessionId = yield* idAllocator.allocate.providerSession({ |
| 927 | providerInstanceId: modelSelection.instanceId, |
| 928 | threadId, |
| 929 | }); |
| 930 | const normalized = yield* ingestor.normalize({ |
| 931 | providerSessionId, |
| 932 | providerInstanceId: modelSelection.instanceId, |
| 933 | threadId, |
| 934 | event: { |
| 935 | type: "turn.terminal", |
| 936 | driver: CODEX_DRIVER, |
| 937 | providerThreadId: idAllocator.derive.providerThread({ |
| 938 | driver: CODEX_DRIVER, |
| 939 | nativeThreadId: "native-thread", |
| 940 | }), |
| 941 | providerTurnId: idAllocator.derive.providerTurn({ |
| 942 | driver: CODEX_DRIVER, |
| 943 | nativeTurnId: "native-turn", |
| 944 | }), |
| 945 | runOrdinal: 1, |
| 946 | status: "completed", |
| 947 | failure: null, |
| 948 | threadDisposition: "reusable", |
| 949 | }, |
| 950 | }); |
| 951 | |
| 952 | assert.deepEqual(normalized, []); |
| 953 | }), |
| 954 | ); |
| 955 | |
| 956 | it.effect("persists an interrupted run's inherited terminal through the live run router", () => |
| 957 | Effect.gen(function* () { |
| 958 | const now = yield* DateTime.now; |
| 959 | const eventSink = yield* EventSink.EventSinkV2; |
| 960 | const projectionStore = yield* ProjectionStore.ProjectionStoreV2; |
| 961 | const ingestor = yield* ProviderEventIngestor.ProviderEventIngestorV2; |
| 962 | const idAllocator = yield* IdAllocator.IdAllocatorV2; |
| 963 | const threadEvent = yield* threadCreatedEvent(now); |
| 964 | const priorRunId = RunId.make("run:provider-event-inherited:prior"); |
| 965 | const currentRunId = RunId.make("run:provider-event-inherited:current"); |
| 966 | const itemId = TurnItemId.make("turn-item:provider-event-inherited"); |
| 967 | const providerSessionId = yield* idAllocator.allocate.providerSession({ |
| 968 | providerInstanceId: modelSelection.instanceId, |
| 969 | threadId: threadEvent.threadId, |
| 970 | }); |
| 971 | const providerThreadId = idAllocator.derive.providerThread({ |
| 972 | driver: CODEX_DRIVER, |
| 973 | nativeThreadId: "native-thread-inherited", |
| 974 | }); |
| 975 | const providerTurnId = idAllocator.derive.providerTurn({ |
| 976 | driver: CODEX_DRIVER, |
| 977 | nativeTurnId: "native-turn-inherited", |
| 978 | }); |
| 979 | const runningItem = { |
| 980 | id: itemId, |
| 981 | threadId: threadEvent.threadId, |
| 982 | runId: priorRunId, |
| 983 | nodeId: NodeId.make("node:provider-event-inherited"), |
| 984 | providerThreadId, |
| 985 | providerTurnId, |
| 986 | nativeItemRef: null, |
| 987 | parentItemId: null, |
| 988 | ordinal: 101, |
| 989 | status: "running", |
| 990 | title: "Inherited background command", |
| 991 | startedAt: now, |
| 992 | completedAt: null, |
| 993 | updatedAt: now, |
| 994 | type: "command_execution", |
| 995 | input: "sleep 60", |
| 996 | } satisfies OrchestrationV2TurnItem; |
| 997 | const terminalItem = { |
| 998 | ...runningItem, |
| 999 | status: "completed" as const, |
| 1000 | completedAt: now, |
| 1001 | updatedAt: now, |
| 1002 | }; |
| 1003 | |
| 1004 | yield* eventSink.write({ events: [threadEvent] }); |
| 1005 | yield* ingestor.ingestNormalized({ |
| 1006 | providerSessionId, |
| 1007 | providerInstanceId: modelSelection.instanceId, |
| 1008 | threadId: threadEvent.threadId, |
| 1009 | runId: priorRunId, |
| 1010 | event: { type: "turn_item.updated", driver: CODEX_DRIVER, turnItem: runningItem }, |
| 1011 | }); |
| 1012 | |
| 1013 | const identity: ProviderEventRouteIdentity = { |
| 1014 | threadId: threadEvent.threadId, |
| 1015 | runId: currentRunId, |
| 1016 | attemptId: RunAttemptId.make("attempt:provider-event-inherited:current"), |
| 1017 | providerThreadId, |
| 1018 | }; |
| 1019 | const inheritedBackgroundTurnItems = selectInheritedBackgroundTurnItems({ |
| 1020 | threadId: threadEvent.threadId, |
| 1021 | currentProviderThreadId: providerThreadId, |
| 1022 | currentRunOrdinal: 2, |
| 1023 | runs: [ |
| 1024 | { |
| 1025 | id: priorRunId, |
| 1026 | threadId: threadEvent.threadId, |
| 1027 | ordinal: 1, |
| 1028 | status: "interrupted", |
| 1029 | } as OrchestrationV2Run, |
| 1030 | { |
| 1031 | id: currentRunId, |
| 1032 | threadId: threadEvent.threadId, |
| 1033 | ordinal: 2, |
| 1034 | status: "running", |
| 1035 | } as OrchestrationV2Run, |
| 1036 | ], |
| 1037 | turnItems: [runningItem], |
| 1038 | }); |
| 1039 | const routeState = makeProviderEventRoutingState({ |
| 1040 | identity, |
| 1041 | inheritedBackgroundTurnItems, |
| 1042 | providerTurnId: null, |
| 1043 | }); |
| 1044 | const terminalEvent = { |
| 1045 | type: "turn_item.updated", |
| 1046 | driver: CODEX_DRIVER, |
| 1047 | turnItem: terminalItem, |
| 1048 | } as const; |
| 1049 | const [accepted] = routeProviderEvent(terminalEvent, identity, routeState); |
| 1050 | assert.isTrue(accepted); |
| 1051 | |
| 1052 | const stored = yield* ingestor.ingestNormalized({ |
| 1053 | providerSessionId, |
| 1054 | providerInstanceId: modelSelection.instanceId, |
| 1055 | threadId: threadEvent.threadId, |
| 1056 | runId: currentRunId, |
| 1057 | event: terminalEvent, |
| 1058 | }); |
| 1059 | const projection = yield* projectionStore.getThreadProjection(threadEvent.threadId); |
| 1060 | const persisted = projection.turnItems.find((item) => item.id === itemId); |
| 1061 | |
| 1062 | assert.equal(stored.length, 1); |
| 1063 | assert.equal(stored[0]?.event.type, "turn-item.updated"); |
| 1064 | assert.equal(persisted?.runId, priorRunId); |
| 1065 | assert.equal(persisted?.threadId, threadEvent.threadId); |
| 1066 | assert.equal(persisted?.status, "completed"); |
| 1067 | }), |
| 1068 | ); |
| 1069 | |
| 1070 | it.effect("persists a completed run's late background terminal exactly once", () => |
| 1071 | Effect.gen(function* () { |
| 1072 | const now = yield* DateTime.now; |
| 1073 | const eventSink = yield* EventSink.EventSinkV2; |
| 1074 | const eventStore = yield* EventStore.EventStoreV2; |
| 1075 | const ingestor = yield* ProviderEventIngestor.ProviderEventIngestorV2; |
| 1076 | const idAllocator = yield* IdAllocator.IdAllocatorV2; |
| 1077 | const threadEvent = yield* threadCreatedEvent(now); |
| 1078 | const priorRunId = RunId.make("run:provider-event-completed:prior"); |
| 1079 | const currentRunId = RunId.make("run:provider-event-completed:current"); |
| 1080 | const itemId = TurnItemId.make("turn-item:provider-event-completed"); |
| 1081 | const providerSessionId = yield* idAllocator.allocate.providerSession({ |
| 1082 | providerInstanceId: modelSelection.instanceId, |
| 1083 | threadId: threadEvent.threadId, |
| 1084 | }); |
| 1085 | const providerThreadId = idAllocator.derive.providerThread({ |
| 1086 | driver: CODEX_DRIVER, |
| 1087 | nativeThreadId: "native-thread-completed", |
| 1088 | }); |
| 1089 | const providerTurnId = idAllocator.derive.providerTurn({ |
| 1090 | driver: CODEX_DRIVER, |
| 1091 | nativeTurnId: "native-turn-completed", |
| 1092 | }); |
| 1093 | const runningItem = { |
| 1094 | id: itemId, |
| 1095 | threadId: threadEvent.threadId, |
| 1096 | runId: priorRunId, |
| 1097 | nodeId: NodeId.make("node:provider-event-completed"), |
| 1098 | providerThreadId, |
| 1099 | providerTurnId, |
| 1100 | nativeItemRef: null, |
| 1101 | parentItemId: null, |
| 1102 | ordinal: 101, |
| 1103 | status: "running", |
| 1104 | title: "Completed run background command", |
| 1105 | startedAt: now, |
| 1106 | completedAt: null, |
| 1107 | updatedAt: now, |
| 1108 | type: "command_execution", |
| 1109 | input: "sleep 60", |
| 1110 | } satisfies OrchestrationV2TurnItem; |
| 1111 | const terminalEvent = { |
| 1112 | type: "turn_item.updated", |
| 1113 | driver: CODEX_DRIVER, |
| 1114 | turnItem: { |
| 1115 | ...runningItem, |
| 1116 | status: "completed" as const, |
| 1117 | completedAt: now, |
| 1118 | updatedAt: now, |
| 1119 | }, |
| 1120 | } as const; |
| 1121 | |
| 1122 | yield* eventSink.write({ events: [threadEvent] }); |
| 1123 | yield* ingestor.ingestNormalized({ |
| 1124 | providerSessionId, |
| 1125 | providerInstanceId: modelSelection.instanceId, |
| 1126 | threadId: threadEvent.threadId, |
| 1127 | runId: priorRunId, |
| 1128 | event: { type: "turn_item.updated", driver: CODEX_DRIVER, turnItem: runningItem }, |
| 1129 | }); |
| 1130 | |
| 1131 | const priorIdentity: ProviderEventRouteIdentity = { |
| 1132 | threadId: threadEvent.threadId, |
| 1133 | runId: priorRunId, |
| 1134 | attemptId: RunAttemptId.make("attempt:provider-event-completed:prior"), |
| 1135 | providerThreadId, |
| 1136 | }; |
| 1137 | const currentIdentity: ProviderEventRouteIdentity = { |
| 1138 | threadId: threadEvent.threadId, |
| 1139 | runId: currentRunId, |
| 1140 | attemptId: RunAttemptId.make("attempt:provider-event-completed:current"), |
| 1141 | providerThreadId, |
| 1142 | }; |
| 1143 | const inheritedBackgroundTurnItems = selectInheritedBackgroundTurnItems({ |
| 1144 | threadId: threadEvent.threadId, |
| 1145 | currentProviderThreadId: providerThreadId, |
| 1146 | currentRunOrdinal: 2, |
| 1147 | runs: [ |
| 1148 | { |
| 1149 | id: priorRunId, |
| 1150 | threadId: threadEvent.threadId, |
| 1151 | ordinal: 1, |
| 1152 | status: "completed", |
| 1153 | } as OrchestrationV2Run, |
| 1154 | { |
| 1155 | id: currentRunId, |
| 1156 | threadId: threadEvent.threadId, |
| 1157 | ordinal: 2, |
| 1158 | status: "running", |
| 1159 | } as OrchestrationV2Run, |
| 1160 | ], |
| 1161 | turnItems: [runningItem], |
| 1162 | }); |
| 1163 | const routers = [ |
| 1164 | { |
| 1165 | identity: priorIdentity, |
| 1166 | state: makeProviderEventRoutingState({ |
| 1167 | identity: priorIdentity, |
| 1168 | providerTurnId: providerTurnId, |
| 1169 | }), |
| 1170 | }, |
| 1171 | { |
| 1172 | identity: currentIdentity, |
| 1173 | state: makeProviderEventRoutingState({ |
| 1174 | identity: currentIdentity, |
| 1175 | inheritedBackgroundTurnItems, |
| 1176 | providerTurnId: null, |
| 1177 | }), |
| 1178 | }, |
| 1179 | ]; |
| 1180 | const acceptedRouters = routers.filter( |
| 1181 | ({ identity, state }) => routeProviderEvent(terminalEvent, identity, state)[0], |
| 1182 | ); |
| 1183 | |
| 1184 | yield* Effect.forEach( |
| 1185 | acceptedRouters, |
| 1186 | ({ identity }) => |
| 1187 | ingestor.ingestNormalized({ |
| 1188 | providerSessionId, |
| 1189 | providerInstanceId: modelSelection.instanceId, |
| 1190 | threadId: threadEvent.threadId, |
| 1191 | runId: identity.runId, |
| 1192 | event: terminalEvent, |
| 1193 | }), |
| 1194 | { concurrency: 1 }, |
| 1195 | ); |
| 1196 | |
| 1197 | const storedEvents = yield* eventStore |
| 1198 | .read({ threadId: threadEvent.threadId }) |
| 1199 | .pipe(Stream.runCollect); |
| 1200 | const storedTerminals = Array.from(storedEvents).filter( |
| 1201 | (stored) => |
| 1202 | stored.event.type === "turn-item.updated" && |
| 1203 | stored.event.payload.id === itemId && |
| 1204 | stored.event.payload.status === "completed", |
| 1205 | ); |
| 1206 | |
| 1207 | assert.equal(storedTerminals.length, 1); |
| 1208 | assert.equal(acceptedRouters.length, 1); |
| 1209 | assert.equal(acceptedRouters[0]?.identity.runId, priorRunId); |
| 1210 | }), |
| 1211 | ); |
| 1212 | |
| 1213 | it.effect.each(["interrupted", "control"] as const)( |
| 1214 | "dismisses only native questions when a provider turn ends with %s", |
| 1215 | (terminal) => |
| 1216 | Effect.gen(function* () { |
| 1217 | const now = yield* DateTime.now; |
| 1218 | const eventSink = yield* EventSink.EventSinkV2; |
| 1219 | const eventStore = yield* EventStore.EventStoreV2; |
| 1220 | const projectionStore = yield* ProjectionStore.ProjectionStoreV2; |
| 1221 | const ingestor = yield* ProviderEventIngestor.ProviderEventIngestorV2; |
| 1222 | const idAllocator = yield* IdAllocator.IdAllocatorV2; |
| 1223 | const threadEvent = yield* threadCreatedEvent(now); |
| 1224 | const threadId = threadEvent.threadId; |
| 1225 | const providerSessionId = yield* idAllocator.allocate.providerSession({ |
| 1226 | providerInstanceId: modelSelection.instanceId, |
| 1227 | threadId, |
| 1228 | }); |
| 1229 | const providerThreadId = idAllocator.derive.providerThread({ |
| 1230 | driver: CODEX_DRIVER, |
| 1231 | nativeThreadId: `${threadId}:questions`, |
| 1232 | }); |
| 1233 | const providerTurnId = idAllocator.derive.providerTurn({ |
| 1234 | driver: CODEX_DRIVER, |
| 1235 | nativeTurnId: `${threadId}:questions`, |
| 1236 | }); |
| 1237 | const otherTurnId = idAllocator.derive.providerTurn({ |
| 1238 | driver: CODEX_DRIVER, |
| 1239 | nativeTurnId: `${threadId}:other-turn`, |
| 1240 | }); |
| 1241 | const specs = [ |
| 1242 | { key: "native", responseCapability: { type: "live", providerSessionId } }, |
| 1243 | { |
| 1244 | key: "unavailable", |
| 1245 | responseCapability: { type: "not_resumable", reason: "Turn ended" }, |
| 1246 | }, |
| 1247 | { key: "message", responseCapability: { type: "message" } }, |
| 1248 | { key: "answered", responseCapability: { type: "live", providerSessionId } }, |
| 1249 | { key: "other-turn", responseCapability: { type: "live", providerSessionId } }, |
| 1250 | { key: "approval", responseCapability: { type: "live", providerSessionId } }, |
| 1251 | { key: "retained", responseCapability: { type: "live", providerSessionId } }, |
| 1252 | ] as const; |
| 1253 | const fixtures = specs.map((spec, ordinal) => { |
| 1254 | const nodeId = NodeId.make(`${threadId}:${spec.key}`); |
| 1255 | const resolved = spec.key === "answered"; |
| 1256 | const request: OrchestrationV2RuntimeRequest = { |
| 1257 | id: RuntimeRequestId.make(`${threadId}:${spec.key}`), |
| 1258 | nodeId, |
| 1259 | providerTurnId: spec.key === "other-turn" ? otherTurnId : providerTurnId, |
| 1260 | nativeRequestRef: null, |
| 1261 | kind: spec.key === "approval" ? "command" : "user_input", |
| 1262 | status: resolved ? "resolved" : "pending", |
| 1263 | responseCapability: spec.responseCapability, |
| 1264 | createdAt: now, |
| 1265 | resolvedAt: resolved ? now : null, |
| 1266 | }; |
| 1267 | const node: OrchestrationV2ExecutionNode = { |
| 1268 | id: nodeId, |
| 1269 | threadId, |
| 1270 | runId: null, |
| 1271 | parentNodeId: null, |
| 1272 | rootNodeId: nodeId, |
| 1273 | kind: spec.key === "approval" ? "approval_request" : "user_input_request", |
| 1274 | status: resolved ? "completed" : "waiting", |
| 1275 | countsForRun: false, |
| 1276 | providerThreadId, |
| 1277 | providerTurnId: request.providerTurnId, |
| 1278 | nativeItemRef: null, |
| 1279 | runtimeRequestId: request.id, |
| 1280 | checkpointScopeId: null, |
| 1281 | startedAt: now, |
| 1282 | completedAt: resolved ? now : null, |
| 1283 | }; |
| 1284 | const item: OrchestrationV2TurnItem = { |
| 1285 | id: TurnItemId.make(`${threadId}:${spec.key}`), |
| 1286 | threadId, |
| 1287 | runId: null, |
| 1288 | nodeId, |
| 1289 | providerThreadId, |
| 1290 | providerTurnId: request.providerTurnId, |
| 1291 | nativeItemRef: null, |
| 1292 | parentItemId: null, |
| 1293 | ordinal, |
| 1294 | status: resolved ? "completed" : "waiting", |
| 1295 | title: "Which option?", |
| 1296 | startedAt: now, |
| 1297 | completedAt: resolved ? now : null, |
| 1298 | updatedAt: now, |
| 1299 | ...(spec.key === "approval" |
| 1300 | ? { type: "approval_request", requestId: request.id, requestKind: "command" } |
| 1301 | : { |
| 1302 | type: "user_input_request", |
| 1303 | requestId: request.id, |
| 1304 | questions: [], |
| 1305 | ...(spec.key === "message" ? { responseMode: "message" as const } : {}), |
| 1306 | }), |
| 1307 | }; |
| 1308 | return { key: spec.key, request, node, item }; |
| 1309 | }); |
| 1310 | const seedEvents: Array<OrchestrationV2DomainEvent> = [threadEvent]; |
| 1311 | for (const fixture of fixtures) { |
| 1312 | for (const payload of [ |
| 1313 | { type: "runtime-request.updated" as const, payload: fixture.request }, |
| 1314 | { type: "node.updated" as const, payload: fixture.node }, |
| 1315 | { type: "turn-item.updated" as const, payload: fixture.item }, |
| 1316 | ]) { |
| 1317 | seedEvents.push({ |
| 1318 | id: yield* idAllocator.allocate.event({ threadId }), |
| 1319 | threadId, |
| 1320 | occurredAt: now, |
| 1321 | ...payload, |
| 1322 | }); |
| 1323 | } |
| 1324 | } |
| 1325 | yield* eventSink.write({ events: seedEvents }); |
| 1326 | const retainedRuntimeRequestIds = [RuntimeRequestId.make(`${threadId}:retained`)]; |
| 1327 | const input = { |
| 1328 | providerSessionId, |
| 1329 | providerInstanceId: modelSelection.instanceId, |
| 1330 | threadId, |
| 1331 | event: |
| 1332 | terminal === "control" |
| 1333 | ? { |
| 1334 | type: "turn.terminal" as const, |
| 1335 | driver: CODEX_DRIVER, |
| 1336 | providerThreadId, |
| 1337 | providerTurnId, |
| 1338 | runOrdinal: 1, |
| 1339 | status: "completed" as const, |
| 1340 | failure: null, |
| 1341 | threadDisposition: "reusable" as const, |
| 1342 | retainedRuntimeRequestIds, |
| 1343 | } |
| 1344 | : { |
| 1345 | type: "provider_turn.updated" as const, |
| 1346 | driver: CODEX_DRIVER, |
| 1347 | retainedRuntimeRequestIds, |
| 1348 | providerTurn: { |
| 1349 | id: providerTurnId, |
| 1350 | providerThreadId, |
| 1351 | nodeId: NodeId.make(`${threadId}:root`), |
| 1352 | runAttemptId: null, |
| 1353 | nativeTurnRef: null, |
| 1354 | ordinal: 1, |
| 1355 | status: terminal, |
| 1356 | startedAt: now, |
| 1357 | completedAt: now, |
| 1358 | }, |
| 1359 | }, |
| 1360 | }; |
| 1361 | const stored = yield* ingestor.ingestNormalized(input); |
| 1362 | const projection = yield* projectionStore.getThreadProjection(threadId); |
| 1363 | for (const fixture of fixtures) { |
| 1364 | const closed = fixture.key === "native" || fixture.key === "unavailable"; |
| 1365 | const request = projection.runtimeRequests.find( |
| 1366 | (item) => item.id === fixture.request.id, |
| 1367 | )!; |
| 1368 | const node = projection.nodes.find((item) => item.id === fixture.node.id)!; |
| 1369 | const item = projection.turnItems.find((item) => item.id === fixture.item.id)!; |
| 1370 | if (closed) { |
| 1371 | assert.equal(request.status, "cancelled"); |
| 1372 | assert.isNotNull(request.resolvedAt); |
| 1373 | assert.equal(node.status, "cancelled"); |
| 1374 | assert.isNotNull(node.completedAt); |
| 1375 | assert.equal(item.status, "cancelled"); |
| 1376 | assert.isNotNull(item.completedAt); |
| 1377 | } else { |
| 1378 | assert.equal(request.status, fixture.request.status); |
| 1379 | assert.equal(node.status, fixture.node.status); |
| 1380 | assert.equal(item.status, fixture.item.status); |
| 1381 | } |
| 1382 | } |
| 1383 | assert.equal( |
| 1384 | stored.filter((entry) => entry.event.type === "runtime-request.updated").length, |
| 1385 | 2, |
| 1386 | ); |
| 1387 | assert.deepEqual( |
| 1388 | (yield* projectionStore.getPendingNativeUserInputs( |
| 1389 | threadId, |
| 1390 | providerTurnId, |
| 1391 | )).runtimeRequests.map((request) => request.id), |
| 1392 | retainedRuntimeRequestIds, |
| 1393 | ); |
| 1394 | const replayed = yield* eventStore.read({ threadId }).pipe(Stream.runCollect); |
| 1395 | assert.equal( |
| 1396 | replayed.filter( |
| 1397 | (entry) => |
| 1398 | entry.event.type === "runtime-request.updated" && |
| 1399 | entry.event.payload.status === "cancelled", |
| 1400 | ).length, |
| 1401 | 2, |
| 1402 | ); |
| 1403 | const repeated = yield* ingestor.ingestNormalized(input); |
| 1404 | assert.isFalse(repeated.some((entry) => entry.event.type === "runtime-request.updated")); |
| 1405 | }), |
| 1406 | ); |
| 1407 | |
| 1408 | it.effect( |
| 1409 | "preserves an answer committed after terminal normalization reads a pending question", |
| 1410 | () => |
| 1411 | Effect.gen(function* () { |
| 1412 | const eventSink = yield* EventSink.EventSinkV2; |
| 1413 | const eventStore = yield* EventStore.EventStoreV2; |
| 1414 | const projections = yield* ProjectionStore.ProjectionStoreV2; |
| 1415 | const idAllocator = yield* IdAllocator.IdAllocatorV2; |
| 1416 | const now = yield* DateTime.now; |
| 1417 | const threadEvent = yield* threadCreatedEvent(now); |
| 1418 | const threadId = threadEvent.threadId; |
| 1419 | const providerSessionId = yield* idAllocator.allocate.providerSession({ |
| 1420 | providerInstanceId: modelSelection.instanceId, |
| 1421 | threadId, |
| 1422 | }); |
| 1423 | const providerThreadId = idAllocator.derive.providerThread({ |
| 1424 | driver: CODEX_DRIVER, |
| 1425 | nativeThreadId: `${threadId}:race`, |
| 1426 | }); |
| 1427 | const providerTurnId = idAllocator.derive.providerTurn({ |
| 1428 | driver: CODEX_DRIVER, |
| 1429 | nativeTurnId: `${threadId}:race`, |
| 1430 | }); |
| 1431 | const nodeId = NodeId.make(`${threadId}:question`); |
| 1432 | const request: OrchestrationV2RuntimeRequest = { |
| 1433 | id: RuntimeRequestId.make(`${threadId}:question`), |
| 1434 | nodeId, |
| 1435 | providerTurnId, |
| 1436 | nativeRequestRef: null, |
| 1437 | kind: "user_input", |
| 1438 | status: "pending", |
| 1439 | responseCapability: { type: "live", providerSessionId }, |
| 1440 | createdAt: now, |
| 1441 | resolvedAt: null, |
| 1442 | }; |
| 1443 | const node: OrchestrationV2ExecutionNode = { |
| 1444 | id: nodeId, |
| 1445 | threadId, |
| 1446 | runId: null, |
| 1447 | parentNodeId: null, |
| 1448 | rootNodeId: nodeId, |
| 1449 | kind: "user_input_request", |
| 1450 | status: "waiting", |
| 1451 | countsForRun: false, |
| 1452 | providerThreadId, |
| 1453 | providerTurnId, |
| 1454 | nativeItemRef: null, |
| 1455 | runtimeRequestId: request.id, |
| 1456 | checkpointScopeId: null, |
| 1457 | startedAt: now, |
| 1458 | completedAt: null, |
| 1459 | }; |
| 1460 | const item: OrchestrationV2TurnItem = { |
| 1461 | id: TurnItemId.make(`${threadId}:question`), |
| 1462 | threadId, |
| 1463 | runId: null, |
| 1464 | nodeId, |
| 1465 | providerThreadId, |
| 1466 | providerTurnId, |
| 1467 | nativeItemRef: null, |
| 1468 | parentItemId: null, |
| 1469 | ordinal: 1, |
| 1470 | status: "waiting", |
| 1471 | title: "Which option?", |
| 1472 | startedAt: now, |
| 1473 | completedAt: null, |
| 1474 | updatedAt: now, |
| 1475 | type: "user_input_request", |
| 1476 | requestId: request.id, |
| 1477 | questions: [], |
| 1478 | }; |
| 1479 | const seedEvents: Array<OrchestrationV2DomainEvent> = [threadEvent]; |
| 1480 | for (const payload of [ |
| 1481 | { type: "runtime-request.updated" as const, payload: request }, |
| 1482 | { type: "node.updated" as const, payload: node }, |
| 1483 | { type: "turn-item.updated" as const, payload: item }, |
| 1484 | ]) |
| 1485 | seedEvents.push({ |
| 1486 | id: yield* idAllocator.allocate.event({ threadId }), |
| 1487 | threadId, |
| 1488 | occurredAt: now, |
| 1489 | ...payload, |
| 1490 | }); |
| 1491 | yield* eventSink.write({ events: seedEvents }); |
| 1492 | const normalized = yield* Deferred.make<void>(); |
| 1493 | const releaseTerminalWrite = yield* Deferred.make<void>(); |
| 1494 | const gatedSink = EventSink.EventSinkV2.of({ |
| 1495 | ...eventSink, |
| 1496 | write: (input) => |
| 1497 | Deferred.succeed(normalized, undefined).pipe( |
| 1498 | Effect.andThen(Deferred.await(releaseTerminalWrite)), |
| 1499 | Effect.andThen(eventSink.write(input)), |
| 1500 | ), |
| 1501 | }); |
| 1502 | const ingestor = yield* ProviderEventIngestor.ProviderEventIngestorV2.pipe( |
| 1503 | Effect.provide(Layer.fresh(ProviderEventIngestor.layer)), |
| 1504 | Effect.provideService(EventSink.EventSinkV2, gatedSink), |
| 1505 | ); |
| 1506 | const terminal = yield* ingestor |
| 1507 | .ingestNormalized({ |
| 1508 | providerSessionId, |
| 1509 | providerInstanceId: modelSelection.instanceId, |
| 1510 | threadId, |
| 1511 | event: { |
| 1512 | type: "provider_turn.updated", |
| 1513 | driver: CODEX_DRIVER, |
| 1514 | providerTurn: { |
| 1515 | id: providerTurnId, |
| 1516 | providerThreadId, |
| 1517 | nodeId, |
| 1518 | runAttemptId: null, |
| 1519 | nativeTurnRef: null, |
| 1520 | ordinal: 1, |
| 1521 | status: "completed", |
| 1522 | startedAt: now, |
| 1523 | completedAt: now, |
| 1524 | }, |
| 1525 | }, |
| 1526 | }) |
| 1527 | .pipe(Effect.forkScoped); |
| 1528 | yield* Deferred.await(normalized); |
| 1529 | const answers = { decision: "Use the existing workspace" }; |
| 1530 | const responseEvents: Array<OrchestrationV2DomainEvent> = []; |
| 1531 | for (const payload of [ |
| 1532 | { |
| 1533 | type: "runtime-request.updated" as const, |
| 1534 | payload: { ...request, status: "resolved" as const, answers, resolvedAt: now }, |
| 1535 | }, |
| 1536 | { |
| 1537 | type: "node.updated" as const, |
| 1538 | payload: { ...node, status: "completed" as const, completedAt: now }, |
| 1539 | }, |
| 1540 | { |
| 1541 | type: "turn-item.updated" as const, |
| 1542 | payload: { ...item, status: "completed" as const, completedAt: now }, |
| 1543 | }, |
| 1544 | ]) |
| 1545 | responseEvents.push({ |
| 1546 | id: yield* idAllocator.allocate.event({ threadId }), |
| 1547 | threadId, |
| 1548 | occurredAt: now, |
| 1549 | ...payload, |
| 1550 | }); |
| 1551 | yield* eventSink.write({ events: responseEvents }); |
| 1552 | yield* Deferred.succeed(releaseTerminalWrite, undefined); |
| 1553 | const committedTerminal = yield* Fiber.join(terminal); |
| 1554 | assert.deepEqual( |
| 1555 | committedTerminal.map((stored) => stored.event.type), |
| 1556 | ["provider-turn.updated"], |
| 1557 | ); |
| 1558 | const projection = yield* projections.getThreadProjection(threadId); |
| 1559 | const answered = projection.runtimeRequests.find((entry) => entry.id === request.id)!; |
| 1560 | assert.equal(answered.status, "resolved"); |
| 1561 | assert.deepEqual(answered.answers, answers); |
| 1562 | assert.equal(projection.nodes.find((entry) => entry.id === nodeId)!.status, "completed"); |
| 1563 | assert.equal( |
| 1564 | projection.turnItems.find((entry) => entry.id === item.id)!.status, |
| 1565 | "completed", |
| 1566 | ); |
| 1567 | const history = yield* eventStore.read({ threadId }).pipe(Stream.runCollect); |
| 1568 | assert.isFalse( |
| 1569 | history.some( |
| 1570 | (stored) => |
| 1571 | stored.event.type === "runtime-request.updated" && |
| 1572 | stored.event.payload.status === "cancelled", |
| 1573 | ), |
| 1574 | ); |
| 1575 | }), |
| 1576 | ); |
| 1577 | |
| 1578 | it.effect("persists a failed provider terminal as one expected error item", () => |
| 1579 | Effect.gen(function* () { |
| 1580 | const now = yield* DateTime.now; |
| 1581 | const retryStartedAt = DateTime.makeUnsafe(DateTime.toEpochMillis(now) - 5_000); |
| 1582 | const eventSink = yield* EventSink.EventSinkV2; |
| 1583 | const projectionStore = yield* ProjectionStore.ProjectionStoreV2; |
| 1584 | const ingestor = yield* ProviderEventIngestor.ProviderEventIngestorV2; |
| 1585 | const idAllocator = yield* IdAllocator.IdAllocatorV2; |
| 1586 | const threadEvent = yield* threadCreatedEvent(now); |
| 1587 | const providerSessionId = yield* idAllocator.allocate.providerSession({ |
| 1588 | providerInstanceId: modelSelection.instanceId, |
| 1589 | threadId: threadEvent.threadId, |
| 1590 | }); |
| 1591 | const providerThreadId = idAllocator.derive.providerThread({ |
| 1592 | driver: CODEX_DRIVER, |
| 1593 | nativeThreadId: "native-thread-failed", |
| 1594 | }); |
| 1595 | const providerTurnId = idAllocator.derive.providerTurn({ |
| 1596 | driver: CODEX_DRIVER, |
| 1597 | nativeTurnId: "native-turn-failed", |
| 1598 | }); |
| 1599 | |
| 1600 | yield* eventSink.write({ events: [threadEvent] }); |
| 1601 | const stored = yield* ingestor.ingestNormalized({ |
| 1602 | providerSessionId, |
| 1603 | providerInstanceId: modelSelection.instanceId, |
| 1604 | threadId: threadEvent.threadId, |
| 1605 | event: { |
| 1606 | type: "turn.terminal", |
| 1607 | driver: CODEX_DRIVER, |
| 1608 | providerThreadId, |
| 1609 | providerTurnId, |
| 1610 | runOrdinal: 1, |
| 1611 | failureItemOrdinal: 102, |
| 1612 | status: "failed", |
| 1613 | failure: makeProviderFailure({ |
| 1614 | message: "Invalid reasoning effort.", |
| 1615 | code: "invalid_request", |
| 1616 | class: "validation_error", |
| 1617 | }), |
| 1618 | retry: { |
| 1619 | attempt: 3, |
| 1620 | maxAttempts: 3, |
| 1621 | retryDelayMs: 2_000, |
| 1622 | }, |
| 1623 | retryStartedAt, |
| 1624 | threadDisposition: "reusable", |
| 1625 | }, |
| 1626 | }); |
| 1627 | |
| 1628 | const projection = yield* projectionStore.getThreadProjection(threadEvent.threadId); |
| 1629 | const errorItems = projection.visibleTurnItems.filter( |
| 1630 | (candidate) => candidate.item.type === "error", |
| 1631 | ); |
| 1632 | |
| 1633 | assert.equal(stored.length, 1); |
| 1634 | assert.equal(stored[0]?.event.type, "turn-item.updated"); |
| 1635 | assert.equal(errorItems.length, 1); |
| 1636 | const errorItem = errorItems[0]?.item; |
| 1637 | assert.equal(errorItem?.type, "error"); |
| 1638 | if (errorItem?.type !== "error") return; |
| 1639 | assert.equal(errorItem.failure.message, "Invalid reasoning effort."); |
| 1640 | assert.equal(errorItem.failure.code, "invalid_request"); |
| 1641 | assert.deepEqual(errorItem.retry, { |
| 1642 | attempt: 3, |
| 1643 | maxAttempts: 3, |
| 1644 | retryDelayMs: 2_000, |
| 1645 | }); |
| 1646 | const errorStartedAt = errorItem.startedAt; |
| 1647 | assert.ok(errorStartedAt); |
| 1648 | assert.equal(DateTime.toEpochMillis(errorStartedAt), DateTime.toEpochMillis(retryStartedAt)); |
| 1649 | assert.equal(errorItem.providerThreadId, providerThreadId); |
| 1650 | assert.equal(errorItem.providerTurnId, providerTurnId); |
| 1651 | }), |
| 1652 | ); |
| 1653 | |
| 1654 | it.effect("stores tool image bytes only where a tool-output-image asset serves them", () => |
| 1655 | Effect.gen(function* () { |
| 1656 | const now = yield* DateTime.now; |
| 1657 | const eventSink = yield* EventSink.EventSinkV2; |
| 1658 | const eventStore = yield* EventStore.EventStoreV2; |
| 1659 | const projectionStore = yield* ProjectionStore.ProjectionStoreV2; |
| 1660 | const ingestor = yield* ProviderEventIngestor.ProviderEventIngestorV2; |
| 1661 | const idAllocator = yield* IdAllocator.IdAllocatorV2; |
| 1662 | const threadEvent = yield* threadCreatedEvent(now); |
| 1663 | const providerSessionId = yield* idAllocator.allocate.providerSession({ |
| 1664 | providerInstanceId: modelSelection.instanceId, |
| 1665 | threadId: threadEvent.threadId, |
| 1666 | }); |
| 1667 | const readBase64 = Buffer.alloc(30_000, 7).toString("base64"); |
| 1668 | const screenshotBase64 = Buffer.alloc(20_000, 9).toString("base64"); |
| 1669 | const toolItem = ( |
| 1670 | id: string, |
| 1671 | ordinal: number, |
| 1672 | toolName: string, |
| 1673 | output: unknown, |
| 1674 | ): OrchestrationV2TurnItem => ({ |
| 1675 | id: TurnItemId.make(id), |
| 1676 | threadId: threadEvent.threadId, |
| 1677 | runId: null, |
| 1678 | nodeId: null, |
| 1679 | providerThreadId: null, |
| 1680 | providerTurnId: null, |
| 1681 | nativeItemRef: null, |
| 1682 | parentItemId: null, |
| 1683 | ordinal, |
| 1684 | status: "completed", |
| 1685 | title: toolName, |
| 1686 | startedAt: now, |
| 1687 | completedAt: now, |
| 1688 | updatedAt: now, |
| 1689 | type: "dynamic_tool", |
| 1690 | toolName, |
| 1691 | input: {}, |
| 1692 | output, |
| 1693 | }); |
| 1694 | const read = toolItem("turn-item:read-image", 1, "Read", { |
| 1695 | type: "image", |
| 1696 | file: { base64: readBase64, type: "image/png", originalSize: 30_000 }, |
| 1697 | }); |
| 1698 | const screenshot = toolItem("turn-item:screenshot", 2, "mcp__t3-code__device_screenshot", { |
| 1699 | content: [ |
| 1700 | { |
| 1701 | type: "image", |
| 1702 | source: { type: "base64", media_type: "image/png", data: screenshotBase64 }, |
| 1703 | }, |
| 1704 | ], |
| 1705 | }); |
| 1706 | |
| 1707 | yield* eventSink.write({ events: [threadEvent] }); |
| 1708 | for (const turnItem of [read, screenshot]) { |
| 1709 | yield* ingestor.ingestNormalized({ |
| 1710 | providerSessionId, |
| 1711 | providerInstanceId: modelSelection.instanceId, |
| 1712 | threadId: threadEvent.threadId, |
| 1713 | event: { type: "turn_item.updated", driver: CODEX_DRIVER, turnItem }, |
| 1714 | }); |
| 1715 | } |
| 1716 | |
| 1717 | const storedEvents = yield* eventStore |
| 1718 | .read({ threadId: threadEvent.threadId, eventType: "turn-item.updated" }) |
| 1719 | .pipe(Stream.runCollect); |
| 1720 | const storedJson = JSON.stringify(Array.from(storedEvents, (stored) => stored.event)); |
| 1721 | const projectedRead = yield* projectionStore.getTurnItem({ |
| 1722 | threadId: threadEvent.threadId, |
| 1723 | itemId: read.id, |
| 1724 | }); |
| 1725 | const projectedScreenshot = yield* projectionStore.getTurnItem({ |
| 1726 | threadId: threadEvent.threadId, |
| 1727 | itemId: screenshot.id, |
| 1728 | }); |
| 1729 | |
| 1730 | assert.equal(storedJson.includes(readBase64), false); |
| 1731 | assert.equal(storedJson.includes(screenshotBase64), true); |
| 1732 | assert.deepEqual(projectedRead?.type === "dynamic_tool" ? projectedRead.output : null, { |
| 1733 | type: "image", |
| 1734 | file: { type: "image/png", originalSize: 30_000, sizeBytes: 30_000 }, |
| 1735 | }); |
| 1736 | assert.deepEqual( |
| 1737 | toolOutputImages( |
| 1738 | projectedScreenshot?.type === "dynamic_tool" ? projectedScreenshot.output : null, |
| 1739 | ), |
| 1740 | [{ mimeType: "image/png", data: screenshotBase64 }], |
| 1741 | ); |
| 1742 | }), |
| 1743 | ); |
| 1744 | |
| 1745 | it.effect("keeps a steered message pending until the provider reports it read", () => |
| 1746 | Effect.gen(function* () { |
| 1747 | const now = yield* DateTime.now; |
| 1748 | const eventSink = yield* EventSink.EventSinkV2; |
| 1749 | const projectionStore = yield* ProjectionStore.ProjectionStoreV2; |
| 1750 | const ingestor = yield* ProviderEventIngestor.ProviderEventIngestorV2; |
| 1751 | const idAllocator = yield* IdAllocator.IdAllocatorV2; |
| 1752 | const threadEvent = yield* threadCreatedEvent(now); |
| 1753 | const providerSessionId = yield* idAllocator.allocate.providerSession({ |
| 1754 | providerInstanceId: modelSelection.instanceId, |
| 1755 | threadId: threadEvent.threadId, |
| 1756 | }); |
| 1757 | const runId = RunId.make("run:steer-read"); |
| 1758 | const messageId = MessageId.make("message:steer-read"); |
| 1759 | const steer: OrchestrationV2TurnItem = { |
| 1760 | id: idAllocator.derive.userTurnItem({ messageId }), |
| 1761 | threadId: threadEvent.threadId, |
| 1762 | runId, |
| 1763 | nodeId: null, |
| 1764 | providerThreadId: null, |
| 1765 | providerTurnId: null, |
| 1766 | nativeItemRef: null, |
| 1767 | parentItemId: null, |
| 1768 | ordinal: 1, |
| 1769 | status: "completed", |
| 1770 | title: null, |
| 1771 | startedAt: now, |
| 1772 | completedAt: now, |
| 1773 | updatedAt: now, |
| 1774 | type: "user_message", |
| 1775 | createdBy: "user", |
| 1776 | creationSource: "web", |
| 1777 | messageId, |
| 1778 | inputIntent: "steer", |
| 1779 | text: "Steer behind a command", |
| 1780 | attachments: [], |
| 1781 | }; |
| 1782 | yield* eventSink.write({ events: [threadEvent] }); |
| 1783 | const ingest = (event: ProviderAdapter.ProviderAdapterV2Event) => |
| 1784 | ingestor.ingestNormalized({ |
| 1785 | providerSessionId, |
| 1786 | providerInstanceId: modelSelection.instanceId, |
| 1787 | threadId: threadEvent.threadId, |
| 1788 | event, |
| 1789 | }); |
| 1790 | const projected = projectionStore.getTurnItem({ |
| 1791 | threadId: threadEvent.threadId, |
| 1792 | itemId: steer.id, |
| 1793 | }); |
| 1794 | const steerUpdated = (status: "pending" | "completed") => |
| 1795 | ingest({ |
| 1796 | type: "steer.updated", |
| 1797 | driver: CODEX_DRIVER, |
| 1798 | threadId: threadEvent.threadId, |
| 1799 | runId, |
| 1800 | messageId, |
| 1801 | status, |
| 1802 | }); |
| 1803 | |
| 1804 | yield* ingest({ type: "turn_item.updated", driver: CODEX_DRIVER, turnItem: steer }); |
| 1805 | yield* steerUpdated("pending"); |
| 1806 | assert.deepInclude(yield* projected, { status: "pending", completedAt: null }); |
| 1807 | yield* steerUpdated("completed"); |
| 1808 | assert.equal((yield* projected)?.status, "completed"); |
| 1809 | }), |
| 1810 | ); |
| 1811 | |
| 1812 | it.effect("stores a bounded copy of a large file-change diff in both SQLite tables", () => |
| 1813 | Effect.gen(function* () { |
| 1814 | const now = yield* DateTime.now; |
| 1815 | const eventSink = yield* EventSink.EventSinkV2; |
| 1816 | const projectionStore = yield* ProjectionStore.ProjectionStoreV2; |
| 1817 | const ingestor = yield* ProviderEventIngestor.ProviderEventIngestorV2; |
| 1818 | const idAllocator = yield* IdAllocator.IdAllocatorV2; |
| 1819 | const threadEvent = yield* threadCreatedEvent(now); |
| 1820 | const providerSessionId = yield* idAllocator.allocate.providerSession({ |
| 1821 | providerInstanceId: modelSelection.instanceId, |
| 1822 | threadId: threadEvent.threadId, |
| 1823 | }); |
| 1824 | const largeDiff = `@@ -0,0 +1 @@\n${"+generated line\n".repeat(150_000)}`; |
| 1825 | const fileChange = ( |
| 1826 | id: string, |
| 1827 | ordinal: number, |
| 1828 | diffStr: string, |
| 1829 | ): OrchestrationV2TurnItem => ({ |
| 1830 | id: TurnItemId.make(id), |
| 1831 | threadId: threadEvent.threadId, |
| 1832 | runId: null, |
| 1833 | nodeId: null, |
| 1834 | providerThreadId: null, |
| 1835 | providerTurnId: null, |
| 1836 | nativeItemRef: null, |
| 1837 | parentItemId: null, |
| 1838 | ordinal, |
| 1839 | status: "completed", |
| 1840 | title: null, |
| 1841 | startedAt: now, |
| 1842 | completedAt: now, |
| 1843 | updatedAt: now, |
| 1844 | type: "file_change", |
| 1845 | fileName: "src/generated.ts", |
| 1846 | diffStr, |
| 1847 | }); |
| 1848 | const large = fileChange("turn-item:large-diff", 1, largeDiff); |
| 1849 | const small = fileChange("turn-item:small-diff", 2, "@@ -1 +1 @@\n-old\n+new\n"); |
| 1850 | |
| 1851 | yield* eventSink.write({ events: [threadEvent] }); |
| 1852 | for (const turnItem of [large, small]) { |
| 1853 | yield* ingestor.ingestNormalized({ |
| 1854 | providerSessionId, |
| 1855 | providerInstanceId: modelSelection.instanceId, |
| 1856 | threadId: threadEvent.threadId, |
| 1857 | event: { type: "turn_item.updated", driver: CODEX_DRIVER, turnItem }, |
| 1858 | }); |
| 1859 | } |
| 1860 | |
| 1861 | const sql = yield* SqlClient.SqlClient; |
| 1862 | const storedBytes = yield* sql<{ readonly events: number; readonly projection: number }>` |
| 1863 | SELECT |
| 1864 | (SELECT SUM(LENGTH(payload_json)) FROM orchestration_events |
| 1865 | WHERE stream_id = ${threadEvent.threadId} AND event_type = 'turn-item.updated') AS events, |
| 1866 | (SELECT SUM(LENGTH(payload_json)) FROM orchestration_v2_projection_turn_items |
| 1867 | WHERE thread_id = ${threadEvent.threadId}) AS projection |
| 1868 | `; |
| 1869 | const projectedLarge = yield* projectionStore.getTurnItem({ |
| 1870 | threadId: threadEvent.threadId, |
| 1871 | itemId: large.id, |
| 1872 | }); |
| 1873 | const projectedSmall = yield* projectionStore.getTurnItem({ |
| 1874 | threadId: threadEvent.threadId, |
| 1875 | itemId: small.id, |
| 1876 | }); |
| 1877 | |
| 1878 | assert.isAbove(largeDiff.length, 2_000_000); |
| 1879 | assert.isBelow(storedBytes[0]!.events, 100_000); |
| 1880 | assert.isBelow(storedBytes[0]!.projection, 100_000); |
| 1881 | assert.isTrue(projectedLarge?.type === "file_change"); |
| 1882 | if (projectedLarge?.type !== "file_change") return; |
| 1883 | assert.equal(projectedLarge.fileName, "src/generated.ts"); |
| 1884 | assert.isBelow(projectedLarge.diffStr?.length ?? 0, 40_000); |
| 1885 | assert.isTrue(largeDiff.startsWith(projectedLarge.diffStr?.split("\n…")[0] ?? "-")); |
| 1886 | assert.deepEqual(projectedSmall, small); |
| 1887 | }), |
| 1888 | ); |
| 1889 | |
| 1890 | it.effect("routes provider-owned child artifacts to their child app thread", () => |
| 1891 | Effect.gen(function* () { |
| 1892 | const now = yield* DateTime.now; |
| 1893 | const ingestor = yield* ProviderEventIngestor.ProviderEventIngestorV2; |
| 1894 | const idAllocator = yield* IdAllocator.IdAllocatorV2; |
| 1895 | const rootEvent = yield* threadCreatedEvent(now); |
| 1896 | if (rootEvent.type !== "thread.created") { |
| 1897 | throw new Error("Expected a thread.created fixture event"); |
| 1898 | } |
| 1899 | const childThreadId = idAllocator.derive.threadFromProviderThread({ |
| 1900 | driver: CODEX_DRIVER, |
| 1901 | nativeThreadId: "native-subagent-thread", |
| 1902 | }); |
| 1903 | const childRootNodeId = NodeId.make("node:subagent-root"); |
| 1904 | const childThread: OrchestrationV2AppThread = { |
| 1905 | ...rootEvent.payload, |
| 1906 | id: childThreadId, |
| 1907 | title: "inspect package", |
| 1908 | activeProviderThreadId: null, |
| 1909 | lineage: { |
| 1910 | parentThreadId: rootEvent.threadId, |
| 1911 | relationshipToParent: "subagent", |
| 1912 | rootThreadId: rootEvent.threadId, |
| 1913 | }, |
| 1914 | forkedFrom: { |
| 1915 | type: "node", |
| 1916 | nodeId: NodeId.make("node:parent-subagent"), |
| 1917 | }, |
| 1918 | }; |
| 1919 | const providerSessionId = yield* idAllocator.allocate.providerSession({ |
| 1920 | providerInstanceId: modelSelection.instanceId, |
| 1921 | threadId: rootEvent.threadId, |
| 1922 | }); |
| 1923 | |
| 1924 | const failedSpans: Array<string> = []; |
| 1925 | const tracer = Tracer.make({ |
| 1926 | span(options) { |
| 1927 | const span = new Tracer.NativeSpan(options); |
| 1928 | const end = span.end.bind(span); |
| 1929 | span.end = (time, exit) => { |
| 1930 | end(time, exit); |
| 1931 | if (Exit.isFailure(exit)) failedSpans.push(span.name); |
| 1932 | }; |
| 1933 | return span; |
| 1934 | }, |
| 1935 | }); |
| 1936 | const threadEvents = yield* ingestor |
| 1937 | .normalize({ |
| 1938 | providerSessionId, |
| 1939 | providerInstanceId: modelSelection.instanceId, |
| 1940 | threadId: rootEvent.threadId, |
| 1941 | event: { |
| 1942 | type: "app_thread.created", |
| 1943 | driver: CODEX_DRIVER, |
| 1944 | appThread: childThread, |
| 1945 | }, |
| 1946 | }) |
| 1947 | .pipe(Effect.withTracer(tracer)); |
| 1948 | // A new child's existence check expects a miss; a failed span there reads as an error in Tempo. |
| 1949 | assert.deepEqual(failedSpans, []); |
| 1950 | const messageEvents = yield* ingestor.normalize({ |
| 1951 | providerSessionId, |
| 1952 | providerInstanceId: modelSelection.instanceId, |
| 1953 | threadId: rootEvent.threadId, |
| 1954 | event: { |
| 1955 | type: "message.updated", |
| 1956 | driver: CODEX_DRIVER, |
| 1957 | message: { |
| 1958 | createdBy: "agent", |
| 1959 | creationSource: "provider", |
| 1960 | id: MessageId.make("message:subagent-response"), |
| 1961 | threadId: childThreadId, |
| 1962 | runId: null, |
| 1963 | nodeId: childRootNodeId, |
| 1964 | role: "assistant", |
| 1965 | text: "Subagent result", |
| 1966 | attachments: [], |
| 1967 | streaming: false, |
| 1968 | createdAt: now, |
| 1969 | updatedAt: now, |
| 1970 | }, |
| 1971 | }, |
| 1972 | }); |
| 1973 | |
| 1974 | assert.equal(threadEvents[0]?.type, "thread.created"); |
| 1975 | assert.equal(threadEvents[0]?.threadId, childThreadId); |
| 1976 | assert.equal(messageEvents[0]?.type, "message.updated"); |
| 1977 | assert.equal(messageEvents[0]?.threadId, childThreadId); |
| 1978 | }), |
| 1979 | ); |
| 1980 | |
| 1981 | it.effect.each([ |
| 1982 | { parentMoved: false, failure: null, temporaryBranch: "t3/abcd1234" }, |
| 1983 | { parentMoved: true, failure: null, temporaryBranch: "t3/abcd1234" }, |
| 1984 | { parentMoved: false, failure: "write", temporaryBranch: "t3/abcd1234" }, |
| 1985 | { parentMoved: false, failure: "git", temporaryBranch: "t3/abcd1234" }, |
| 1986 | { parentMoved: false, failure: null, temporaryBranch: "my-prefix/abcd1234" }, |
| 1987 | ])( |
| 1988 | "rechecks a pending native child's branch after the rename scan: %j", |
| 1989 | ({ parentMoved, failure, temporaryBranch }) => |
| 1990 | Effect.gen(function* () { |
| 1991 | const { worktreePath, command } = yield* makeSharedWorktree; |
| 1992 | if (temporaryBranch !== "t3/abcd1234") |
| 1993 | yield* command(worktreePath, ["branch", "-m", temporaryBranch]); |
| 1994 | const git = yield* GitVcsDriver.GitVcsDriver; |
| 1995 | const now = yield* DateTime.now; |
| 1996 | const eventSink = yield* EventSink.EventSinkV2; |
| 1997 | const projections = yield* ProjectionStore.ProjectionStoreV2; |
| 1998 | const ids = yield* IdAllocator.IdAllocatorV2; |
| 1999 | const rootEvent = yield* threadCreatedEvent(now); |
| 2000 | if (rootEvent.type !== "thread.created") throw new Error("Expected thread fixture"); |
| 2001 | const parent = { |
| 2002 | ...rootEvent.payload, |
| 2003 | branch: temporaryBranch, |
| 2004 | worktreePath, |
| 2005 | }; |
| 2006 | yield* eventSink.write({ events: [{ ...rootEvent, payload: parent }] }); |
| 2007 | const childId = ids.derive.threadFromProviderThread({ |
| 2008 | driver: CODEX_DRIVER, |
| 2009 | nativeThreadId: `child-created-during-rename:${parentMoved}:${failure}:${temporaryBranch}`, |
| 2010 | }); |
| 2011 | const beforeCommit = yield* Deferred.make< |
| 2012 | void, |
| 2013 | ProviderEventIngestor.ProviderEventIngestorV2Error |
| 2014 | >(); |
| 2015 | const allowCommit = yield* Deferred.make<void>(); |
| 2016 | let correctionWrites = 0; |
| 2017 | const gatedSink = EventSink.EventSinkV2.of({ |
| 2018 | ...eventSink, |
| 2019 | write: (input) => { |
| 2020 | if ( |
| 2021 | input.events.some( |
| 2022 | (event) => event.type === "thread.metadata-updated" && event.threadId === childId, |
| 2023 | ) |
| 2024 | ) { |
| 2025 | correctionWrites++; |
| 2026 | if (failure === "write" && correctionWrites === 1) |
| 2027 | return Effect.fail(new EventSink.EventSinkWriteError({ eventCount: 1 })); |
| 2028 | } |
| 2029 | return input.events.some( |
| 2030 | (event) => event.type === "thread.created" && event.threadId === childId, |
| 2031 | ) |
| 2032 | ? Deferred.succeed(beforeCommit, undefined).pipe( |
| 2033 | Effect.andThen(Deferred.await(allowCommit)), |
| 2034 | Effect.andThen(eventSink.write(input)), |
| 2035 | ) |
| 2036 | : eventSink.write(input); |
| 2037 | }, |
| 2038 | }); |
| 2039 | const ingestor = yield* ProviderEventIngestor.ProviderEventIngestorV2.pipe( |
| 2040 | Effect.provide(Layer.fresh(ProviderEventIngestor.layer)), |
| 2041 | Effect.provideService(EventSink.EventSinkV2, gatedSink), |
| 2042 | Effect.provideService(GitVcsDriver.GitVcsDriver, { |
| 2043 | ...git, |
| 2044 | resolveBranchName: (cwd, refName) => |
| 2045 | failure === "git" |
| 2046 | ? Effect.fail( |
| 2047 | new GitCommandError({ |
| 2048 | operation: "ingestor-test", |
| 2049 | cwd, |
| 2050 | command: "rev-parse", |
| 2051 | detail: "Read failed", |
| 2052 | }), |
| 2053 | ) |
| 2054 | : git.resolveBranchName(cwd, refName), |
| 2055 | }), |
| 2056 | ); |
| 2057 | const providerSessionId = yield* ids.allocate.providerSession({ |
| 2058 | providerInstanceId: modelSelection.instanceId, |
| 2059 | threadId: parent.id, |
| 2060 | }); |
| 2061 | const creationInput = { |
| 2062 | providerSessionId, |
| 2063 | providerInstanceId: modelSelection.instanceId, |
| 2064 | threadId: parent.id, |
| 2065 | event: { |
| 2066 | type: "app_thread.created", |
| 2067 | driver: CODEX_DRIVER, |
| 2068 | appThread: { |
| 2069 | ...parent, |
| 2070 | id: childId, |
| 2071 | lineage: { |
| 2072 | parentThreadId: parent.id, |
| 2073 | rootThreadId: parent.id, |
| 2074 | relationshipToParent: "subagent", |
| 2075 | }, |
| 2076 | }, |
| 2077 | }, |
| 2078 | } satisfies ProviderEventIngestor.ProviderEventIngestInput; |
| 2079 | const creation = yield* ingestor.ingestNormalized(creationInput).pipe( |
| 2080 | Effect.tapCause((cause) => Deferred.failCause(beforeCommit, cause)), |
| 2081 | Effect.forkScoped, |
| 2082 | ); |
| 2083 | yield* Deferred.await(beforeCommit); |
| 2084 | yield* command(worktreePath, ["branch", "-m", "generated-branch"]); |
| 2085 | yield* eventSink.write({ |
| 2086 | events: [ |
| 2087 | { |
| 2088 | ...rootEvent, |
| 2089 | id: yield* ids.allocate.event({ threadId: parent.id }), |
| 2090 | type: "thread.metadata-updated", |
| 2091 | payload: { |
| 2092 | ...parent, |
| 2093 | branch: parentMoved ? "user/checkout" : "generated-branch", |
| 2094 | worktreePath: parentMoved ? `${worktreePath}-moved` : worktreePath, |
| 2095 | }, |
| 2096 | }, |
| 2097 | ], |
| 2098 | }); |
| 2099 | const snapshot = yield* projections.getShellSnapshot(); |
| 2100 | assert.isFalse( |
| 2101 | [...snapshot.threads, ...snapshot.archivedThreads].some( |
| 2102 | (thread) => |
| 2103 | thread.worktreePath === parent.worktreePath && thread.branch === parent.branch, |
| 2104 | ), |
| 2105 | ); |
| 2106 | yield* Deferred.succeed(allowCommit, undefined); |
| 2107 | const stored = yield* Fiber.join(creation); |
| 2108 | assert.equal(stored[0]?.event.type, "thread.created"); |
| 2109 | const child = yield* projections.getThread(childId); |
| 2110 | assert.equal(child.branch, failure === null ? "generated-branch" : parent.branch); |
| 2111 | assert.equal(child.worktreePath, parent.worktreePath); |
| 2112 | assert.equal(correctionWrites, failure === "git" ? 0 : 1); |
| 2113 | assert.deepEqual(yield* ingestor.ingestNormalized(creationInput), []); |
| 2114 | }), |
| 2115 | ); |
| 2116 | |
| 2117 | // regression: #20079 the renamed-branch correction must not rewrite a child that needs none |
| 2118 | it.effect.each(["explicit", "detached"])( |
| 2119 | "preserves a native child's branch when the checkout needs no correction: %s", |
| 2120 | (scenario) => |
| 2121 | Effect.gen(function* () { |
| 2122 | const { worktreePath, command } = yield* makeSharedWorktree; |
| 2123 | const git = yield* GitVcsDriver.GitVcsDriver; |
| 2124 | const branch = scenario === "explicit" ? "child-feature" : "t3/abcd1234"; |
| 2125 | if (scenario === "explicit") yield* command(worktreePath, ["branch", "-m", branch]); |
| 2126 | if (scenario === "detached") yield* command(worktreePath, ["checkout", "--detach"]); |
| 2127 | const now = yield* DateTime.now; |
| 2128 | const sink = yield* EventSink.EventSinkV2; |
| 2129 | const projections = yield* ProjectionStore.ProjectionStoreV2; |
| 2130 | const ids = yield* IdAllocator.IdAllocatorV2; |
| 2131 | const root = yield* threadCreatedEvent(now); |
| 2132 | if (root.type !== "thread.created") throw new Error("Expected thread fixture"); |
| 2133 | yield* sink.write({ |
| 2134 | events: [ |
| 2135 | { ...root, payload: { ...root.payload, branch: "parent-feature", worktreePath } }, |
| 2136 | ], |
| 2137 | }); |
| 2138 | const childId = ids.derive.threadFromProviderThread({ |
| 2139 | driver: CODEX_DRIVER, |
| 2140 | nativeThreadId: `no-correction:${scenario}`, |
| 2141 | }); |
| 2142 | let gitReads = 0; |
| 2143 | const ingestor = yield* ProviderEventIngestor.ProviderEventIngestorV2.pipe( |
| 2144 | Effect.provide(Layer.fresh(ProviderEventIngestor.layer)), |
| 2145 | Effect.provideService(GitVcsDriver.GitVcsDriver, { |
| 2146 | ...git, |
| 2147 | resolveBranchName: (cwd, refName) => { |
| 2148 | gitReads++; |
| 2149 | return git.resolveBranchName(cwd, refName); |
| 2150 | }, |
| 2151 | }), |
| 2152 | ); |
| 2153 | const stored = yield* ingestor.ingestNormalized({ |
| 2154 | providerSessionId: yield* ids.allocate.providerSession({ |
| 2155 | providerInstanceId: modelSelection.instanceId, |
| 2156 | threadId: root.threadId, |
| 2157 | }), |
| 2158 | providerInstanceId: modelSelection.instanceId, |
| 2159 | threadId: root.threadId, |
| 2160 | event: { |
| 2161 | type: "app_thread.created", |
| 2162 | driver: CODEX_DRIVER, |
| 2163 | appThread: { |
| 2164 | ...root.payload, |
| 2165 | id: childId, |
| 2166 | branch, |
| 2167 | worktreePath, |
| 2168 | lineage: { |
| 2169 | parentThreadId: root.threadId, |
| 2170 | rootThreadId: root.threadId, |
| 2171 | relationshipToParent: "subagent", |
| 2172 | }, |
| 2173 | }, |
| 2174 | }, |
| 2175 | }); |
| 2176 | assert.equal((yield* projections.getThread(childId)).branch, branch); |
| 2177 | assert.equal(stored.length, 1); |
| 2178 | assert.equal(gitReads, 1); |
| 2179 | }), |
| 2180 | ); |
| 2181 | |
| 2182 | it.effect("updates provider child metadata without recreating its saved thread", () => |
| 2183 | Effect.gen(function* () { |
| 2184 | const now = yield* DateTime.now; |
| 2185 | const eventSink = yield* EventSink.EventSinkV2; |
| 2186 | const projectionStore = yield* ProjectionStore.ProjectionStoreV2; |
| 2187 | const ingestor = yield* ProviderEventIngestor.ProviderEventIngestorV2; |
| 2188 | const idAllocator = yield* IdAllocator.IdAllocatorV2; |
| 2189 | const rootEvent = yield* threadCreatedEvent(now); |
| 2190 | if (rootEvent.type !== "thread.created") throw new Error("Expected thread fixture"); |
| 2191 | const childThreadId = idAllocator.derive.threadFromProviderThread({ |
| 2192 | driver: CODEX_DRIVER, |
| 2193 | nativeThreadId: "workflow-member-metadata", |
| 2194 | }); |
| 2195 | const providerSessionId = yield* idAllocator.allocate.providerSession({ |
| 2196 | providerInstanceId: modelSelection.instanceId, |
| 2197 | threadId: rootEvent.threadId, |
| 2198 | }); |
| 2199 | const offered = { |
| 2200 | ...rootEvent.payload, |
| 2201 | id: childThreadId, |
| 2202 | title: "Inspect checkout", |
| 2203 | activeProviderThreadId: null, |
| 2204 | lineage: { |
| 2205 | parentThreadId: rootEvent.threadId, |
| 2206 | relationshipToParent: "subagent" as const, |
| 2207 | rootThreadId: rootEvent.threadId, |
| 2208 | }, |
| 2209 | }; |
| 2210 | const ingest = (appThread: OrchestrationV2AppThread) => |
| 2211 | ingestor.ingestNormalized({ |
| 2212 | providerSessionId, |
| 2213 | providerInstanceId: modelSelection.instanceId, |
| 2214 | threadId: rootEvent.threadId, |
| 2215 | event: { type: "app_thread.created", driver: CODEX_DRIVER, appThread }, |
| 2216 | }); |
| 2217 | yield* eventSink.write({ events: [rootEvent] }); |
| 2218 | const created = yield* Effect.all([ingest(offered), ingest(offered)], { |
| 2219 | concurrency: "unbounded", |
| 2220 | }); |
| 2221 | assert.deepEqual( |
| 2222 | created.flat().map((stored) => stored.event.type), |
| 2223 | ["thread.created"], |
| 2224 | ); |
| 2225 | const saved = { |
| 2226 | ...offered, |
| 2227 | archivedAt: now, |
| 2228 | deletedAt: now, |
| 2229 | activeProviderThreadId: rootEvent.payload.activeProviderThreadId, |
| 2230 | branch: "user/checkout", |
| 2231 | }; |
| 2232 | yield* eventSink.write({ |
| 2233 | events: [ |
| 2234 | { |
| 2235 | ...rootEvent, |
| 2236 | id: yield* idAllocator.allocate.event({ threadId: childThreadId }), |
| 2237 | threadId: childThreadId, |
| 2238 | type: "thread.deleted", |
| 2239 | payload: saved, |
| 2240 | }, |
| 2241 | ], |
| 2242 | }); |
| 2243 | for (const metadata of [ |
| 2244 | { title: "Review checkout", modelSelection }, |
| 2245 | { title: "Review checkout", modelSelection: { ...modelSelection, model: "gpt-6.1-sol" } }, |
| 2246 | ]) { |
| 2247 | const updated = yield* ingest({ ...offered, ...metadata }); |
| 2248 | assert.deepEqual( |
| 2249 | updated.map((stored) => stored.event.type), |
| 2250 | ["thread.metadata-updated"], |
| 2251 | ); |
| 2252 | const thread = yield* projectionStore.getThread(childThreadId); |
| 2253 | assert.deepEqual(thread, { ...saved, ...metadata, updatedAt: thread.updatedAt }); |
| 2254 | } |
| 2255 | assert.deepEqual( |
| 2256 | yield* ingest({ |
| 2257 | ...offered, |
| 2258 | title: "Review checkout", |
| 2259 | modelSelection: { ...modelSelection, model: "gpt-6.1-sol" }, |
| 2260 | }), |
| 2261 | [], |
| 2262 | ); |
| 2263 | }), |
| 2264 | ); |
| 2265 | |
| 2266 | it.effect( |
| 2267 | "recovers a missing Claude child once and preserves saved metadata on later recovery", |
| 2268 | () => |
| 2269 | Effect.gen(function* () { |
| 2270 | const now = yield* DateTime.now; |
| 2271 | const eventSink = yield* EventSink.EventSinkV2; |
| 2272 | const projectionStore = yield* ProjectionStore.ProjectionStoreV2; |
| 2273 | const ingestor = yield* ProviderEventIngestor.ProviderEventIngestorV2; |
| 2274 | const idAllocator = yield* IdAllocator.IdAllocatorV2; |
| 2275 | const rootEvent = yield* threadCreatedEvent(now); |
| 2276 | if (rootEvent.type !== "thread.created") throw new Error("Expected thread fixture"); |
| 2277 | const driver = ProviderDriverKind.make("claude"); |
| 2278 | const providerInstanceId = ProviderInstanceId.make("claude"); |
| 2279 | const childThreadId = idAllocator.derive.threadFromProviderThread({ |
| 2280 | driver, |
| 2281 | nativeThreadId: "recovered-claude-child", |
| 2282 | }); |
| 2283 | const providerSessionId = yield* idAllocator.allocate.providerSession({ |
| 2284 | providerInstanceId, |
| 2285 | threadId: rootEvent.threadId, |
| 2286 | }); |
| 2287 | const recovered: OrchestrationV2AppThread = { |
| 2288 | ...rootEvent.payload, |
| 2289 | id: childThreadId, |
| 2290 | title: "Recovered subagent", |
| 2291 | providerInstanceId, |
| 2292 | modelSelection: { instanceId: providerInstanceId, model: "claude-sonnet-4-6" }, |
| 2293 | activeProviderThreadId: null, |
| 2294 | createdBy: "agent", |
| 2295 | creationSource: "provider", |
| 2296 | lineage: { |
| 2297 | parentThreadId: rootEvent.threadId, |
| 2298 | relationshipToParent: "subagent", |
| 2299 | rootThreadId: rootEvent.threadId, |
| 2300 | }, |
| 2301 | }; |
| 2302 | const ingest = (appThread: OrchestrationV2AppThread, createOnly: boolean) => |
| 2303 | ingestor.ingestNormalized({ |
| 2304 | providerSessionId, |
| 2305 | providerInstanceId, |
| 2306 | threadId: rootEvent.threadId, |
| 2307 | event: { type: "app_thread.created", driver, appThread, createOnly }, |
| 2308 | }); |
| 2309 | yield* eventSink.write({ events: [rootEvent] }); |
| 2310 | const created = yield* Effect.all([ingest(recovered, true), ingest(recovered, true)], { |
| 2311 | concurrency: "unbounded", |
| 2312 | }); |
| 2313 | assert.deepEqual( |
| 2314 | created.flat().map((stored) => stored.event.type), |
| 2315 | ["thread.created"], |
| 2316 | ); |
| 2317 | assert.deepEqual(yield* projectionStore.getThread(childThreadId), recovered); |
| 2318 | |
| 2319 | yield* ingest( |
| 2320 | { |
| 2321 | ...recovered, |
| 2322 | title: "Review authentication", |
| 2323 | modelSelection: { instanceId: providerInstanceId, model: "claude-opus-4-6" }, |
| 2324 | }, |
| 2325 | false, |
| 2326 | ); |
| 2327 | const saved = yield* projectionStore.getThread(childThreadId); |
| 2328 | assert.equal(saved.title, "Review authentication"); |
| 2329 | assert.equal(saved.modelSelection.model, "claude-opus-4-6"); |
| 2330 | assert.deepEqual(yield* ingest(recovered, true), []); |
| 2331 | assert.deepEqual(yield* projectionStore.getThread(childThreadId), saved); |
| 2332 | }), |
| 2333 | ); |
| 2334 | |
| 2335 | it.effect("moves a native subagent's thread to its later model through a database lock", () => |
| 2336 | Effect.gen(function* () { |
| 2337 | const now = yield* DateTime.now; |
| 2338 | const eventSink = yield* EventSink.EventSinkV2; |
| 2339 | const projectionStore = yield* ProjectionStore.ProjectionStoreV2; |
| 2340 | const ingestor = yield* ProviderEventIngestor.ProviderEventIngestorV2; |
| 2341 | const idAllocator = yield* IdAllocator.IdAllocatorV2; |
| 2342 | const rootEvent = yield* threadCreatedEvent(now); |
| 2343 | if (rootEvent.type !== "thread.created") { |
| 2344 | throw new Error("Expected a thread.created fixture event"); |
| 2345 | } |
| 2346 | const childThreadId = idAllocator.derive.threadFromProviderThread({ |
| 2347 | driver: CODEX_DRIVER, |
| 2348 | nativeThreadId: "native-late-model-subagent", |
| 2349 | }); |
| 2350 | const providerSessionId = yield* idAllocator.allocate.providerSession({ |
| 2351 | providerInstanceId: modelSelection.instanceId, |
| 2352 | threadId: rootEvent.threadId, |
| 2353 | }); |
| 2354 | const ingest = (event: ProviderEventIngestor.ProviderEventIngestInput["event"]) => |
| 2355 | ingestor.ingestNormalized({ |
| 2356 | providerSessionId, |
| 2357 | providerInstanceId: modelSelection.instanceId, |
| 2358 | threadId: rootEvent.threadId, |
| 2359 | event, |
| 2360 | }); |
| 2361 | yield* eventSink.write({ events: [rootEvent] }); |
| 2362 | yield* ingest({ |
| 2363 | type: "app_thread.created", |
| 2364 | driver: CODEX_DRIVER, |
| 2365 | appThread: { |
| 2366 | ...rootEvent.payload, |
| 2367 | id: childThreadId, |
| 2368 | title: "review design", |
| 2369 | modelSelection: { |
| 2370 | ...modelSelection, |
| 2371 | options: [{ id: "reasoningEffort", value: "xhigh" }], |
| 2372 | }, |
| 2373 | activeProviderThreadId: null, |
| 2374 | lineage: { |
| 2375 | parentThreadId: rootEvent.threadId, |
| 2376 | relationshipToParent: "subagent", |
| 2377 | rootThreadId: rootEvent.threadId, |
| 2378 | }, |
| 2379 | }, |
| 2380 | }); |
| 2381 | const subagentUpdated = { |
| 2382 | type: "subagent.updated", |
| 2383 | driver: CODEX_DRIVER, |
| 2384 | subagent: { |
| 2385 | id: NodeId.make("node:late-model-subagent"), |
| 2386 | threadId: rootEvent.threadId, |
| 2387 | runId: null, |
| 2388 | parentNodeId: NodeId.make("node:root"), |
| 2389 | origin: "provider_native", |
| 2390 | createdBy: "agent", |
| 2391 | driver: CODEX_DRIVER, |
| 2392 | providerInstanceId: modelSelection.instanceId, |
| 2393 | providerThreadId: null, |
| 2394 | childThreadId, |
| 2395 | nativeTaskRef: null, |
| 2396 | prompt: "Review the design", |
| 2397 | title: "review design", |
| 2398 | model: "gpt-6.1-sol", |
| 2399 | status: "running", |
| 2400 | result: null, |
| 2401 | startedAt: now, |
| 2402 | completedAt: null, |
| 2403 | updatedAt: now, |
| 2404 | }, |
| 2405 | } satisfies ProviderEventIngestor.ProviderEventIngestInput["event"]; |
| 2406 | |
| 2407 | const first = yield* ingest(subagentUpdated); |
| 2408 | const repeated = yield* ingest(subagentUpdated); |
| 2409 | const childThread = yield* projectionStore.getThread(childThreadId); |
| 2410 | |
| 2411 | assert.deepEqual( |
| 2412 | first.map((stored) => [stored.event.type, stored.event.threadId]), |
| 2413 | [ |
| 2414 | ["subagent.updated", rootEvent.threadId], |
| 2415 | ["thread.model-selection-updated", childThreadId], |
| 2416 | ], |
| 2417 | ); |
| 2418 | assert.deepEqual( |
| 2419 | repeated.map((stored) => stored.event.type), |
| 2420 | ["subagent.updated"], |
| 2421 | ); |
| 2422 | assert.deepEqual(childThread.modelSelection, { |
| 2423 | instanceId: modelSelection.instanceId, |
| 2424 | model: "gpt-6.1-sol", |
| 2425 | }); |
| 2426 | |
| 2427 | const reportedSelection = { |
| 2428 | instanceId: modelSelection.instanceId, |
| 2429 | model: "gpt-6.1-sol", |
| 2430 | options: [{ id: "reasoningEffort", value: "low" }], |
| 2431 | }; |
| 2432 | const reported = yield* ingest({ |
| 2433 | ...subagentUpdated, |
| 2434 | subagent: { ...subagentUpdated.subagent, modelSelection: reportedSelection }, |
| 2435 | }); |
| 2436 | assert.deepEqual( |
| 2437 | reported.map((stored) => stored.event.type), |
| 2438 | ["subagent.updated", "thread.model-selection-updated"], |
| 2439 | ); |
| 2440 | assert.deepEqual( |
| 2441 | (yield* projectionStore.getThread(childThreadId)).modelSelection, |
| 2442 | reportedSelection, |
| 2443 | ); |
| 2444 | |
| 2445 | const recovered = yield* ingest(subagentUpdated); |
| 2446 | assert.deepEqual( |
| 2447 | recovered.map((stored) => stored.event.type), |
| 2448 | ["subagent.updated"], |
| 2449 | ); |
| 2450 | assert.deepEqual( |
| 2451 | (yield* projectionStore.getThread(childThreadId)).modelSelection, |
| 2452 | reportedSelection, |
| 2453 | ); |
| 2454 | }).pipe( |
| 2455 | // Fresh: the shared test layer has already built the ingestor this replaces. |
| 2456 | Effect.provide( |
| 2457 | Layer.fresh( |
| 2458 | layerFailingWrites([lockTimeout, lockTimeout], (events) => |
| 2459 | events.some((event) => event.type === "thread.model-selection-updated"), |
| 2460 | ), |
| 2461 | ), |
| 2462 | ), |
| 2463 | ), |
| 2464 | ); |
| 2465 | }); |
| 2466 | |
| 2467 | const ingestCompletedProviderTurn = ( |
| 2468 | guard: Pick< |
| 2469 | Parameters<ProviderEventIngestor.ProviderEventIngestorV2Shape["ingestNormalized"]>[0], |
| 2470 | "writeIfRunCurrent" | "writeIfProviderThreadOwner" |
| 2471 | >, |
| 2472 | ) => |
| 2473 | Effect.gen(function* () { |
| 2474 | const now = yield* DateTime.now; |
| 2475 | const eventSink = yield* EventSink.EventSinkV2; |
| 2476 | const projectionStore = yield* ProjectionStore.ProjectionStoreV2; |
| 2477 | const ingestor = yield* ProviderEventIngestor.ProviderEventIngestorV2; |
| 2478 | const idAllocator = yield* IdAllocator.IdAllocatorV2; |
| 2479 | const threadEvent = yield* threadCreatedEvent(now); |
| 2480 | yield* eventSink.write({ events: [threadEvent] }); |
| 2481 | const providerTurnId = ProviderTurnId.make("provider-turn:locked"); |
| 2482 | const exit = yield* ingestor |
| 2483 | .ingestNormalized({ |
| 2484 | providerSessionId: yield* idAllocator.allocate.providerSession({ |
| 2485 | providerInstanceId: modelSelection.instanceId, |
| 2486 | threadId: threadEvent.threadId, |
| 2487 | }), |
| 2488 | providerInstanceId: modelSelection.instanceId, |
| 2489 | threadId: threadEvent.threadId, |
| 2490 | event: { |
| 2491 | type: "provider_turn.updated", |
| 2492 | driver: CODEX_DRIVER, |
| 2493 | providerTurn: { |
| 2494 | id: providerTurnId, |
| 2495 | providerThreadId: ProviderThreadId.make("provider-thread:locked"), |
| 2496 | nodeId: NodeId.make("node:locked-turn"), |
| 2497 | runAttemptId: null, |
| 2498 | nativeTurnRef: null, |
| 2499 | ordinal: 1, |
| 2500 | status: "completed", |
| 2501 | startedAt: now, |
| 2502 | completedAt: now, |
| 2503 | }, |
| 2504 | }, |
| 2505 | ...guard, |
| 2506 | }) |
| 2507 | .pipe(Effect.exit); |
| 2508 | const projection = yield* projectionStore.getThreadProjection(threadEvent.threadId); |
| 2509 | return { exit, providerTurnId, persisted: projection.providerTurns.map((turn) => turn.id) }; |
| 2510 | }); |
| 2511 | |
| 2512 | // Neither guard matches a stored run, so a guarded write correctly commits nothing. |
| 2513 | const runId = RunId.make("run:locked"); |
| 2514 | const activeAttemptId = RunAttemptId.make("attempt:locked"); |
| 2515 | it.effect.each([ |
| 2516 | { route: "write", guard: {} }, |
| 2517 | { |
| 2518 | route: "writeIfRunCurrent", |
| 2519 | guard: { writeIfRunCurrent: { runId, activeAttemptId, expectedStatus: "running" } }, |
| 2520 | }, |
| 2521 | { |
| 2522 | route: "writeIfProviderThreadOwner", |
| 2523 | guard: { |
| 2524 | writeIfProviderThreadOwner: { |
| 2525 | providerThreadId: ProviderThreadId.make("provider-thread:locked"), |
| 2526 | runId, |
| 2527 | activeAttemptId, |
| 2528 | expectedLastRunOrdinal: 1, |
| 2529 | }, |
| 2530 | }, |
| 2531 | }, |
| 2532 | ] as const)("a provider event $route outlasts a brief database lock", ({ route, guard }) => |
| 2533 | Effect.gen(function* () { |
| 2534 | const { exit, providerTurnId, persisted } = yield* ingestCompletedProviderTurn(guard); |
| 2535 | assert.isTrue(Exit.isSuccess(exit)); |
| 2536 | assert.deepEqual(persisted, route === "write" ? [providerTurnId] : []); |
| 2537 | }).pipe(Effect.provide(layerFailingWrites([lockTimeout, lockTimeout]))), |
| 2538 | ); |
| 2539 | |
| 2540 | // The write after the queued errors would succeed, so failing proves the retries stopped. |
| 2541 | it.effect.each([ |
| 2542 | { failure: "stays locked", errors: [lockTimeout, lockTimeout, lockTimeout] }, |
| 2543 | { failure: "fails for another reason", errors: [constraintFailure] }, |
| 2544 | ])("fails a provider event write that $failure", ({ errors }) => |
| 2545 | Effect.gen(function* () { |
| 2546 | const { exit, persisted } = yield* ingestCompletedProviderTurn({}); |
| 2547 | assert.instanceOf( |
| 2548 | Exit.isFailure(exit) ? Cause.squash(exit.cause) : undefined, |
| 2549 | ProviderEventIngestor.ProviderEventPublishError, |
| 2550 | ); |
| 2551 | assert.isEmpty(persisted); |
| 2552 | }).pipe(Effect.provide(layerFailingWrites(errors))), |
| 2553 | ); |