@devries-cameron20 / t3code-1 / commits / 94baff19c6
Fork 0

The best way to control Claude Code, Codex and more from any computer or phone

fix(server): Antigravity resumes after its local harness fails (#23654)

Co-authored-by: maria-rcks <maria@kuuro.net> Co-authored-by: Julius Marminge <julius0216@outlook.com>

Pedro Satin committed Nov 19, 2026 at 03:31 UTC 94baff19c6526a1bfa5ce2f9a2c37cbfb3703f31
6 files changed +241 -17
apps/server/scripts/acp-mock-agent.ts
+26 -1
@@ -18,6 +18,7 @@ import { beginAcpMockPrompt } from "./acpMockCancellationState.ts";
18
19 const requestLogPath = process.env.T3_ACP_REQUEST_LOG_PATH;
20 const antigravityProfile = process.env.T3_ACP_ANTIGRAVITY === "1";
21 +const harnessFailure = process.env.T3_ACP_ANTIGRAVITY_HARNESS_FAILURE;
22 // GitHub Copilot CLI advertises URL-shaped session mode ids.
23 const copilotModesProfile = process.env.T3_ACP_COPILOT_MODES === "1";
24 const copilotModeId = (name: string) =>
@@ -515,6 +516,16 @@ const program = Effect.gen(function* () {
516 },
517 });
518
519 + const writeHarnessFailure = Effect.promise(
520 + () =>
521 + new Promise<void>((resolve) => {
522 + process.stderr.write(
523 + "I1022 11:04:37.910207 1865358 local_connection.py:551] harness stderr: Agent run failed: failed to handle call hook response: no pending hook for ID hook_request_2\n",
524 + () => resolve(),
525 + );
526 + }),
527 + );
528 +
529 const finishPrompt = (
530 targetSessionId: string,
531 stopReason: AcpSchema.StopReason,
@@ -1005,6 +1016,18 @@ const program = Effect.gen(function* () {
1016 );
1017 return {};
1018 }
1019 + if (
1020 + harnessFailure === "1" ||
1021 + harnessFailure === "pending" ||
1022 + harnessFailure === "after-end-turn"
1023 + ) {
1024 + if (harnessFailure === "after-end-turn") {
1025 + return yield* finishPrompt(requestedSessionId, "end_turn");
1026 + }
1027 + yield* writeHarnessFailure;
1028 + if (harnessFailure === "pending") return yield* Effect.never;
1029 + return yield* finishPrompt(requestedSessionId, "end_turn");
1030 + }
1031
1032 if (clientFsProbePath !== undefined && clientFsProbeLogPath !== undefined) {
1033 const probes = [
@@ -2138,7 +2161,9 @@ const program = Effect.gen(function* () {
2161 typeof params === "object" && params !== null ? Reflect.get(params, "code") : undefined;
2162 process.exit(typeof code === "number" ? code : 19);
2163 })
2141 - : Effect.void,
2164 + : method === "_test/harness-failure"
2165 + ? writeHarnessFailure
2166 + : Effect.void,
2167 );
2168
2169 return yield* Effect.never;
apps/server/src/orchestration-v2/Adapters/AntigravityAdapterV2.test.ts
+151
@@ -40,6 +40,7 @@ import { normalizeAntigravitySessionUpdate } from "../../provider/acp/Antigravit
40 import { ANTIGRAVITY_SIGN_IN_REQUIRED_MESSAGE } from "../../provider/antigravityAuthSupport.ts";
41 import * as IdAllocator from "@t3tools/provider-core/server/IdAllocator";
42 import * as ProviderAdapter from "@t3tools/provider-core/server/ProviderAdapter";
43 +import { makeTurnInput } from "./AcpAdapterV2.testkit.ts";
44 import type * as ProviderContinuationRequests from "@t3tools/provider-core/server/ProviderContinuationRequests";
45 import {
46 type AntigravityAdapterV2RuntimeInput,
@@ -151,6 +152,156 @@ const layerSession = Layer.mergeAll(
152 ),
153 );
154
155 +describe("AntigravityAdapterV2 harness recovery", () => {
156 + const recoverHarnessFailure = (
157 + harnessFailure: "1" | "pending" | "after-end-turn",
158 + firstTurnStatuses: ReadonlyArray<"failed" | "completed">,
159 + ) => {
160 + let terminationCount = 0;
161 + return Effect.gen(function* () {
162 + const childProcessSpawner = yield* ChildProcessSpawner.ChildProcessSpawner;
163 + const fileSystem = yield* FileSystem.FileSystem;
164 + const path = yield* Path.Path;
165 + const crypto = yield* SecureRandom.SecureRandom;
166 + const mockAgentPath = yield* path.fromFileUrl(
167 + new URL("../../../scripts/acp-mock-agent.ts", import.meta.url),
168 + );
169 + const workspace = yield* fileSystem.makeTempDirectoryScoped();
170 + const requestLogPath = path.join(workspace, "requests.jsonl");
171 + let processCount = 0;
172 + const firstRuntimeTerminated = yield* Deferred.make<void>();
173 + let firstRuntime: AcpSessionRuntime.AcpSessionRuntime["Service"] | undefined;
174 + const instanceId = ProviderInstanceId.make("antigravity-recovery-test");
175 + const adapter = yield* makeAntigravityAdapterV2({
176 + instanceId,
177 + selfInvocation: yield* resolveSelfInvocation(),
178 + makeRuntime: (input) =>
179 + Effect.suspend(() => {
180 + processCount += 1;
181 + const processNumber = processCount;
182 + return makeAntigravityAcpRuntime({
183 + ...input,
184 + onTermination: (error) =>
185 + (input.onTermination?.(error) ?? Effect.void).pipe(
186 + Effect.tap(
187 + Effect.sync(() => {
188 + terminationCount += 1;
189 + }),
190 + ),
191 + Effect.andThen(
192 + processNumber === 1
193 + ? Deferred.succeed(firstRuntimeTerminated, undefined)
194 + : Effect.void,
195 + ),
196 + ),
197 + childProcessSpawner,
198 + spawn: {
199 + command: process.execPath,
200 + args: [mockAgentPath],
201 + cwd: input.cwd,
202 + env: {
203 + T3_ACP_ANTIGRAVITY: "1",
204 + T3_ACP_REQUEST_LOG_PATH: requestLogPath,
205 + ...(processNumber === 1
206 + ? { T3_ACP_ANTIGRAVITY_HARNESS_FAILURE: harnessFailure }
207 + : {}),
208 + },
209 + },
210 + }).pipe(
211 + Effect.provideService(SecureRandom.SecureRandom, crypto),
212 + Effect.tap((runtime) =>
213 + Effect.sync(() => {
214 + if (processNumber === 1) firstRuntime = runtime;
215 + }),
216 + ),
217 + );
218 + }),
219 + withProcess: (_stop, task) => task,
220 + defaultModel: Effect.undefined,
221 + });
222 + const threadId = ThreadId.make("thread-antigravity-recovery");
223 + const modelSelection = { instanceId, model: "gemini-test-low" } as const;
224 + const runtimePolicy = ProviderAdapter.ProviderAdapterV2RuntimePolicy.make({
225 + runtimeMode: "full-access",
226 + interactionMode: "default",
227 + cwd: workspace,
228 + });
229 + const session = yield* adapter.openSession({
230 + threadId,
231 + providerSessionId: ProviderSessionId.make("provider-session-antigravity-recovery"),
232 + modelSelection,
233 + runtimePolicy,
234 + });
235 + const events = yield* Queue.unbounded<ProviderAdapter.ProviderAdapterV2Event>();
236 + yield* session.events.pipe(
237 + Stream.runForEach((event) => Queue.offer(events, event)),
238 + Effect.forkScoped,
239 + );
240 + const providerThread = yield* session.ensureThread({
241 + threadId,
242 + modelSelection,
243 + runtimePolicy,
244 + });
245 + const now = yield* DateTime.now;
246 + const terminal = Stream.fromQueue(events).pipe(
247 + Stream.filter((event) => event.type === "turn.terminal"),
248 + Stream.runHead,
249 + );
250 + yield* session.startTurn(
251 + makeTurnInput({ threadId, providerThread, instanceId, runtimePolicy, modelSelection, now }),
252 + );
253 + const first = yield* terminal;
254 + assert.equal(first._tag, "Some");
255 + if (first._tag === "Some") assert.include(firstTurnStatuses, first.value.status);
256 + if (harnessFailure === "after-end-turn") {
257 + assert.exists(firstRuntime);
258 + yield* firstRuntime!.notify("_test/harness-failure", {});
259 + }
260 + // The adapter must retire the first process even when end_turn won the race.
261 + yield* Deferred.await(firstRuntimeTerminated);
262 + yield* session.startTurn(
263 + makeTurnInput({
264 + threadId,
265 + providerThread,
266 + instanceId,
267 + runtimePolicy,
268 + modelSelection,
269 + now,
270 + ordinal: 2,
271 + }),
272 + );
273 + const completed = yield* terminal;
274 + assert.equal(completed._tag, "Some");
275 + if (completed._tag === "Some") assert.equal(completed.value.status, "completed");
276 + assert.equal(processCount, 2);
277 + assert.equal(terminationCount, 1);
278 + const requests = (yield* fileSystem.readFileString(requestLogPath))
279 + .trim()
280 + .split("\n")
281 + .map((line) => JSON.parse(line) as { method?: string; params?: { sessionId?: string } });
282 + const resumeRequests = requests.filter((request) => request.method === "session/resume");
283 + assert.equal(resumeRequests.length, 1);
284 + assert.equal(resumeRequests[0]?.params?.sessionId, "mock-session-1");
285 + }).pipe(
286 + Effect.provide(layerSession),
287 + Effect.scoped,
288 + Effect.tap(Effect.sync(() => assert.equal(terminationCount, 1))),
289 + );
290 + };
291 +
292 + it.live("fails an open turn and resumes its saved session after a harness failure", () =>
293 + recoverHarnessFailure("pending", ["failed"]),
294 + );
295 +
296 + it.live("resumes the saved session after a harness failure races end_turn", () =>
297 + recoverHarnessFailure("1", ["failed", "completed"]),
298 + );
299 +
300 + it.live("resumes in a fresh process when the native failure arrives after end_turn", () =>
301 + recoverHarnessFailure("after-end-turn", ["completed"]),
302 + );
303 +});
304 +
305 describe("AntigravityAdapterV2 compaction", () => {
306 it.live.each([false, true])(
307 "negotiates native goals (%s) without enabling manual compaction",
apps/server/src/provider/Drivers/AntigravityDriver.test.ts
+4 -1
@@ -339,7 +339,10 @@ it.layer(layerTest)("AntigravityDriver", (it) => {
339 h.second.harnessPath,
340 ]);
341 for (const launch of nativeLaunches) {
342 - expect(launch.args).toEqual(hostPlatform === "linux" ? ["--uid="] : []);
342 + expect(launch.args).toEqual([
343 + ...(hostPlatform === "linux" ? ["--uid="] : []),
344 + "--alsologtostderr",
345 + ]);
346 expect(launch.profileDirectory).toBe(h.profileDirectory);
347 expect(launch.forceFileStorage).toBe("1");
348 expect(launch.extendEnv).toBe(false);
apps/server/src/provider/antigravityAuthSupport.test.ts
+32 -1
@@ -91,7 +91,7 @@ describe("Antigravity process environment", () => {
91 expect(baseEnv).toEqual(original);
92 expect(spawn).toEqual({
93 command: "/release/acp",
94 - args: ["--uid="],
94 + args: ["--uid=", "--alsologtostderr"],
95 cwd: "/project",
96 extendEnv: false,
97 env: {
@@ -401,6 +401,37 @@ describe("Antigravity stdout compatibility", () => {
401 });
402
403 describe("Antigravity stderr compatibility", () => {
404 + it.effect.each([
405 + "I1022 11:04:37.910207 1865358 local_connection.py:551] harness stderr: Agent run failed: failed to handle call hook response: no pending hook for ID hook_request_2",
406 + "E1022 11:04:37.910207 1865358 server.py:5275] Failed to rebuild agent for session native-session; connection unrecoverable",
407 + "E1022 11:04:37.910207 1865358 server.py:5266] Agent connection lost for session native-session after rebuild retry",
408 + ])("retires the transport on a native harness failure: %s", (line) =>
409 + Effect.gen(function* () {
410 + const handleStderr = makeAntigravityStderrHandler();
411 + yield* handleStderr(line.slice(0, 70));
412 + const error = yield* handleStderr(`${line.slice(70)}\r\n`).pipe(Effect.flip);
413 + expect(error._tag).toBe("AcpTransportError");
414 + expect(error.message).not.toContain("native-session");
415 + expect(error.message).not.toContain("hook_request_2");
416 + }),
417 + );
418 +
419 + it.effect(
420 + "does not classify task prose or unrelated native warnings as a transport failure",
421 + () =>
422 + Effect.gen(function* () {
423 + const handleStderr = makeAntigravityStderrHandler();
424 + yield* handleStderr("Agent run failed: example from a task\n");
425 + yield* handleStderr("Agent connection was lost and could not be re-established\n");
426 + yield* handleStderr(
427 + "I1022 11:04:37.910207 1865358 server.py:5142] Agent connection closed for session native-session; rebuilding agent and retrying the turn once.\n",
428 + );
429 + yield* handleStderr(
430 + "W1022 11:04:37.910207 1865358 telemetry.py:431] No business auth manager configured\n",
431 + );
432 + }),
433 + );
434 +
435 it.effect("forwards fragmented native sign-in URLs from runtime 1.1.1", () =>
436 Effect.gen(function* () {
437 const urls: string[] = [];
apps/server/src/provider/antigravityAuthSupport.ts
+20 -9
@@ -470,6 +470,8 @@ export function buildAntigravityAcpSpawnInput(input: {
470 return {
471 command: input.installation.executablePath,
472 // The agy CLI has no harness sibling and serves ACP through its `acp` subcommand.
473 + // The harness can survive a failed local agent and return end_turn; its native
474 + // logs are the failure signal, so the transport must receive them.
475 args:
476 input.installation.harnessPath === null
477 ? [
@@ -480,9 +482,7 @@ export function buildAntigravityAcpSpawnInput(input: {
482 // The CLI joins this beneath gemini-dir, including when the value is absolute.
483 "antigravity-acp",
484 ]
483 - : input.profile.platform === "linux"
484 - ? ["--uid="]
485 - : [],
485 + : [...(input.profile.platform === "linux" ? ["--uid="] : []), "--alsologtostderr"],
486 cwd: input.cwd,
487 env: {
488 ...antigravityEnvironment(
@@ -602,6 +602,10 @@ export function makeAntigravityStdoutTransform(
602 });
603 }
604
605 +// glog records from the managed harness after which its ACP process can no longer run a turn.
606 +const ANTIGRAVITY_HARNESS_FAILURE_LOG =
607 + /^[IWEF]\d{4} \d{2}:\d{2}:\d{2}\.\d+ +\d+ (?:local_connection\.py:\d+\] harness stderr: Agent run failed:|server\.py:\d+\] (?:Failed to rebuild agent for session |Agent connection lost for session ))/;
608 +
609 export function makeAntigravityStderrHandler(
610 input: {
611 readonly onAuthorizationUrl?: (
@@ -610,18 +614,25 @@ export function makeAntigravityStderrHandler(
614 } = {},
615 ) {
616 let pending = "";
613 - const handleLine = (line: string) => {
617 + // Untraced: with native logs on stderr this runs for every harness log line.
618 + const handleLine = Effect.fnUntraced(function* (line: string) {
619 const message = line.endsWith("\r") ? line.slice(0, -1) : line;
615 - if (message.length > maxBrowserHelperLineLength) {
616 - return Effect.void;
620 + if (message.length > maxBrowserHelperLineLength) return;
621 + if (ANTIGRAVITY_HARNESS_FAILURE_LOG.test(message)) {
622 + return yield* new AcpErrors.AcpTransportError({
623 + method: "session/prompt",
624 + detail:
625 + "Antigravity's local agent connection failed. Send a follow-up to resume with a fresh process.",
626 + cause: undefined,
627 + });
628 }
629 const url = message.startsWith(ANTIGRAVITY_AUTH_STDOUT_PREFIX)
630 ? Effect.succeed(message.slice(ANTIGRAVITY_AUTH_STDOUT_PREFIX.length))
631 : message.startsWith(ANTIGRAVITY_AUTH_BROWSER_MARKER)
632 ? decodeBrowserHelperUrl(message.slice(ANTIGRAVITY_AUTH_BROWSER_MARKER.length))
633 : undefined;
623 - if (url === undefined) return Effect.void;
624 - return url.pipe(
634 + if (url === undefined) return;
635 + yield* url.pipe(
636 Effect.flatMap(parseAntigravityAuthorizationUrl),
637 Effect.matchEffect({
638 onFailure: () => Effect.void,
@@ -631,7 +642,7 @@ export function makeAntigravityStderrHandler(
642 : Effect.fail(authSupportError(ANTIGRAVITY_SIGN_IN_REQUIRED_MESSAGE)),
643 }),
644 );
634 - };
645 + });
646
647 return Effect.fn("antigravityAuthSupport.handleStderr")(function* (text: string) {
648 const lines = `${pending}${text}`.split("\n");
packages/provider-acp/src/server/AcpSessionRuntime.ts
+8 -5
@@ -1291,6 +1291,7 @@ export const make = Effect.fn("AcpSessionRuntime.make")(function* (
1291 Option.none(),
1292 );
1293 const stoppingRef = yield* Ref.make(false);
1294 + const terminationRecorded = yield* Deferred.make<void>();
1295 const stderrFailure = yield* Deferred.make<never, EffectAcpErrors.AcpError>();
1296 const stderrTailRef = yield* Ref.make("");
1297 const homeDirectory = yield* HostProcess.HomeDirectory;
@@ -1352,10 +1353,13 @@ export const make = Effect.fn("AcpSessionRuntime.make")(function* (
1353 : ([true, Option.some(enriched)] as const),
1354 );
1355 if (!firstTermination) {
1355 - return;
1356 + return yield* Deferred.await(terminationRecorded);
1357 }
1357 - yield* closeActiveAssistantSegment({ queue: eventQueue, assistantSegmentRef });
1358 - yield* Queue.offer(eventQueue, { _tag: "ConnectionTerminated", error: enriched });
1358 + yield* Effect.gen(function* () {
1359 + yield* closeActiveAssistantSegment({ queue: eventQueue, assistantSegmentRef });
1360 + yield* Queue.offer(eventQueue, { _tag: "ConnectionTerminated", error: enriched });
1361 + yield* options.onTermination?.(enriched) ?? Effect.void;
1362 + }).pipe(Effect.ensuring(Deferred.succeed(terminationRecorded, undefined)));
1363 });
1364
1365 const logRequest = (event: AcpSessionRequestLogEvent) =>
@@ -1666,8 +1670,8 @@ export const make = Effect.fn("AcpSessionRuntime.make")(function* (
1670 ),
1671 Effect.catch((error) =>
1672 Effect.gen(function* () {
1669 - yield* Deferred.fail(stderrFailure, error);
1673 yield* recordTermination(error);
1674 + yield* Deferred.fail(stderrFailure, error);
1675 yield* child.kill({ forceKillAfter: "1 second" }).pipe(Effect.ignore);
1676 }),
1677 ),
@@ -1694,7 +1698,6 @@ export const make = Effect.fn("AcpSessionRuntime.make")(function* (
1698 ...(options.onIncomingRequest ? { onIncomingRequest: options.onIncomingRequest } : {}),
1699 onTermination: (error) =>
1700 recordTermination(error).pipe(
1697 - Effect.andThen(options.onTermination?.(error) ?? Effect.void),
1701 Effect.ensuring(
1702 Scope.close(runtimeScope, Exit.fail(error)).pipe(Effect.forkDetach, Effect.asVoid),
1703 ),