The best way to control Claude Code, Codex and more from any computer or phone
Mirrored from https://github.com/Camo67/t3code.git
Synced Nov 21, 07:15 UTC
fix(server): Antigravity resumes after its local harness fails (#23654)
Co-authored-by: maria-rcks <maria@kuuro.net> Co-authored-by: Julius Marminge <julius0216@outlook.com>
Pedro Satin committed
Nov 19, 2026 at 03:31 UTC
94baff19c6526a1bfa5ce2f9a2c37cbfb3703f31
6 files changed
+241
-17
apps/server/scripts/acp-mock-agent.ts
+26
-1
@@ -18,6 +18,7 @@ import { beginAcpMockPrompt } from "./acpMockCancellationState.ts";
18
19
const requestLogPath = process.env.T3_ACP_REQUEST_LOG_PATH;
20
const antigravityProfile = process.env.T3_ACP_ANTIGRAVITY === "1";
21
+const harnessFailure = process.env.T3_ACP_ANTIGRAVITY_HARNESS_FAILURE;
22
// GitHub Copilot CLI advertises URL-shaped session mode ids.
23
const copilotModesProfile = process.env.T3_ACP_COPILOT_MODES === "1";
24
const copilotModeId = (name: string) =>
@@ -515,6 +516,16 @@ const program = Effect.gen(function* () {
516
},
517
});
518
519
+ const writeHarnessFailure = Effect.promise(
520
+ () =>
521
+ new Promise<void>((resolve) => {
522
+ process.stderr.write(
523
+ "I1022 11:04:37.910207 1865358 local_connection.py:551] harness stderr: Agent run failed: failed to handle call hook response: no pending hook for ID hook_request_2\n",
524
+ () => resolve(),
525
+ );
526
+ }),
527
+ );
528
+
529
const finishPrompt = (
530
targetSessionId: string,
531
stopReason: AcpSchema.StopReason,
@@ -1005,6 +1016,18 @@ const program = Effect.gen(function* () {
1016
);
1017
return {};
1018
}
1019
+ if (
1020
+ harnessFailure === "1" ||
1021
+ harnessFailure === "pending" ||
1022
+ harnessFailure === "after-end-turn"
1023
+ ) {
1024
+ if (harnessFailure === "after-end-turn") {
1025
+ return yield* finishPrompt(requestedSessionId, "end_turn");
1026
+ }
1027
+ yield* writeHarnessFailure;
1028
+ if (harnessFailure === "pending") return yield* Effect.never;
1029
+ return yield* finishPrompt(requestedSessionId, "end_turn");
1030
+ }
1031
1032
if (clientFsProbePath !== undefined && clientFsProbeLogPath !== undefined) {
1033
const probes = [
@@ -2138,7 +2161,9 @@ const program = Effect.gen(function* () {
2161
typeof params === "object" && params !== null ? Reflect.get(params, "code") : undefined;
2162
process.exit(typeof code === "number" ? code : 19);
2163
})
2141
- : Effect.void,
2164
+ : method === "_test/harness-failure"
2165
+ ? writeHarnessFailure
2166
+ : Effect.void,
2167
);
2168
2169
return yield* Effect.never;
apps/server/src/orchestration-v2/Adapters/AntigravityAdapterV2.test.ts
+151
@@ -40,6 +40,7 @@ import { normalizeAntigravitySessionUpdate } from "../../provider/acp/Antigravit
40
import { ANTIGRAVITY_SIGN_IN_REQUIRED_MESSAGE } from "../../provider/antigravityAuthSupport.ts";
41
import * as IdAllocator from "@t3tools/provider-core/server/IdAllocator";
42
import * as ProviderAdapter from "@t3tools/provider-core/server/ProviderAdapter";
43
+import { makeTurnInput } from "./AcpAdapterV2.testkit.ts";
44
import type * as ProviderContinuationRequests from "@t3tools/provider-core/server/ProviderContinuationRequests";
45
import {
46
type AntigravityAdapterV2RuntimeInput,
@@ -151,6 +152,156 @@ const layerSession = Layer.mergeAll(
152
),
153
);
154
155
+describe("AntigravityAdapterV2 harness recovery", () => {
156
+ const recoverHarnessFailure = (
157
+ harnessFailure: "1" | "pending" | "after-end-turn",
158
+ firstTurnStatuses: ReadonlyArray<"failed" | "completed">,
159
+ ) => {
160
+ let terminationCount = 0;
161
+ return Effect.gen(function* () {
162
+ const childProcessSpawner = yield* ChildProcessSpawner.ChildProcessSpawner;
163
+ const fileSystem = yield* FileSystem.FileSystem;
164
+ const path = yield* Path.Path;
165
+ const crypto = yield* SecureRandom.SecureRandom;
166
+ const mockAgentPath = yield* path.fromFileUrl(
167
+ new URL("../../../scripts/acp-mock-agent.ts", import.meta.url),
168
+ );
169
+ const workspace = yield* fileSystem.makeTempDirectoryScoped();
170
+ const requestLogPath = path.join(workspace, "requests.jsonl");
171
+ let processCount = 0;
172
+ const firstRuntimeTerminated = yield* Deferred.make<void>();
173
+ let firstRuntime: AcpSessionRuntime.AcpSessionRuntime["Service"] | undefined;
174
+ const instanceId = ProviderInstanceId.make("antigravity-recovery-test");
175
+ const adapter = yield* makeAntigravityAdapterV2({
176
+ instanceId,
177
+ selfInvocation: yield* resolveSelfInvocation(),
178
+ makeRuntime: (input) =>
179
+ Effect.suspend(() => {
180
+ processCount += 1;
181
+ const processNumber = processCount;
182
+ return makeAntigravityAcpRuntime({
183
+ ...input,
184
+ onTermination: (error) =>
185
+ (input.onTermination?.(error) ?? Effect.void).pipe(
186
+ Effect.tap(
187
+ Effect.sync(() => {
188
+ terminationCount += 1;
189
+ }),
190
+ ),
191
+ Effect.andThen(
192
+ processNumber === 1
193
+ ? Deferred.succeed(firstRuntimeTerminated, undefined)
194
+ : Effect.void,
195
+ ),
196
+ ),
197
+ childProcessSpawner,
198
+ spawn: {
199
+ command: process.execPath,
200
+ args: [mockAgentPath],
201
+ cwd: input.cwd,
202
+ env: {
203
+ T3_ACP_ANTIGRAVITY: "1",
204
+ T3_ACP_REQUEST_LOG_PATH: requestLogPath,
205
+ ...(processNumber === 1
206
+ ? { T3_ACP_ANTIGRAVITY_HARNESS_FAILURE: harnessFailure }
207
+ : {}),
208
+ },
209
+ },
210
+ }).pipe(
211
+ Effect.provideService(SecureRandom.SecureRandom, crypto),
212
+ Effect.tap((runtime) =>
213
+ Effect.sync(() => {
214
+ if (processNumber === 1) firstRuntime = runtime;
215
+ }),
216
+ ),
217
+ );
218
+ }),
219
+ withProcess: (_stop, task) => task,
220
+ defaultModel: Effect.undefined,
221
+ });
222
+ const threadId = ThreadId.make("thread-antigravity-recovery");
223
+ const modelSelection = { instanceId, model: "gemini-test-low" } as const;
224
+ const runtimePolicy = ProviderAdapter.ProviderAdapterV2RuntimePolicy.make({
225
+ runtimeMode: "full-access",
226
+ interactionMode: "default",
227
+ cwd: workspace,
228
+ });
229
+ const session = yield* adapter.openSession({
230
+ threadId,
231
+ providerSessionId: ProviderSessionId.make("provider-session-antigravity-recovery"),
232
+ modelSelection,
233
+ runtimePolicy,
234
+ });
235
+ const events = yield* Queue.unbounded<ProviderAdapter.ProviderAdapterV2Event>();
236
+ yield* session.events.pipe(
237
+ Stream.runForEach((event) => Queue.offer(events, event)),
238
+ Effect.forkScoped,
239
+ );
240
+ const providerThread = yield* session.ensureThread({
241
+ threadId,
242
+ modelSelection,
243
+ runtimePolicy,
244
+ });
245
+ const now = yield* DateTime.now;
246
+ const terminal = Stream.fromQueue(events).pipe(
247
+ Stream.filter((event) => event.type === "turn.terminal"),
248
+ Stream.runHead,
249
+ );
250
+ yield* session.startTurn(
251
+ makeTurnInput({ threadId, providerThread, instanceId, runtimePolicy, modelSelection, now }),
252
+ );
253
+ const first = yield* terminal;
254
+ assert.equal(first._tag, "Some");
255
+ if (first._tag === "Some") assert.include(firstTurnStatuses, first.value.status);
256
+ if (harnessFailure === "after-end-turn") {
257
+ assert.exists(firstRuntime);
258
+ yield* firstRuntime!.notify("_test/harness-failure", {});
259
+ }
260
+ // The adapter must retire the first process even when end_turn won the race.
261
+ yield* Deferred.await(firstRuntimeTerminated);
262
+ yield* session.startTurn(
263
+ makeTurnInput({
264
+ threadId,
265
+ providerThread,
266
+ instanceId,
267
+ runtimePolicy,
268
+ modelSelection,
269
+ now,
270
+ ordinal: 2,
271
+ }),
272
+ );
273
+ const completed = yield* terminal;
274
+ assert.equal(completed._tag, "Some");
275
+ if (completed._tag === "Some") assert.equal(completed.value.status, "completed");
276
+ assert.equal(processCount, 2);
277
+ assert.equal(terminationCount, 1);
278
+ const requests = (yield* fileSystem.readFileString(requestLogPath))
279
+ .trim()
280
+ .split("\n")
281
+ .map((line) => JSON.parse(line) as { method?: string; params?: { sessionId?: string } });
282
+ const resumeRequests = requests.filter((request) => request.method === "session/resume");
283
+ assert.equal(resumeRequests.length, 1);
284
+ assert.equal(resumeRequests[0]?.params?.sessionId, "mock-session-1");
285
+ }).pipe(
286
+ Effect.provide(layerSession),
287
+ Effect.scoped,
288
+ Effect.tap(Effect.sync(() => assert.equal(terminationCount, 1))),
289
+ );
290
+ };
291
+
292
+ it.live("fails an open turn and resumes its saved session after a harness failure", () =>
293
+ recoverHarnessFailure("pending", ["failed"]),
294
+ );
295
+
296
+ it.live("resumes the saved session after a harness failure races end_turn", () =>
297
+ recoverHarnessFailure("1", ["failed", "completed"]),
298
+ );
299
+
300
+ it.live("resumes in a fresh process when the native failure arrives after end_turn", () =>
301
+ recoverHarnessFailure("after-end-turn", ["completed"]),
302
+ );
303
+});
304
+
305
describe("AntigravityAdapterV2 compaction", () => {
306
it.live.each([false, true])(
307
"negotiates native goals (%s) without enabling manual compaction",
apps/server/src/provider/Drivers/AntigravityDriver.test.ts
+4
-1
@@ -339,7 +339,10 @@ it.layer(layerTest)("AntigravityDriver", (it) => {
339
h.second.harnessPath,
340
]);
341
for (const launch of nativeLaunches) {
342
- expect(launch.args).toEqual(hostPlatform === "linux" ? ["--uid="] : []);
342
+ expect(launch.args).toEqual([
343
+ ...(hostPlatform === "linux" ? ["--uid="] : []),
344
+ "--alsologtostderr",
345
+ ]);
346
expect(launch.profileDirectory).toBe(h.profileDirectory);
347
expect(launch.forceFileStorage).toBe("1");
348
expect(launch.extendEnv).toBe(false);
apps/server/src/provider/antigravityAuthSupport.test.ts
+32
-1
@@ -91,7 +91,7 @@ describe("Antigravity process environment", () => {
91
expect(baseEnv).toEqual(original);
92
expect(spawn).toEqual({
93
command: "/release/acp",
94
- args: ["--uid="],
94
+ args: ["--uid=", "--alsologtostderr"],
95
cwd: "/project",
96
extendEnv: false,
97
env: {
@@ -401,6 +401,37 @@ describe("Antigravity stdout compatibility", () => {
401
});
402
403
describe("Antigravity stderr compatibility", () => {
404
+ it.effect.each([
405
+ "I1022 11:04:37.910207 1865358 local_connection.py:551] harness stderr: Agent run failed: failed to handle call hook response: no pending hook for ID hook_request_2",
406
+ "E1022 11:04:37.910207 1865358 server.py:5275] Failed to rebuild agent for session native-session; connection unrecoverable",
407
+ "E1022 11:04:37.910207 1865358 server.py:5266] Agent connection lost for session native-session after rebuild retry",
408
+ ])("retires the transport on a native harness failure: %s", (line) =>
409
+ Effect.gen(function* () {
410
+ const handleStderr = makeAntigravityStderrHandler();
411
+ yield* handleStderr(line.slice(0, 70));
412
+ const error = yield* handleStderr(`${line.slice(70)}\r\n`).pipe(Effect.flip);
413
+ expect(error._tag).toBe("AcpTransportError");
414
+ expect(error.message).not.toContain("native-session");
415
+ expect(error.message).not.toContain("hook_request_2");
416
+ }),
417
+ );
418
+
419
+ it.effect(
420
+ "does not classify task prose or unrelated native warnings as a transport failure",
421
+ () =>
422
+ Effect.gen(function* () {
423
+ const handleStderr = makeAntigravityStderrHandler();
424
+ yield* handleStderr("Agent run failed: example from a task\n");
425
+ yield* handleStderr("Agent connection was lost and could not be re-established\n");
426
+ yield* handleStderr(
427
+ "I1022 11:04:37.910207 1865358 server.py:5142] Agent connection closed for session native-session; rebuilding agent and retrying the turn once.\n",
428
+ );
429
+ yield* handleStderr(
430
+ "W1022 11:04:37.910207 1865358 telemetry.py:431] No business auth manager configured\n",
431
+ );
432
+ }),
433
+ );
434
+
435
it.effect("forwards fragmented native sign-in URLs from runtime 1.1.1", () =>
436
Effect.gen(function* () {
437
const urls: string[] = [];
apps/server/src/provider/antigravityAuthSupport.ts
+20
-9
@@ -470,6 +470,8 @@ export function buildAntigravityAcpSpawnInput(input: {
470
return {
471
command: input.installation.executablePath,
472
// The agy CLI has no harness sibling and serves ACP through its `acp` subcommand.
473
+ // The harness can survive a failed local agent and return end_turn; its native
474
+ // logs are the failure signal, so the transport must receive them.
475
args:
476
input.installation.harnessPath === null
477
? [
@@ -480,9 +482,7 @@ export function buildAntigravityAcpSpawnInput(input: {
482
// The CLI joins this beneath gemini-dir, including when the value is absolute.
483
"antigravity-acp",
484
]
483
- : input.profile.platform === "linux"
484
- ? ["--uid="]
485
- : [],
485
+ : [...(input.profile.platform === "linux" ? ["--uid="] : []), "--alsologtostderr"],
486
cwd: input.cwd,
487
env: {
488
...antigravityEnvironment(
@@ -602,6 +602,10 @@ export function makeAntigravityStdoutTransform(
602
});
603
}
604
605
+// glog records from the managed harness after which its ACP process can no longer run a turn.
606
+const ANTIGRAVITY_HARNESS_FAILURE_LOG =
607
+ /^[IWEF]\d{4} \d{2}:\d{2}:\d{2}\.\d+ +\d+ (?:local_connection\.py:\d+\] harness stderr: Agent run failed:|server\.py:\d+\] (?:Failed to rebuild agent for session |Agent connection lost for session ))/;
608
+
609
export function makeAntigravityStderrHandler(
610
input: {
611
readonly onAuthorizationUrl?: (
@@ -610,18 +614,25 @@ export function makeAntigravityStderrHandler(
614
} = {},
615
) {
616
let pending = "";
613
- const handleLine = (line: string) => {
617
+ // Untraced: with native logs on stderr this runs for every harness log line.
618
+ const handleLine = Effect.fnUntraced(function* (line: string) {
619
const message = line.endsWith("\r") ? line.slice(0, -1) : line;
615
- if (message.length > maxBrowserHelperLineLength) {
616
- return Effect.void;
620
+ if (message.length > maxBrowserHelperLineLength) return;
621
+ if (ANTIGRAVITY_HARNESS_FAILURE_LOG.test(message)) {
622
+ return yield* new AcpErrors.AcpTransportError({
623
+ method: "session/prompt",
624
+ detail:
625
+ "Antigravity's local agent connection failed. Send a follow-up to resume with a fresh process.",
626
+ cause: undefined,
627
+ });
628
}
629
const url = message.startsWith(ANTIGRAVITY_AUTH_STDOUT_PREFIX)
630
? Effect.succeed(message.slice(ANTIGRAVITY_AUTH_STDOUT_PREFIX.length))
631
: message.startsWith(ANTIGRAVITY_AUTH_BROWSER_MARKER)
632
? decodeBrowserHelperUrl(message.slice(ANTIGRAVITY_AUTH_BROWSER_MARKER.length))
633
: undefined;
623
- if (url === undefined) return Effect.void;
624
- return url.pipe(
634
+ if (url === undefined) return;
635
+ yield* url.pipe(
636
Effect.flatMap(parseAntigravityAuthorizationUrl),
637
Effect.matchEffect({
638
onFailure: () => Effect.void,
@@ -631,7 +642,7 @@ export function makeAntigravityStderrHandler(
642
: Effect.fail(authSupportError(ANTIGRAVITY_SIGN_IN_REQUIRED_MESSAGE)),
643
}),
644
);
634
- };
645
+ });
646
647
return Effect.fn("antigravityAuthSupport.handleStderr")(function* (text: string) {
648
const lines = `${pending}${text}`.split("\n");
packages/provider-acp/src/server/AcpSessionRuntime.ts
+8
-5
@@ -1291,6 +1291,7 @@ export const make = Effect.fn("AcpSessionRuntime.make")(function* (
1291
Option.none(),
1292
);
1293
const stoppingRef = yield* Ref.make(false);
1294
+ const terminationRecorded = yield* Deferred.make<void>();
1295
const stderrFailure = yield* Deferred.make<never, EffectAcpErrors.AcpError>();
1296
const stderrTailRef = yield* Ref.make("");
1297
const homeDirectory = yield* HostProcess.HomeDirectory;
@@ -1352,10 +1353,13 @@ export const make = Effect.fn("AcpSessionRuntime.make")(function* (
1353
: ([true, Option.some(enriched)] as const),
1354
);
1355
if (!firstTermination) {
1355
- return;
1356
+ return yield* Deferred.await(terminationRecorded);
1357
}
1357
- yield* closeActiveAssistantSegment({ queue: eventQueue, assistantSegmentRef });
1358
- yield* Queue.offer(eventQueue, { _tag: "ConnectionTerminated", error: enriched });
1358
+ yield* Effect.gen(function* () {
1359
+ yield* closeActiveAssistantSegment({ queue: eventQueue, assistantSegmentRef });
1360
+ yield* Queue.offer(eventQueue, { _tag: "ConnectionTerminated", error: enriched });
1361
+ yield* options.onTermination?.(enriched) ?? Effect.void;
1362
+ }).pipe(Effect.ensuring(Deferred.succeed(terminationRecorded, undefined)));
1363
});
1364
1365
const logRequest = (event: AcpSessionRequestLogEvent) =>
@@ -1666,8 +1670,8 @@ export const make = Effect.fn("AcpSessionRuntime.make")(function* (
1670
),
1671
Effect.catch((error) =>
1672
Effect.gen(function* () {
1669
- yield* Deferred.fail(stderrFailure, error);
1673
yield* recordTermination(error);
1674
+ yield* Deferred.fail(stderrFailure, error);
1675
yield* child.kill({ forceKillAfter: "1 second" }).pipe(Effect.ignore);
1676
}),
1677
),
@@ -1694,7 +1698,6 @@ export const make = Effect.fn("AcpSessionRuntime.make")(function* (
1698
...(options.onIncomingRequest ? { onIncomingRequest: options.onIncomingRequest } : {}),
1699
onTermination: (error) =>
1700
recordTermination(error).pipe(
1697
- Effect.andThen(options.onTermination?.(error) ?? Effect.void),
1701
Effect.ensuring(
1702
Scope.close(runtimeScope, Exit.fail(error)).pipe(Effect.forkDetach, Effect.asVoid),
1703
),