@devries-cameron20 / t3code-1 / commits / cc59f99e73
Fork 0

The best way to control Claude Code, Codex and more from any computer or phone

fix(server): keep PR watches active after host failures (#29450)

## Problem PR watches stopped after eight failed host reads, so a prolonged network outage prevented later CI and review updates from reaching the thread. ## Change Keep watches active after provider failures reported as `failed`. Read detail and activity concurrently without allowing an activity failure to cancel or hide a permanent detail failure. Repeated known permanent failures still stop the watch and notify the agent; rate limits still pause reads. Non-rate-limited GitHub HTTP 403 responses and GraphQL `FORBIDDEN` responses now use a distinct `forbidden` reason so access refusals are counted instead of retried indefinitely. GraphQL error types survive the client mapping, after the existing requested-reviewer fallback. The provider's `failed` category also includes unclassified host and decode errors. Those continue retrying at the existing two-minute cadence until the watch ends or is stopped. A fixed outage deadline would recreate the recovery failure for longer outages. ## Scope and approval This is a focused fix for an obvious bug in existing PR watch recovery, covered by the small bug-fix exception in CONTRIBUTING.md. The changes cover watcher recovery, a GitHub failure classification needed to preserve permanent-error handling, and their regression tests. The new server-side `forbidden` reason does not change RPC contracts or add settings. ## Verification `vp run check:changed` passed, including 154 tests across five files. Regression coverage includes a 30-minute outage followed by one CI-success notification, mixed activity/detail failures, resetting a prior failure count, and repeated forbidden reads stopping the watch. Provider tests distinguish HTTP 403 and exclusively permanent GraphQL refusals from retryable HTTP 503 and mixed or untyped GraphQL errors. The mixed-failure and GraphQL classification cases failed before their fixes. Seven focused requested-reviewer fallback/refusal tests also pass. Model: GPT 6.1 Sol. Harness: Codex.

Yash Singh committed Nov 2, 2026 at 14:26 UTC cc59f99e735300120a6fc74d9a0b9df7c98fa81c
8 files changed +228 -69
apps/server/src/orchestration-v2/PullRequestWatchReactor.ts
+26 -14
@@ -50,13 +50,13 @@ const FINGERPRINT_REREAD_MS = 30 * 60_000;
50
51 const isProviderError = Schema.is(PullRequestProviderError);
52
53 -const isRateLimited = (cause: Cause.Cause<PullRequestService.PullRequestError>) =>
53 +const providerFailureReason = (cause: Cause.Cause<PullRequestService.PullRequestError>) =>
54 Option.match(Cause.findErrorOption(cause), {
55 - onNone: () => false,
55 + onNone: () => null,
56 onSome: (error) =>
57 - error._tag === "PullRequestOperationError" &&
58 - isProviderError(error.cause) &&
59 - error.cause.reason === "rate-limited",
57 + error._tag === "PullRequestOperationError" && isProviderError(error.cause)
58 + ? error.cause.reason
59 + : null,
60 });
61
62 const logFailure =
@@ -444,7 +444,7 @@ export const make = Effect.gen(function* () {
444 (cause): Effect.Effect<ProviderChangeRequestWatchFingerprint | null | "paused"> =>
445 Cause.hasInterruptsOnly(cause)
446 ? Effect.interrupt
447 - : isRateLimited(cause)
447 + : providerFailureReason(cause) === "rate-limited"
448 ? Effect.succeed("paused")
449 : Effect.logDebug("pull request watch fingerprint failed", {
450 pullRequest: group.key,
@@ -472,21 +472,33 @@ export const make = Effect.gen(function* () {
472 const reference = referenceOf(group);
473 // Invalidate cached reads before using the newer fingerprint; clients receive no notification.
474 if (fingerprint !== null) yield* pullRequests.invalidate({ reference });
475 - const read = yield* Effect.exit(
476 - Effect.all(
477 - [
478 - pullRequests.detail({ ...reference, allowStale: false }),
475 + const [detailRead, activityRead] = yield* Effect.all(
476 + [
477 + Effect.exit(pullRequests.detail({ ...reference, allowStale: false })),
478 + Effect.exit(
479 plan.activity
480 ? pullRequests.activity(reference).pipe(Effect.asSome)
481 : Effect.succeed(Option.none<PullRequestActivity>()),
482 - ],
483 - { concurrency: 2 },
484 - ),
482 + ),
483 + ],
484 + { concurrency: 2 },
485 );
486 + // Activity failures must not cancel or hide a permanent detail failure.
487 + const read = Exit.isFailure(detailRead)
488 + ? Exit.failCause(detailRead.cause)
489 + : Exit.isFailure(activityRead)
490 + ? Exit.failCause(activityRead.cause)
491 + : Exit.succeed([detailRead.value, activityRead.value] as const);
492 if (Exit.isFailure(read)) {
493 if (Cause.hasInterruptsOnly(read.cause)) return yield* Effect.failCause(read.cause);
494 lastReads.delete(group.key);
489 - if (isRateLimited(read.cause)) return;
495 + const reason = providerFailureReason(read.cause);
496 + if (reason === "rate-limited") return;
497 + // Transport and host failures can outlast the failure limit; keep the watch for recovery.
498 + if (reason === "failed") {
499 + readFailures.delete(group.key);
500 + return yield* Effect.failCause(read.cause);
501 + }
502 const failures = (readFailures.get(group.key) ?? 0) + 1;
503 readFailures.set(group.key, failures);
504 if (failures >= READ_FAILURE_LIMIT && (yield* eachTarget(group, giveUp))) {
apps/server/src/orchestration-v2/runtimeLayer.test.ts
+147 -54
@@ -1,3 +1,4 @@
1 +import { PullRequestProviderError } from "@t3tools/source-control-core/server/PullRequestProvider";
2 import { SourceControlProviderKind } from "@t3tools/contracts";
3 import { limitRecoveryCommand } from "./UsageLimitRecoveryWorker.ts";
4 import * as McpProviderSessions from "@t3tools/provider-core/server/McpProviderSessions";
@@ -3172,63 +3173,155 @@ it.layer(layerTest)("OrchestrationV2LayerLive lifecycle", (it) => {
3173 }),
3174 );
3175
3175 - it.effect("ends a watch it cannot read, and tells the agent", () =>
3176 - Effect.gen(function* () {
3177 - const orchestrator = yield* Orchestrator.OrchestratorV2;
3178 - const threadId = ThreadId.make("runtime-pull-request-watch-unreadable");
3179 - const projectId = ProjectId.make("pr-watch-unreadable-project");
3180 - yield* seedProject({
3181 - projectId,
3182 - title: "Watch unreadable",
3183 - workspaceRoot: "/workspace/watch-unreadable",
3184 - defaultModelSelection: null,
3185 - createdAt: "2026-10-01T00:00:00.000Z",
3186 - });
3187 - yield* orchestrator.dispatch({
3188 - type: "thread.create",
3189 - createdBy: "user",
3190 - creationSource: "web",
3191 - commandId: CommandId.make("pr-watch-unreadable-create"),
3192 - threadId,
3193 - projectId,
3194 - title: "Watch unreadable",
3195 - modelSelection,
3196 - runtimeMode: "full-access",
3197 - interactionMode: "default",
3198 - branch: null,
3199 - worktreePath: null,
3200 - });
3201 - yield* orchestrator.dispatch({
3202 - type: "thread.pull-request.watch",
3203 - commandId: CommandId.make("pr-watch-unreadable-start"),
3204 - threadId,
3205 - host: "github.com",
3206 - repository: "pingdotgg/t3code",
3207 - number: 8,
3208 - watching: true,
3209 - link: { url: "https://github.com/pingdotgg/t3code/pull/8", source: "agent" },
3210 - });
3211 - const reactor = yield* PullRequestWatchReactor.make.pipe(
3212 - Effect.provide(
3213 - Layer.mergeAll(
3214 - NodeServices.layer,
3215 - Layer.mock(PullRequestService.PullRequestService)({
3216 - detail: () => Effect.die("host unreachable"),
3217 - activity: () => Effect.die("host unreachable"),
3176 + it.effect.each([
3177 + { reason: "failed", activityFails: false },
3178 + { reason: "not-found", activityFails: false },
3179 + { reason: "failed", activityFails: true },
3180 + { reason: "not-found", activityFails: true },
3181 + { reason: "forbidden", activityFails: true },
3182 + ] as const)(
3183 + "handles repeated host read failures: $reason (activity fails: $activityFails)",
3184 + ({ reason, activityFails }) =>
3185 + Effect.gen(function* () {
3186 + const orchestrator = yield* Orchestrator.OrchestratorV2;
3187 + const threadId = ThreadId.make(
3188 + `runtime-pull-request-watch-unreadable-${reason}-${activityFails}`,
3189 + );
3190 + const projectId = ProjectId.make(`pr-watch-unreadable-project-${reason}-${activityFails}`);
3191 + yield* seedProject({
3192 + projectId,
3193 + title: "Watch unreadable",
3194 + workspaceRoot: "/workspace/watch-unreadable",
3195 + defaultModelSelection: null,
3196 + createdAt: "2026-10-01T00:00:00.000Z",
3197 + });
3198 + yield* orchestrator.dispatch({
3199 + type: "thread.create",
3200 + createdBy: "user",
3201 + creationSource: "web",
3202 + commandId: CommandId.make(`pr-watch-unreadable-create-${reason}-${activityFails}`),
3203 + threadId,
3204 + projectId,
3205 + title: "Watch unreadable",
3206 + modelSelection,
3207 + runtimeMode: "full-access",
3208 + interactionMode: "default",
3209 + branch: null,
3210 + worktreePath: null,
3211 + });
3212 + yield* orchestrator.dispatch({
3213 + type: "thread.pull-request.watch",
3214 + commandId: CommandId.make(`pr-watch-unreadable-start-${reason}-${activityFails}`),
3215 + threadId,
3216 + host: "github.com",
3217 + repository: "pingdotgg/t3code",
3218 + number: 8,
3219 + watching: true,
3220 + link: { url: "https://github.com/pingdotgg/t3code/pull/8", source: "agent" },
3221 + });
3222 + let disconnected = false;
3223 + let passed = false;
3224 + let genericFailure = false;
3225 + const activityFailures = yield* Queue.unbounded<void>();
3226 + const readFailure = (operation: string, failure: "failed" | "not-found" | "forbidden") =>
3227 + new PullRequestOperationError({
3228 + operation,
3229 + detail: "host read failed",
3230 + cause: new PullRequestProviderError({
3231 + provider: SourceControlProviderKind.make("github"),
3232 + operation,
3233 + reason: failure,
3234 + detail: "host read failed",
3235 }),
3236 + });
3237 + const reactor = yield* PullRequestWatchReactor.make.pipe(
3238 + Effect.provide(
3239 + Layer.mergeAll(
3240 + NodeServices.layer,
3241 + Layer.mock(PullRequestService.PullRequestService)({
3242 + watchFingerprint: () => Effect.succeed(null),
3243 + detail: () =>
3244 + genericFailure
3245 + ? Effect.die("host read failed")
3246 + : disconnected
3247 + ? (activityFails ? Queue.take(activityFailures) : Effect.void).pipe(
3248 + Effect.andThen(Effect.fail(readFailure("detail", reason))),
3249 + )
3250 + : Effect.succeed({
3251 + ...watchedPullRequestDetail({
3252 + projectId,
3253 + number: 8,
3254 + at: "2026-10-02T12:00:00.000Z",
3255 + }),
3256 + headSha: "abc1234def",
3257 + checks: [
3258 + {
3259 + name: "test",
3260 + status: passed ? ("success" as const) : ("pending" as const),
3261 + description: null,
3262 + url: null,
3263 + required: true,
3264 + },
3265 + ],
3266 + }),
3267 + activity: () =>
3268 + disconnected && activityFails
3269 + ? Queue.offer(activityFailures, undefined).pipe(
3270 + Effect.andThen(Effect.fail(readFailure("activity", "failed"))),
3271 + )
3272 + : Effect.succeed({
3273 + comments: [],
3274 + commentCount: 0,
3275 + commentsTruncated: false,
3276 + reviewThreads: [],
3277 + commits: [],
3278 + }),
3279 + }),
3280 + ),
3281 ),
3220 - ),
3221 - );
3222 - for (let pass = 0; pass < 15; pass += 1) yield* reactor.sweep;
3282 + );
3283 + const worker = yield* EffectWorker.OrchestrationEffectWorkerV2;
3284 + const sweep = reactor.sweep.pipe(Effect.andThen(worker.drain()));
3285 + const summaries = Effect.map(
3286 + orchestrator.getThreadRecords(threadId, ["messages"]),
3287 + ({ messages }) => messages.flatMap((message) => message.notification?.summary ?? []),
3288 + );
3289 + yield* sweep;
3290 + assert.deepEqual(yield* summaries, []);
3291 + disconnected = true;
3292 + for (let pass = 0; pass < 15; pass += 1) {
3293 + genericFailure = reason === "failed" && (pass === 0 || (pass >= 2 && pass <= 8));
3294 + yield* TestClock.adjust("2 minutes");
3295 + yield* sweep;
3296 + }
3297
3224 - const thread = yield* orchestrator.getThreadShell(threadId);
3225 - assert.isUndefined(thread?.pullRequests?.[0]?.watch);
3226 - const { messages } = yield* orchestrator.getThreadRecords(threadId, ["messages"]);
3227 - assert.deepEqual(
3228 - messages.flatMap((message) => message.notification?.summary ?? []),
3229 - ["#8: stopped watching, could not read it"],
3230 - );
3231 - }),
3298 + const thread = yield* orchestrator.getThreadShell(threadId);
3299 + if (reason !== "failed") {
3300 + assert.isUndefined(thread?.pullRequests?.[0]?.watch);
3301 + assert.deepEqual(yield* summaries, ["#8: stopped watching, could not read it"]);
3302 + return;
3303 + }
3304 + assert.isDefined(thread?.pullRequests?.[0]?.watch);
3305 + assert.deepEqual(yield* summaries, []);
3306 + disconnected = false;
3307 + genericFailure = false;
3308 + passed = true;
3309 + yield* sweep;
3310 + assert.deepEqual(yield* summaries, ["#8: checks passed"]);
3311 + yield* TestClock.adjust("10 minutes");
3312 + yield* sweep;
3313 + assert.deepEqual(yield* summaries, ["#8: checks passed"]);
3314 + yield* orchestrator.dispatch({
3315 + type: "thread.pull-request.watch",
3316 + commandId: CommandId.make(`pr-watch-unreadable-stop-${reason}-${activityFails}`),
3317 + threadId,
3318 + host: "github.com",
3319 + repository: "pingdotgg/t3code",
3320 + number: 8,
3321 + watching: false,
3322 + });
3323 + yield* worker.drain();
3324 + }),
3325 );
3326
3327 it.effect("wakes once for a state it reported, across a restart and a new watch", () =>
packages/source-control-core/src/server/PullRequestProvider.ts
+1
@@ -47,6 +47,7 @@ export class PullRequestProviderError extends Schema.TaggedError<PullRequestProv
47 reason: Schema.Literals([
48 "missing-tool",
49 "unauthenticated",
50 + "forbidden",
51 "rate-limited",
52 "not-found",
53 "failed",
packages/source-control-github/src/server/GitHubPullRequestProvider.test.ts
+38
@@ -69,6 +69,44 @@ it.effect("maps credential verification failures without relabeling operation fa
69 }),
70 );
71
72 +it.effect.each([
73 + { status: 403, githubErrorTypes: [], reason: "forbidden" },
74 + { status: 503, githubErrorTypes: [], reason: "failed" },
75 + { status: 200, githubErrorTypes: ["FORBIDDEN"], reason: "forbidden" },
76 + { status: 200, githubErrorTypes: ["INTERNAL"], reason: "failed" },
77 + { status: 200, githubErrorTypes: ["FORBIDDEN", "NOT_FOUND"], reason: "forbidden" },
78 + { status: 200, githubErrorTypes: ["FORBIDDEN", "INTERNAL"], reason: "failed" },
79 + { status: 200, githubErrorTypes: ["FORBIDDEN", null], reason: "failed" },
80 +])("classifies GitHub response $status as $reason", ({ status, githubErrorTypes, reason }) =>
81 + Effect.gen(function* () {
82 + const provider = yield* make.pipe(
83 + Effect.provide(
84 + Layer.mock(GitHubPullRequestApi.GitHubPullRequestApi)({
85 + revalidateChecks: (_input, read) => read,
86 + getPullRequestDetail: () =>
87 + Effect.fail(
88 + new GitHubRequestPolicy.GitHubApiResponseError({
89 + host: "github.com",
90 + operation: "detail",
91 + status,
92 + githubErrorTypes,
93 + }),
94 + ),
95 + }),
96 + ),
97 + );
98 + const error = yield* provider
99 + .getChangeRequest({
100 + cwd: "/w",
101 + repository: "acme/web",
102 + host: "github.com",
103 + number: 7,
104 + })
105 + .pipe(Effect.flip);
106 + expect(error.reason).toBe(reason);
107 + }),
108 +);
109 +
110 it.effect("advertises standalone branch deletion for source repository writers", () =>
111 Effect.gen(function* () {
112 let action: string | undefined;
packages/source-control-github/src/server/GitHubPullRequestProvider.ts
+9
@@ -120,6 +120,15 @@ function gitHubProviderFailure(
120 };
121 case "SourceControlRateLimitPausedError":
122 return { reason: "rate-limited", retryAt: error.retryAt };
123 + case "GitHubApiResponseError":
124 + return {
125 + reason:
126 + error.status === 403 ||
127 + (error.githubErrorTypes?.includes("FORBIDDEN") &&
128 + error.githubErrorTypes.every((type) => type === "FORBIDDEN" || type === "NOT_FOUND"))
129 + ? "forbidden"
130 + : "failed",
131 + };
132 case "GitHubApiNotFoundError":
133 return error.credential === undefined || error.credential === "app"
134 ? { reason: "not-found" }
packages/source-control-github/src/server/GitHubRequestPolicy.ts
+1
@@ -85,6 +85,7 @@ export class GitHubApiResponseError extends Schema.TaggedError<GitHubApiResponse
85 operation: Schema.String,
86 status: Schema.Int,
87 githubErrors: Schema.optionalKey(Schema.Array(Schema.String)),
88 + githubErrorTypes: Schema.optionalKey(Schema.Array(Schema.NullOr(Schema.String))),
89 },
90 ) {
91 override get message(): string {
packages/source-control-github/src/server/gitHubGraphQlClient.test.ts
+3 -1
@@ -302,6 +302,7 @@ describe("call", () => {
302 errors: [
303 { type: "NOT_FOUND", message: "Could not resolve" },
304 { type: "FORBIDDEN", message: "Resource not accessible" },
305 + { message: "Unknown error" },
306 ],
307 }),
308 );
@@ -310,7 +311,8 @@ describe("call", () => {
311 expect(error).toMatchObject({
312 _tag: "GitHubApiResponseError",
313 status: 200,
313 - githubErrors: ["Could not resolve", "Resource not accessible"],
314 + githubErrors: ["Could not resolve", "Resource not accessible", "Unknown error"],
315 + githubErrorTypes: ["NOT_FOUND", "FORBIDDEN", null],
316 });
317 }).pipe(Effect.provide(layer));
318 },
packages/source-control-github/src/server/gitHubGraphQlClient.ts
+3
@@ -248,6 +248,9 @@ const fromClientError = (
248 ...context,
249 status: reason.status ?? 200,
250 githubErrors: reason.errors.map((graphQlError) => graphQlError.message),
251 + githubErrorTypes: reason.errors.map((graphQlError) =>
252 + typeof graphQlError["type"] === "string" ? graphQlError["type"] : null,
253 + ),
254 });
255 };
256