The best way to control Claude Code, Codex and more from any computer or phone
Mirrored from https://github.com/Camo67/t3code.git
Synced Nov 21, 09:45 UTC
fix(server): pull request links whose reads keep failing back off instead of re-reading every minute (#31969)
Julius Marminge committed
Nov 16, 2026 at 13:42 UTC
a10f9e41d378de696f1bb2f1c69580c5df59af18
2 files changed
+106
-1
apps/server/src/orchestration-v2/PullRequestSyncReactor.test.ts
+88
@@ -325,6 +325,24 @@ const sweepAgain = Effect.fn("sweepPullRequestSyncHarness")(function* (
325
yield* reactor.drain;
326
});
327
328
+const sweepFor = Effect.fn("sweepPullRequestSyncHarnessFor")(function* (
329
+ fixture: Harness,
330
+ reactor: PullRequestSyncReactor.PullRequestSyncReactor["Service"],
331
+ minutes: number,
332
+) {
333
+ for (let minute = 0; minute < minutes; minute++) yield* sweepAgain(fixture, reactor);
334
+});
335
+
336
+const failingGitLabLink = makeLink(13, null, {
337
+ host: "gitlab.com",
338
+ url: "https://gitlab.com/owner/repository/-/merge_requests/13",
339
+});
340
+
341
+const gitLabCliFailure = new PullRequestOperationError({
342
+ operation: "summary",
343
+ detail: "GitLab CLI command failed",
344
+});
345
+
346
function runUpdated(
347
threadId: ThreadId,
348
status: OrchestrationV2Run["status"],
@@ -1839,6 +1857,76 @@ describe("PullRequestSyncReactor", () => {
1857
}),
1858
);
1859
1860
+ it.effect(
1861
+ "caps the failed-read backoff at the slow cadence and restarts it after a success",
1862
+ () =>
1863
+ Effect.scoped(
1864
+ Effect.gen(function* () {
1865
+ yield* TestClock.setTime(Date.parse(NOW));
1866
+ const failing = yield* Ref.make(true);
1867
+ const fixture = yield* makeHarness({
1868
+ snapshot: makeSnapshot([makeThread("one", { pullRequests: [failingGitLabLink] })]),
1869
+ summary: (input) =>
1870
+ Ref.get(failing).pipe(
1871
+ Effect.flatMap((fails) =>
1872
+ fails ? Effect.fail(gitLabCliFailure) : Effect.succeed(makeSummary(input)),
1873
+ ),
1874
+ ),
1875
+ });
1876
+ yield* Effect.gen(function* () {
1877
+ const reactor = yield* startAndSweep(fixture);
1878
+ const readMinutes = [0];
1879
+ const sweepMinutes = Effect.fnUntraced(function* (from: number, to: number) {
1880
+ for (let minute = from; minute <= to; minute++) {
1881
+ const before = (yield* Ref.get(fixture.summaryCalls)).length;
1882
+ yield* sweepAgain(fixture, reactor);
1883
+ if ((yield* Ref.get(fixture.summaryCalls)).length > before)
1884
+ readMinutes.push(minute);
1885
+ }
1886
+ });
1887
+
1888
+ yield* sweepMinutes(1, 60);
1889
+ assert.deepStrictEqual(readMinutes, [0, 1, 3, 7, 15, 30, 45, 60]);
1890
+
1891
+ yield* Ref.set(failing, false);
1892
+ yield* sweepMinutes(61, 75);
1893
+ yield* Ref.set(failing, true);
1894
+ readMinutes.length = 0;
1895
+ yield* sweepMinutes(76, 80);
1896
+ assert.deepStrictEqual(readMinutes, [76, 77, 79]);
1897
+ }).pipe(Effect.provide(fixture.layer));
1898
+ }),
1899
+ ),
1900
+ );
1901
+
1902
+ it.effect("reads a backed-off link at once when a refresh asks, then keeps backing off", () =>
1903
+ Effect.scoped(
1904
+ Effect.gen(function* () {
1905
+ yield* TestClock.setTime(Date.parse(NOW));
1906
+ const fixture = yield* makeHarness({
1907
+ snapshot: makeSnapshot([makeThread("one", { pullRequests: [failingGitLabLink] })]),
1908
+ summary: () => Effect.fail(gitLabCliFailure),
1909
+ });
1910
+ yield* Effect.gen(function* () {
1911
+ const reactor = yield* startAndSweep(fixture);
1912
+ yield* sweepFor(fixture, reactor, 4);
1913
+ const backedOffReads = (yield* Ref.get(fixture.summaryCalls)).length;
1914
+
1915
+ yield* reactor.requestSync(failingGitLabLink);
1916
+ yield* Queue.take(fixture.snapshotReads);
1917
+ yield* reactor.drain;
1918
+ assert.strictEqual((yield* Ref.get(fixture.summaryCalls)).length, backedOffReads + 1);
1919
+
1920
+ // The refresh failed at minute 4 with the backoff doubled to 8 minutes.
1921
+ yield* sweepFor(fixture, reactor, 7);
1922
+ assert.strictEqual((yield* Ref.get(fixture.summaryCalls)).length, backedOffReads + 1);
1923
+ yield* sweepAgain(fixture, reactor);
1924
+ assert.strictEqual((yield* Ref.get(fixture.summaryCalls)).length, backedOffReads + 2);
1925
+ }).pipe(Effect.provide(fixture.layer));
1926
+ }),
1927
+ ),
1928
+ );
1929
+
1930
it.effect.each([
1931
"gh pr merge 7 --squash 2>&1 | tail -3",
1932
"cafe pr merge 7 --squash",
apps/server/src/orchestration-v2/PullRequestSyncReactor.ts
+18
-1
@@ -48,6 +48,8 @@ export const QUIET_SYNC_INTERVAL_MS = 5 * 60 * 1_000;
48
// A pull request closed this long ago is rarely reopened, so the sweep stops reading it, as it
49
// does merged ones. A reader that sees it open, or a requested refresh, still reads it at once.
50
const CLOSED_FINAL_AFTER_MS = 24 * 60 * 60 * 1_000;
51
+// A link whose host read failed waits a sweep, then twice as long after each further failure.
52
+const FAILED_READ_BACKOFF_MS = 60 * 1_000;
53
54
const isChangeRequestCloseCommand = (command: string) =>
55
sourceControlDefinitions.definitions.some(
@@ -369,6 +371,7 @@ export const make = Effect.gen(function* () {
371
// linking dozens of pull requests) is read together and shares the summary batches.
372
let requestedSweepQueued = false;
373
const retryStacks = new Set<string>();
374
+ const failedReads = new Map<string, { readonly retryAtMs: number; readonly backoffMs: number }>();
375
// Group keys the host answered not-found for, or this server cannot read (no CLI, signed out,
376
// unsupported host); they are read on the slow cadence until a read succeeds.
377
const missing = new Set<string>();
@@ -376,6 +379,7 @@ export const make = Effect.gen(function* () {
379
const pausedUntil = new Map<string, number>();
380
381
const isDue = (key: string, entries: ReadonlyArray<LinkEntry>, nowMs: number): boolean => {
382
+ if ((failedReads.get(key)?.retryAtMs ?? 0) > nowMs) return false;
383
if (requested.has(key)) return true;
384
const active = entries.filter((entry) => isUnsettled(entry.thread));
385
if (active.length === 0) return false;
@@ -425,12 +429,17 @@ export const make = Effect.gen(function* () {
429
430
// A host-level refresh reaches every project, each retaining its own pending retry.
431
for (const [key, entries] of groups) {
428
- if (pendingRequests.has(threadPullRequestKeyOf(entries[0]!.link))) requested.add(key);
432
+ if (!pendingRequests.has(threadPullRequestKeyOf(entries[0]!.link))) continue;
433
+ requested.add(key);
434
+ // A new request reads at once; a failure after it backs off further, not from the start.
435
+ const failed = failedReads.get(key);
436
+ if (failed !== undefined) failedReads.set(key, { ...failed, retryAtMs: nowMs });
437
}
438
pendingRequests.clear();
439
440
for (const key of lastSyncedAt.keys()) if (!groups.has(key)) lastSyncedAt.delete(key);
441
for (const key of retryStacks) if (!groups.has(key)) retryStacks.delete(key);
442
+ for (const key of failedReads.keys()) if (!groups.has(key)) failedReads.delete(key);
443
for (const key of missing) if (!groups.has(key)) missing.delete(key);
444
for (const key of requested.keys()) if (!groups.has(key)) requested.delete(key);
445
@@ -599,6 +608,7 @@ export const make = Effect.gen(function* () {
608
}
609
// The host answered, so the cadence clock ticks even if a dispatch below is rejected.
610
lastSyncedAt.set(key, nowMs);
611
+ failedReads.delete(key);
612
requested.delete(key);
613
yield* Effect.forEach(
614
entries,
@@ -624,6 +634,13 @@ export const make = Effect.gen(function* () {
634
const retryAt = rateLimitRetryAt(cause);
635
if (retryAt !== undefined) {
636
pausedUntil.set(pauseKey, Math.max(retryAt, pausedUntil.get(pauseKey) ?? 0));
637
+ } else {
638
+ const previous = failedReads.get(key);
639
+ const backoffMs =
640
+ previous === undefined
641
+ ? FAILED_READ_BACKOFF_MS
642
+ : Math.min(previous.backoffMs * 2, SLOW_SYNC_INTERVAL_MS);
643
+ failedReads.set(key, { retryAtMs: nowMs + backoffMs, backoffMs });
644
}
645
const reason = skipReason(cause);
646
const skip = skips.get(reason);