@devries-cameron20 / t3code Mirror · read-only / commits / a10f9e41d3
Fork 0

The best way to control Claude Code, Codex and more from any computer or phone

Mirrored from https://github.com/Camo67/t3code.git

Synced Nov 21, 09:45 UTC

fix(server): pull request links whose reads keep failing back off instead of re-reading every minute (#31969)

Julius Marminge committed Nov 16, 2026 at 13:42 UTC a10f9e41d378de696f1bb2f1c69580c5df59af18
2 files changed +106 -1
apps/server/src/orchestration-v2/PullRequestSyncReactor.test.ts
+88
@@ -325,6 +325,24 @@ const sweepAgain = Effect.fn("sweepPullRequestSyncHarness")(function* (
325 yield* reactor.drain;
326 });
327
328 +const sweepFor = Effect.fn("sweepPullRequestSyncHarnessFor")(function* (
329 + fixture: Harness,
330 + reactor: PullRequestSyncReactor.PullRequestSyncReactor["Service"],
331 + minutes: number,
332 +) {
333 + for (let minute = 0; minute < minutes; minute++) yield* sweepAgain(fixture, reactor);
334 +});
335 +
336 +const failingGitLabLink = makeLink(13, null, {
337 + host: "gitlab.com",
338 + url: "https://gitlab.com/owner/repository/-/merge_requests/13",
339 +});
340 +
341 +const gitLabCliFailure = new PullRequestOperationError({
342 + operation: "summary",
343 + detail: "GitLab CLI command failed",
344 +});
345 +
346 function runUpdated(
347 threadId: ThreadId,
348 status: OrchestrationV2Run["status"],
@@ -1839,6 +1857,76 @@ describe("PullRequestSyncReactor", () => {
1857 }),
1858 );
1859
1860 + it.effect(
1861 + "caps the failed-read backoff at the slow cadence and restarts it after a success",
1862 + () =>
1863 + Effect.scoped(
1864 + Effect.gen(function* () {
1865 + yield* TestClock.setTime(Date.parse(NOW));
1866 + const failing = yield* Ref.make(true);
1867 + const fixture = yield* makeHarness({
1868 + snapshot: makeSnapshot([makeThread("one", { pullRequests: [failingGitLabLink] })]),
1869 + summary: (input) =>
1870 + Ref.get(failing).pipe(
1871 + Effect.flatMap((fails) =>
1872 + fails ? Effect.fail(gitLabCliFailure) : Effect.succeed(makeSummary(input)),
1873 + ),
1874 + ),
1875 + });
1876 + yield* Effect.gen(function* () {
1877 + const reactor = yield* startAndSweep(fixture);
1878 + const readMinutes = [0];
1879 + const sweepMinutes = Effect.fnUntraced(function* (from: number, to: number) {
1880 + for (let minute = from; minute <= to; minute++) {
1881 + const before = (yield* Ref.get(fixture.summaryCalls)).length;
1882 + yield* sweepAgain(fixture, reactor);
1883 + if ((yield* Ref.get(fixture.summaryCalls)).length > before)
1884 + readMinutes.push(minute);
1885 + }
1886 + });
1887 +
1888 + yield* sweepMinutes(1, 60);
1889 + assert.deepStrictEqual(readMinutes, [0, 1, 3, 7, 15, 30, 45, 60]);
1890 +
1891 + yield* Ref.set(failing, false);
1892 + yield* sweepMinutes(61, 75);
1893 + yield* Ref.set(failing, true);
1894 + readMinutes.length = 0;
1895 + yield* sweepMinutes(76, 80);
1896 + assert.deepStrictEqual(readMinutes, [76, 77, 79]);
1897 + }).pipe(Effect.provide(fixture.layer));
1898 + }),
1899 + ),
1900 + );
1901 +
1902 + it.effect("reads a backed-off link at once when a refresh asks, then keeps backing off", () =>
1903 + Effect.scoped(
1904 + Effect.gen(function* () {
1905 + yield* TestClock.setTime(Date.parse(NOW));
1906 + const fixture = yield* makeHarness({
1907 + snapshot: makeSnapshot([makeThread("one", { pullRequests: [failingGitLabLink] })]),
1908 + summary: () => Effect.fail(gitLabCliFailure),
1909 + });
1910 + yield* Effect.gen(function* () {
1911 + const reactor = yield* startAndSweep(fixture);
1912 + yield* sweepFor(fixture, reactor, 4);
1913 + const backedOffReads = (yield* Ref.get(fixture.summaryCalls)).length;
1914 +
1915 + yield* reactor.requestSync(failingGitLabLink);
1916 + yield* Queue.take(fixture.snapshotReads);
1917 + yield* reactor.drain;
1918 + assert.strictEqual((yield* Ref.get(fixture.summaryCalls)).length, backedOffReads + 1);
1919 +
1920 + // The refresh failed at minute 4 with the backoff doubled to 8 minutes.
1921 + yield* sweepFor(fixture, reactor, 7);
1922 + assert.strictEqual((yield* Ref.get(fixture.summaryCalls)).length, backedOffReads + 1);
1923 + yield* sweepAgain(fixture, reactor);
1924 + assert.strictEqual((yield* Ref.get(fixture.summaryCalls)).length, backedOffReads + 2);
1925 + }).pipe(Effect.provide(fixture.layer));
1926 + }),
1927 + ),
1928 + );
1929 +
1930 it.effect.each([
1931 "gh pr merge 7 --squash 2>&1 | tail -3",
1932 "cafe pr merge 7 --squash",
apps/server/src/orchestration-v2/PullRequestSyncReactor.ts
+18 -1
@@ -48,6 +48,8 @@ export const QUIET_SYNC_INTERVAL_MS = 5 * 60 * 1_000;
48 // A pull request closed this long ago is rarely reopened, so the sweep stops reading it, as it
49 // does merged ones. A reader that sees it open, or a requested refresh, still reads it at once.
50 const CLOSED_FINAL_AFTER_MS = 24 * 60 * 60 * 1_000;
51 +// A link whose host read failed waits a sweep, then twice as long after each further failure.
52 +const FAILED_READ_BACKOFF_MS = 60 * 1_000;
53
54 const isChangeRequestCloseCommand = (command: string) =>
55 sourceControlDefinitions.definitions.some(
@@ -369,6 +371,7 @@ export const make = Effect.gen(function* () {
371 // linking dozens of pull requests) is read together and shares the summary batches.
372 let requestedSweepQueued = false;
373 const retryStacks = new Set<string>();
374 + const failedReads = new Map<string, { readonly retryAtMs: number; readonly backoffMs: number }>();
375 // Group keys the host answered not-found for, or this server cannot read (no CLI, signed out,
376 // unsupported host); they are read on the slow cadence until a read succeeds.
377 const missing = new Set<string>();
@@ -376,6 +379,7 @@ export const make = Effect.gen(function* () {
379 const pausedUntil = new Map<string, number>();
380
381 const isDue = (key: string, entries: ReadonlyArray<LinkEntry>, nowMs: number): boolean => {
382 + if ((failedReads.get(key)?.retryAtMs ?? 0) > nowMs) return false;
383 if (requested.has(key)) return true;
384 const active = entries.filter((entry) => isUnsettled(entry.thread));
385 if (active.length === 0) return false;
@@ -425,12 +429,17 @@ export const make = Effect.gen(function* () {
429
430 // A host-level refresh reaches every project, each retaining its own pending retry.
431 for (const [key, entries] of groups) {
428 - if (pendingRequests.has(threadPullRequestKeyOf(entries[0]!.link))) requested.add(key);
432 + if (!pendingRequests.has(threadPullRequestKeyOf(entries[0]!.link))) continue;
433 + requested.add(key);
434 + // A new request reads at once; a failure after it backs off further, not from the start.
435 + const failed = failedReads.get(key);
436 + if (failed !== undefined) failedReads.set(key, { ...failed, retryAtMs: nowMs });
437 }
438 pendingRequests.clear();
439
440 for (const key of lastSyncedAt.keys()) if (!groups.has(key)) lastSyncedAt.delete(key);
441 for (const key of retryStacks) if (!groups.has(key)) retryStacks.delete(key);
442 + for (const key of failedReads.keys()) if (!groups.has(key)) failedReads.delete(key);
443 for (const key of missing) if (!groups.has(key)) missing.delete(key);
444 for (const key of requested.keys()) if (!groups.has(key)) requested.delete(key);
445
@@ -599,6 +608,7 @@ export const make = Effect.gen(function* () {
608 }
609 // The host answered, so the cadence clock ticks even if a dispatch below is rejected.
610 lastSyncedAt.set(key, nowMs);
611 + failedReads.delete(key);
612 requested.delete(key);
613 yield* Effect.forEach(
614 entries,
@@ -624,6 +634,13 @@ export const make = Effect.gen(function* () {
634 const retryAt = rateLimitRetryAt(cause);
635 if (retryAt !== undefined) {
636 pausedUntil.set(pauseKey, Math.max(retryAt, pausedUntil.get(pauseKey) ?? 0));
637 + } else {
638 + const previous = failedReads.get(key);
639 + const backoffMs =
640 + previous === undefined
641 + ? FAILED_READ_BACKOFF_MS
642 + : Math.min(previous.backoffMs * 2, SLOW_SYNC_INTERVAL_MS);
643 + failedReads.set(key, { retryAtMs: nowMs + backoffMs, backoffMs });
644 }
645 const reason = skipReason(cause);
646 const skip = skips.get(reason);