@devries-cameron20 / t3code Mirror · read-only / commits / dd14848153
Fork 0

The best way to control Claude Code, Codex and more from any computer or phone

Mirrored from https://github.com/Camo67/t3code.git

Synced Nov 22, 12:00 UTC

perf(server): idle servers run 100x fewer background queries and 4.8M fewer spans a day (#27098)

Co-authored-by: Claude Opus 5.5 <noreply@anthropic.com>

Julius Marminge committed Oct 29, 2026 at 13:43 UTC dd148481531a0a090780a2ade20588566db099a2
7 files changed +262 -25
apps/server/src/orchestration-v2/ThreadSettlementService.test.ts
+23
@@ -842,6 +842,29 @@ describe("ThreadSettlementServiceV2 worker", () => {
842 ),
843 );
844
845 + it.effect("stops settling the rest of a sweep once auto-settlement is turned off", () =>
846 + Effect.scoped(
847 + Effect.gen(function* () {
848 + yield* TestClock.setTime(Date.parse(NOW));
849 + // Sweeps settle 8 threads at a time, so the ninth starts after a dispatch turned it off.
850 + const threads = Array.from({ length: 9 }, (_, index) => makeThread(`idle-${index}`));
851 + let disable: Effect.Effect<unknown> = Effect.void;
852 + const fixture = yield* makeHarness({
853 + snapshot: makeSnapshot(threads),
854 + settings: { ...DEFAULT_SERVER_SETTINGS, sidebarAutoSettleAfterDays: 3 },
855 + onDispatch: () => disable,
856 + });
857 + disable = fixture.updateSettings({ sidebarAutoSettleAfterDays: null });
858 + yield* Effect.gen(function* () {
859 + const service = yield* ThreadSettlementService.ThreadSettlementServiceV2;
860 + yield* startHarness(service, fixture.activation, fixture.snapshotReads);
861 + const settled = (yield* Ref.get(fixture.commands)).map((command) => command.threadId);
862 + expect(settled).not.toContain(threads[8]!.id);
863 + }).pipe(Effect.provide(fixture.layer));
864 + }),
865 + ),
866 + );
867 +
868 it.effect("reevaluates immediately after a pull request merge", () =>
869 Effect.scoped(
870 Effect.gen(function* () {
apps/server/src/orchestration-v2/ThreadSettlementService.ts
+6 -3
@@ -321,8 +321,10 @@ export const make = Effect.gen(function* () {
321 // Merge events resweep all candidates because unlinked threads settle through branch lookup.
322 const candidates = threads.filter((thread) => isAutoSettlementCandidate(thread, nowMs));
323
324 - const settleThread = Effect.fn("ThreadSettlementServiceV2.settleThread")(
324 + // Runs per candidate thread, thousands per sweep on busy servers; the sweep owns the span.
325 + const settleThread = Effect.fnUntraced(
326 function* (thread: (typeof candidates)[number], pullRequest: SettlementPullRequest | null) {
327 + // thread.auto-settle does not recheck settings, so read them per decision; reads are cached.
328 const currentSettings = resolveProjectSettings(
329 yield* settingsService.getSettings,
330 thread.projectId,
@@ -414,7 +416,8 @@ export const make = Effect.gen(function* () {
416 };
417 const groups = Map.groupBy(lookupCandidates, lookupKey);
418
417 - const wouldSettle = Effect.fn("ThreadSettlementServiceV2.wouldSettle")(function* (
419 + // wouldSettle and pullRequestFor run per lookup group; the git and pull request reads they make carry the spans.
420 + const wouldSettle = Effect.fnUntraced(function* (
421 group: ReadonlyArray<(typeof lookupCandidates)[number]>,
422 pullRequest: SettlementPullRequest,
423 ) {
@@ -435,7 +438,7 @@ export const make = Effect.gen(function* () {
438 });
439 });
440
438 - const pullRequestFor = Effect.fn("ThreadSettlementServiceV2.pullRequestFor")(function* (
441 + const pullRequestFor = Effect.fnUntraced(function* (
442 group: ReadonlyArray<(typeof lookupCandidates)[number]>,
443 ) {
444 const thread = group[0]!;
apps/server/src/persistence/DatabaseStorage.ts
+18 -6
@@ -49,17 +49,30 @@ const make = Effect.gen(function* () {
49 const doctorTargetsThisDatabase =
50 path.resolve(config.dbPath) ===
51 path.resolve(config.baseDir, "userdata", path.basename(config.dbPath));
52 + // Only VACUUM changes these, and doctor refuses to VACUUM while a server has the database open.
53 + let layout: { readonly pageSize: number; readonly incrementalVacuum: boolean } | undefined;
54 + const readLayout = Effect.suspend(() =>
55 + layout !== undefined
56 + ? Effect.succeed(layout)
57 + : Effect.gen(function* () {
58 + const size = yield* sql<{ readonly page_size: number }>`PRAGMA page_size`;
59 + const mode = yield* sql<{ readonly auto_vacuum: number }>`PRAGMA auto_vacuum`;
60 + layout = {
61 + pageSize: size[0]?.page_size ?? 0,
62 + incrementalVacuum: mode[0]?.auto_vacuum === 2,
63 + };
64 + return layout;
65 + }),
66 + );
67 const publishStatus = Effect.gen(function* () {
68 if (!doctorTargetsThisDatabase) return;
69 + const { pageSize, incrementalVacuum } = yield* readLayout;
70 const pages = yield* sql<{ readonly page_count: number }>`PRAGMA page_count`;
71 const free = yield* sql<{ readonly freelist_count: number }>`PRAGMA freelist_count`;
56 - const size = yield* sql<{ readonly page_size: number }>`PRAGMA page_size`;
57 - const mode = yield* sql<{ readonly auto_vacuum: number }>`PRAGMA auto_vacuum`;
58 - const pageSize = size[0]?.page_size ?? 0;
72 const next: DatabaseStorageStatus = {
73 databaseBytes: (pages[0]?.page_count ?? 0) * pageSize,
74 reclaimableBytes: (free[0]?.freelist_count ?? 0) * pageSize,
62 - incrementalVacuumEnabled: mode[0]?.auto_vacuum === 2,
75 + incrementalVacuumEnabled: incrementalVacuum,
76 compactCommand,
77 };
78 // Every set reaches each open Storage page; most passes change nothing.
@@ -71,8 +84,7 @@ const make = Effect.gen(function* () {
84 );
85
86 const reclaim = Effect.gen(function* () {
74 - const mode = yield* sql<{ readonly auto_vacuum: number }>`PRAGMA auto_vacuum`;
75 - if (mode[0]?.auto_vacuum !== 2) return;
87 + if (!(yield* readLayout).incrementalVacuum) return;
88 // The driver steps once per statement; incremental_vacuum frees one page per step. Yield between pages.
89 for (let page = 0; page < RECLAIM_PAGES_PER_PASS; page++) {
90 const free = yield* sql<{ readonly freelist_count: number }>`PRAGMA freelist_count`;
apps/server/src/scheduledTasks/ScheduledTaskService.schedule.test.ts
+62
@@ -3,6 +3,7 @@ import * as NodeCrypto from "@effect/platform-node/NodeCrypto";
3 import { expect, it } from "@effect/vitest";
4 import { ScheduledTaskUpsertInput } from "@t3tools/contracts";
5 import * as DateTime from "effect/DateTime";
6 +import * as Deferred from "effect/Deferred";
7 import * as Effect from "effect/Effect";
8 import * as Layer from "effect/Layer";
9 import * as Schema from "effect/Schema";
@@ -167,3 +168,64 @@ it.effect("keeps the task list order when a toggle or edit rewrites updated_at",
168 }).pipe(Effect.provide(ScheduledTaskService.layer.pipe(Layer.provide(layerDependencies))));
169 }).pipe(Effect.provide(SqlitePersistence.layerMemory)),
170 );
171 +
172 +it.effect(
173 + "idle polls still wake for tasks saved later, recorded due times, and beside far-future tasks",
174 + () =>
175 + Effect.gen(function* () {
176 + yield* TestClock.setTime(Date.parse("2026-07-01T09:00:00.000Z"));
177 + const launched = yield* Deferred.make<string>();
178 + const layerDependencies = Layer.mergeAll(
179 + NodeCrypto.layer,
180 + Scheduler.layer,
181 + Layer.mock(ThreadLaunchService.ThreadLaunchService)({
182 + launch: (input) =>
183 + Deferred.succeed(launched, input.title).pipe(
184 + Effect.as({ threadId: "thread-idle-poll", resumed: false } as never),
185 + ),
186 + }),
187 + Layer.mock(ManagedProjectFolders.ManagedProjectFolders)({
188 + namedProjectsRoot: "/projects",
189 + isScratchProject: () => Effect.succeed(false),
190 + }),
191 + Layer.mock(ThreadManagementService.ThreadManagementService)({}),
192 + Layer.mock(SecretRequests.SecretRequests)({}),
193 + );
194 + yield* Effect.gen(function* () {
195 + const service = yield* ScheduledTaskService.ScheduledTaskService;
196 + // The first poll finds no tasks and remembers that nothing is due.
197 + yield* TestClock.adjust("6 seconds");
198 + // Its year-10000+ next run sorts first as text; it must not hide the task below.
199 + yield* service.upsert(
200 + yield* decodeUpsertInput({
201 + title: "Far future",
202 + prompt: "Review the open pull requests.",
203 + enabled: true,
204 + schedule: { type: "interval", everyMs: 3e14 },
205 + projectId: "project-idle-poll",
206 + workspaceStrategy: { type: "root" },
207 + modelSelection: { instanceId: "codex", model: "gpt-5.4" },
208 + runtimeMode: "full-access",
209 + interactionMode: "default",
210 + }),
211 + );
212 + const created = yield* service.upsert(
213 + yield* decodeUpsertInput({
214 + title: "Idle poll",
215 + prompt: "Review the open pull requests.",
216 + enabled: true,
217 + schedule: { type: "interval", everyMs: 60_000 },
218 + projectId: "project-idle-poll",
219 + workspaceStrategy: { type: "root" },
220 + modelSelection: { instanceId: "codex", model: "gpt-5.4" },
221 + runtimeMode: "full-access",
222 + interactionMode: "default",
223 + }),
224 + );
225 + // This poll records the new task's due time; the later ones must still fire it.
226 + yield* TestClock.adjust("6 seconds");
227 + yield* TestClock.adjust("60 seconds");
228 + expect(yield* Deferred.await(launched)).toBe(created.task.title);
229 + }).pipe(Effect.provide(ScheduledTaskService.layer.pipe(Layer.provide(layerDependencies))));
230 + }).pipe(Effect.provide(SqlitePersistence.layerMemory)),
231 +);
apps/server/src/scheduledTasks/ScheduledTaskService.ts
+69 -1
@@ -396,6 +396,35 @@ export const listDueTasks = Effect.fn("ScheduledTaskService.listDueTasks")(funct
396 return tasks;
397 });
398
399 +// Backstop for a write path that forgets to invalidate; the tick still runs every few seconds.
400 +const DUE_HINT_MAX_AGE_MS = 10 * 60 * 1_000;
401 +
402 +/**
403 + * Remembers when a poller next has work so idle scheduler ticks skip SQL.
404 + * `invalidate` after every write that can move work earlier; a tick records
405 + * only if no write happened since it read `version`.
406 + */
407 +const makeDueHint = () => {
408 + let version = 0;
409 + let hint: { readonly dueAtMs: number; readonly checkedAtMs: number } | null = null;
410 + return {
411 + invalidate: Effect.sync(() => {
412 + version++;
413 + hint = null;
414 + }),
415 + version: Effect.sync(() => version),
416 + isIdle: (nowMs: number) =>
417 + Effect.sync(
418 + () =>
419 + hint !== null && hint.dueAtMs > nowMs && nowMs - hint.checkedAtMs < DUE_HINT_MAX_AGE_MS,
420 + ),
421 + record: (readVersion: number, dueAtMs: number, nowMs: number) =>
422 + Effect.sync(() => {
423 + if (readVersion === version) hint = { dueAtMs, checkedAtMs: nowMs };
424 + }),
425 + };
426 +};
427 +
428 export const layer = Layer.effect(
429 ScheduledTaskService,
430 Effect.gen(function* () {
@@ -426,7 +455,13 @@ export const layer = Layer.effect(
455 const activeRuns = yield* Ref.make<ReadonlySet<ScheduledTaskId>>(new Set());
456 // Notifications re-emit the full list; slow subscribers need only the newest dirty signal.
457 const changesPubSub = yield* PubSub.sliding<void>(1);
429 - const notifyChanged = PubSub.publish(changesPubSub, undefined).pipe(Effect.asVoid);
458 + const dueTasks = makeDueHint();
459 + const pendingWebhooks = makeDueHint();
460 + // Every scheduled_tasks write that can change next_run_at, enabled or run state notifies.
461 + const notifyChanged = dueTasks.invalidate.pipe(
462 + Effect.andThen(PubSub.publish(changesPubSub, undefined)),
463 + Effect.asVoid,
464 + );
465
466 const selectAllRows = () =>
467 sql<ScheduledTaskRow>`
@@ -972,6 +1007,28 @@ export const layer = Layer.effect(
1007
1008 const runDueTasks = Effect.fn("ScheduledTaskService.runDueTasks")(function* () {
1009 const now = yield* localNow;
1010 + const nowMs = DateTime.toEpochMillis(now);
1011 + if (yield* dueTasks.isIdle(nowMs)) return;
1012 + const readVersion = yield* dueTasks.version;
1013 + const [next] = yield* sql<{ next_run_at: string | null }>`
1014 + SELECT MIN(next_run_at) AS next_run_at FROM scheduled_tasks
1015 + WHERE enabled = 1 AND next_run_at IS NOT NULL AND last_run_status <> 'running'
1016 + `.pipe(
1017 + NodeSqliteClient.readOnly,
1018 + Effect.mapError((cause) => taskError("Could not list schedule tasks.", { cause })),
1019 + );
1020 + // Compare as text like listDueTasks: expanded years (`+011533-…`) sort first but parse far ahead.
1021 + const nextRunAt = next?.next_run_at ?? null;
1022 + if (nextRunAt === null || nextRunAt > iso(now)) {
1023 + const nextDueAt =
1024 + nextRunAt === null
1025 + ? Option.some(Infinity)
1026 + : DateTime.make(nextRunAt).pipe(Option.map(DateTime.toEpochMillis));
1027 + if (Option.isSome(nextDueAt) && nextDueAt.value > nowMs) {
1028 + yield* dueTasks.record(readVersion, nextDueAt.value, nowMs);
1029 + }
1030 + return;
1031 + }
1032 const tasks = yield* listDueTasks(now).pipe(
1033 Effect.mapError((cause) => taskError("Could not list schedule tasks.", { cause })),
1034 );
@@ -1695,6 +1752,8 @@ export const layer = Layer.effect(
1752
1753 const triggerWebhook: ScheduledTaskService["Service"]["triggerWebhook"] = (request) =>
1754 triggerWebhookUnobserved(request).pipe(
1755 + // Accepted deliveries may have queued a coalescing window.
1756 + Effect.ensuring(pendingWebhooks.invalidate),
1757 Effect.tapError(() => observeDelivery(request, "error", undefined, undefined)),
1758 Metrics.withMetrics({ timer: Metrics.webhookDeliveryDuration }),
1759 Effect.withSpan("ScheduledTaskService.triggerWebhook", {
@@ -2090,10 +2149,19 @@ export const layer = Layer.effect(
2149
2150 const dispatchPending = Effect.fn("ScheduledTaskService.dispatchPendingWebhooks")(function* () {
2151 const now = DateTime.toEpochMillis(yield* localNow);
2152 + if (yield* pendingWebhooks.isIdle(now)) return;
2153 + const readVersion = yield* pendingWebhooks.version;
2154 yield* sql`UPDATE scheduled_task_webhook_pending SET state = 'dispatching' WHERE state = 'pending' AND due_at <= ${now}`;
2155 const tasks = yield* sql<{ task_id: ScheduledTaskId }>`
2156 SELECT DISTINCT task_id FROM scheduled_task_webhook_pending WHERE state IN ('dispatching', 'running', 'completed', 'failed')
2157 `.pipe(NodeSqliteClient.readOnly);
2158 + // Workers can leave rows behind to retry, so keep polling while any are claimed.
2159 + if (tasks.length === 0) {
2160 + const [next] = yield* sql<{ due_at: number | null }>`
2161 + SELECT MIN(due_at) AS due_at FROM scheduled_task_webhook_pending WHERE state = 'pending'
2162 + `.pipe(NodeSqliteClient.readOnly);
2163 + yield* pendingWebhooks.record(readVersion, next?.due_at ?? Infinity, now);
2164 + }
2165 yield* Effect.forEach(
2166 tasks,
2167 ({ task_id: taskId }) =>
apps/server/src/serverSettings.test.ts
+51 -1
@@ -1047,6 +1047,54 @@ it.layer(NodeServices.layer)("server settings", (it) => {
1047 }).pipe(Effect.provide(layerServerSettings())),
1048 );
1049
1050 + it.effect("reads settings secrets once per settings change", () => {
1051 + let reads = 0;
1052 + const layerSecret = Layer.effect(
1053 + ServerSecretStore.ServerSecretStore,
1054 + Effect.map(ServerSecretStore.ServerSecretStore, (store) => ({
1055 + ...store,
1056 + get: (name: string) =>
1057 + Effect.suspend(() => {
1058 + reads++;
1059 + return store.get(name);
1060 + }),
1061 + })),
1062 + ).pipe(Layer.provide(ServerSecretStore.layer));
1063 + const layerSettings = ServerSettingsModule.layer.pipe(
1064 + Layer.provide(layerSecret),
1065 + Layer.provideMerge(Layer.fresh(SqlitePersistence.layerMemory)),
1066 + Layer.provideMerge(
1067 + Layer.fresh(
1068 + ServerConfig.layerTest(process.cwd(), { prefix: "t3code-settings-secret-cache-test-" }),
1069 + ),
1070 + ),
1071 + );
1072 + return Effect.gen(function* () {
1073 + const instanceId = ProviderInstanceId.make("codex_cached");
1074 + const service = yield* ServerSettingsModule.ServerSettingsService;
1075 + const save = (value: string) =>
1076 + service.updateSettings({
1077 + providerInstances: {
1078 + [instanceId]: {
1079 + driver: ProviderDriverKind.make("codex"),
1080 + environment: [{ name: "API_TOKEN", value, sensitive: true }],
1081 + config: {},
1082 + },
1083 + },
1084 + });
1085 + const token = service.getSettings.pipe(
1086 + Effect.map((settings) => settings.providerInstances[instanceId]?.environment?.[0]?.value),
1087 + );
1088 + yield* save("sk-first");
1089 + reads = 0;
1090 + assert.equal(yield* token, "sk-first");
1091 + assert.equal(yield* token, "sk-first");
1092 + assert.equal(reads, 0);
1093 + yield* save("sk-second");
1094 + assert.equal(yield* token, "sk-second");
1095 + }).pipe(Effect.provide(layerSettings));
1096 + });
1097 +
1098 it.effect("keeps the inline value on disk when secret migration fails", () => {
1099 const cause = new ServerSecretStore.SecretStorePersistError({
1100 resource: "provider environment secret",
@@ -1578,6 +1626,7 @@ it.layer(NodeServices.layer)("server settings", (it) => {
1626 const fileSystem = yield* FileSystem.FileSystem;
1627 let failRename = false;
1628 let settingsPathToFail: string | undefined;
1629 + let readDuringWrite: Effect.Effect<void> = Effect.void;
1630 const writeFailure = PlatformError.systemError({
1631 _tag: "PermissionDenied",
1632 module: "FileSystem",
@@ -1588,7 +1637,7 @@ it.layer(NodeServices.layer)("server settings", (it) => {
1637 ...fileSystem,
1638 rename: (fromPath, toPath) =>
1639 failRename && toPath === settingsPathToFail
1591 - ? Effect.fail(writeFailure)
1640 + ? readDuringWrite.pipe(Effect.andThen(Effect.fail(writeFailure)))
1641 : fileSystem.rename(fromPath, toPath),
1642 });
1643 const instanceId = ProviderInstanceId.make("antigravity_write_failure");
@@ -1610,6 +1659,7 @@ it.layer(NodeServices.layer)("server settings", (it) => {
1659 });
1660
1661 failRename = true;
1662 + readDuringWrite = Effect.ignore(serverSettings.getSettings);
1663 const failedUpdate = yield* serverSettings
1664 .updateProviderInstance({
1665 operation: "upsert",
apps/server/src/serverSettings.ts
+33 -14
@@ -851,6 +851,31 @@ const make = Effect.gen(function* () {
851 },
852 );
853
854 + // Background sweeps read settings every few seconds; reading secret files each time is wasted.
855 + // Every secret write rewrites the settings file, so a new cached object implies fresh secrets.
856 + // A read that overlaps a secret write may see files a rollback then restores, so it is not kept.
857 + let materialized: { readonly source: ServerSettings; readonly value: ServerSettings } | null =
858 + null;
859 + let secretWriteGeneration = 0;
860 + const invalidateMaterialized = Effect.sync(() => {
861 + secretWriteGeneration++;
862 + materialized = null;
863 + });
864 + const getMaterializedSettings = getSettingsFromCache.pipe(
865 + Effect.flatMap((source) => {
866 + if (materialized?.source === source) return Effect.succeed(materialized.value);
867 + const generation = secretWriteGeneration;
868 + return materializeSettingsSecrets(source).pipe(
869 + Effect.map(resolveTextGenerationProvider),
870 + Effect.tap((value) =>
871 + Effect.sync(() => {
872 + if (generation === secretWriteGeneration) materialized = { source, value };
873 + }),
874 + ),
875 + );
876 + }),
877 + );
878 +
879 const materializeChanges = (changes: Stream.Stream<ServerSettings>) =>
880 changes.pipe(
881 Stream.mapEffect((settings) =>
@@ -1026,8 +1051,9 @@ const make = Effect.gen(function* () {
1051 const updated = yield* update(current);
1052 const persisted = yield* persistSettingsSecrets(current, updated);
1053 const next = yield* normalizeServerSettings(persisted.settings);
1029 - const materialized = yield* Effect.uninterruptibleMask(() =>
1054 + const materializedNext = yield* Effect.uninterruptibleMask(() =>
1055 Effect.gen(function* () {
1056 + yield* invalidateMaterialized;
1057 const rollbackSecretChanges = yield* applySettingsSecretChanges(persisted.changes);
1058 const materializedExit = yield* Effect.exit(materializeSettingsSecrets(next));
1059 if (Exit.isFailure(materializedExit)) {
@@ -1040,22 +1066,18 @@ const make = Effect.gen(function* () {
1066 return yield* Effect.failCause(writeExit.cause);
1067 }
1068 return materializedExit.value;
1043 - }),
1069 + }).pipe(Effect.ensuring(invalidateMaterialized)),
1070 );
1071 yield* Cache.set(settingsCache, cacheKey, { settings: next, parseFailure: Option.none() });
1072 + const resolved = resolveTextGenerationProvider(materializedNext);
1073 + materialized = { source: next, value: resolved };
1074 yield* emitChange(next);
1047 - return resolveTextGenerationProvider(materialized);
1075 + return resolved;
1076 }),
1077 );
1078
1079 const withSettingsSnapshot: ServerSettingsService["Service"]["withSettingsSnapshot"] = (use) =>
1052 - writeSemaphore.withPermits(1)(
1053 - getSettingsFromCache.pipe(
1054 - Effect.flatMap(materializeSettingsSecrets),
1055 - Effect.map(resolveTextGenerationProvider),
1056 - Effect.flatMap(use),
1057 - ),
1058 - );
1080 + writeSemaphore.withPermits(1)(getMaterializedSettings.pipe(Effect.flatMap(use)));
1081
1082 const revalidateAndEmit = writeSemaphore.withPermits(1)(
1083 Effect.gen(function* () {
@@ -1160,10 +1182,7 @@ const make = Effect.gen(function* () {
1182 return {
1183 start,
1184 ready: Deferred.await(startedDeferred),
1163 - getSettings: getSettingsFromCache.pipe(
1164 - Effect.flatMap(materializeSettingsSecrets),
1165 - Effect.map(resolveTextGenerationProvider),
1166 - ),
1185 + getSettings: getMaterializedSettings,
1186 updateSettings: (patch) =>
1187 updateAndPersistSettings((current) =>
1188 Effect.succeed(applyServerSettingsPatch(current, patch)),