The best way to control Claude Code, Codex and more from any computer or phone
perf(server): idle servers run 100x fewer background queries and 4.8M fewer spans a day (#27098)
Co-authored-by: Claude Opus 5.5 <noreply@anthropic.com>
Julius Marminge committed
Oct 29, 2026 at 13:43 UTC
dd148481531a0a090780a2ade20588566db099a2
7 files changed
+262
-25
apps/server/src/orchestration-v2/ThreadSettlementService.test.ts
+23
@@ -842,6 +842,29 @@ describe("ThreadSettlementServiceV2 worker", () => {
842
),
843
);
844
845
+ it.effect("stops settling the rest of a sweep once auto-settlement is turned off", () =>
846
+ Effect.scoped(
847
+ Effect.gen(function* () {
848
+ yield* TestClock.setTime(Date.parse(NOW));
849
+ // Sweeps settle 8 threads at a time, so the ninth starts after a dispatch turned it off.
850
+ const threads = Array.from({ length: 9 }, (_, index) => makeThread(`idle-${index}`));
851
+ let disable: Effect.Effect<unknown> = Effect.void;
852
+ const fixture = yield* makeHarness({
853
+ snapshot: makeSnapshot(threads),
854
+ settings: { ...DEFAULT_SERVER_SETTINGS, sidebarAutoSettleAfterDays: 3 },
855
+ onDispatch: () => disable,
856
+ });
857
+ disable = fixture.updateSettings({ sidebarAutoSettleAfterDays: null });
858
+ yield* Effect.gen(function* () {
859
+ const service = yield* ThreadSettlementService.ThreadSettlementServiceV2;
860
+ yield* startHarness(service, fixture.activation, fixture.snapshotReads);
861
+ const settled = (yield* Ref.get(fixture.commands)).map((command) => command.threadId);
862
+ expect(settled).not.toContain(threads[8]!.id);
863
+ }).pipe(Effect.provide(fixture.layer));
864
+ }),
865
+ ),
866
+ );
867
+
868
it.effect("reevaluates immediately after a pull request merge", () =>
869
Effect.scoped(
870
Effect.gen(function* () {
apps/server/src/orchestration-v2/ThreadSettlementService.ts
+6
-3
@@ -321,8 +321,10 @@ export const make = Effect.gen(function* () {
321
// Merge events resweep all candidates because unlinked threads settle through branch lookup.
322
const candidates = threads.filter((thread) => isAutoSettlementCandidate(thread, nowMs));
323
324
- const settleThread = Effect.fn("ThreadSettlementServiceV2.settleThread")(
324
+ // Runs per candidate thread, thousands per sweep on busy servers; the sweep owns the span.
325
+ const settleThread = Effect.fnUntraced(
326
function* (thread: (typeof candidates)[number], pullRequest: SettlementPullRequest | null) {
327
+ // thread.auto-settle does not recheck settings, so read them per decision; reads are cached.
328
const currentSettings = resolveProjectSettings(
329
yield* settingsService.getSettings,
330
thread.projectId,
@@ -414,7 +416,8 @@ export const make = Effect.gen(function* () {
416
};
417
const groups = Map.groupBy(lookupCandidates, lookupKey);
418
417
- const wouldSettle = Effect.fn("ThreadSettlementServiceV2.wouldSettle")(function* (
419
+ // wouldSettle and pullRequestFor run per lookup group; the git and pull request reads they make carry the spans.
420
+ const wouldSettle = Effect.fnUntraced(function* (
421
group: ReadonlyArray<(typeof lookupCandidates)[number]>,
422
pullRequest: SettlementPullRequest,
423
) {
@@ -435,7 +438,7 @@ export const make = Effect.gen(function* () {
438
});
439
});
440
438
- const pullRequestFor = Effect.fn("ThreadSettlementServiceV2.pullRequestFor")(function* (
441
+ const pullRequestFor = Effect.fnUntraced(function* (
442
group: ReadonlyArray<(typeof lookupCandidates)[number]>,
443
) {
444
const thread = group[0]!;
apps/server/src/persistence/DatabaseStorage.ts
+18
-6
@@ -49,17 +49,30 @@ const make = Effect.gen(function* () {
49
const doctorTargetsThisDatabase =
50
path.resolve(config.dbPath) ===
51
path.resolve(config.baseDir, "userdata", path.basename(config.dbPath));
52
+ // Only VACUUM changes these, and doctor refuses to VACUUM while a server has the database open.
53
+ let layout: { readonly pageSize: number; readonly incrementalVacuum: boolean } | undefined;
54
+ const readLayout = Effect.suspend(() =>
55
+ layout !== undefined
56
+ ? Effect.succeed(layout)
57
+ : Effect.gen(function* () {
58
+ const size = yield* sql<{ readonly page_size: number }>`PRAGMA page_size`;
59
+ const mode = yield* sql<{ readonly auto_vacuum: number }>`PRAGMA auto_vacuum`;
60
+ layout = {
61
+ pageSize: size[0]?.page_size ?? 0,
62
+ incrementalVacuum: mode[0]?.auto_vacuum === 2,
63
+ };
64
+ return layout;
65
+ }),
66
+ );
67
const publishStatus = Effect.gen(function* () {
68
if (!doctorTargetsThisDatabase) return;
69
+ const { pageSize, incrementalVacuum } = yield* readLayout;
70
const pages = yield* sql<{ readonly page_count: number }>`PRAGMA page_count`;
71
const free = yield* sql<{ readonly freelist_count: number }>`PRAGMA freelist_count`;
56
- const size = yield* sql<{ readonly page_size: number }>`PRAGMA page_size`;
57
- const mode = yield* sql<{ readonly auto_vacuum: number }>`PRAGMA auto_vacuum`;
58
- const pageSize = size[0]?.page_size ?? 0;
72
const next: DatabaseStorageStatus = {
73
databaseBytes: (pages[0]?.page_count ?? 0) * pageSize,
74
reclaimableBytes: (free[0]?.freelist_count ?? 0) * pageSize,
62
- incrementalVacuumEnabled: mode[0]?.auto_vacuum === 2,
75
+ incrementalVacuumEnabled: incrementalVacuum,
76
compactCommand,
77
};
78
// Every set reaches each open Storage page; most passes change nothing.
@@ -71,8 +84,7 @@ const make = Effect.gen(function* () {
84
);
85
86
const reclaim = Effect.gen(function* () {
74
- const mode = yield* sql<{ readonly auto_vacuum: number }>`PRAGMA auto_vacuum`;
75
- if (mode[0]?.auto_vacuum !== 2) return;
87
+ if (!(yield* readLayout).incrementalVacuum) return;
88
// The driver steps once per statement; incremental_vacuum frees one page per step. Yield between pages.
89
for (let page = 0; page < RECLAIM_PAGES_PER_PASS; page++) {
90
const free = yield* sql<{ readonly freelist_count: number }>`PRAGMA freelist_count`;
apps/server/src/scheduledTasks/ScheduledTaskService.schedule.test.ts
+62
@@ -3,6 +3,7 @@ import * as NodeCrypto from "@effect/platform-node/NodeCrypto";
3
import { expect, it } from "@effect/vitest";
4
import { ScheduledTaskUpsertInput } from "@t3tools/contracts";
5
import * as DateTime from "effect/DateTime";
6
+import * as Deferred from "effect/Deferred";
7
import * as Effect from "effect/Effect";
8
import * as Layer from "effect/Layer";
9
import * as Schema from "effect/Schema";
@@ -167,3 +168,64 @@ it.effect("keeps the task list order when a toggle or edit rewrites updated_at",
168
}).pipe(Effect.provide(ScheduledTaskService.layer.pipe(Layer.provide(layerDependencies))));
169
}).pipe(Effect.provide(SqlitePersistence.layerMemory)),
170
);
171
+
172
+it.effect(
173
+ "idle polls still wake for tasks saved later, recorded due times, and beside far-future tasks",
174
+ () =>
175
+ Effect.gen(function* () {
176
+ yield* TestClock.setTime(Date.parse("2026-07-01T09:00:00.000Z"));
177
+ const launched = yield* Deferred.make<string>();
178
+ const layerDependencies = Layer.mergeAll(
179
+ NodeCrypto.layer,
180
+ Scheduler.layer,
181
+ Layer.mock(ThreadLaunchService.ThreadLaunchService)({
182
+ launch: (input) =>
183
+ Deferred.succeed(launched, input.title).pipe(
184
+ Effect.as({ threadId: "thread-idle-poll", resumed: false } as never),
185
+ ),
186
+ }),
187
+ Layer.mock(ManagedProjectFolders.ManagedProjectFolders)({
188
+ namedProjectsRoot: "/projects",
189
+ isScratchProject: () => Effect.succeed(false),
190
+ }),
191
+ Layer.mock(ThreadManagementService.ThreadManagementService)({}),
192
+ Layer.mock(SecretRequests.SecretRequests)({}),
193
+ );
194
+ yield* Effect.gen(function* () {
195
+ const service = yield* ScheduledTaskService.ScheduledTaskService;
196
+ // The first poll finds no tasks and remembers that nothing is due.
197
+ yield* TestClock.adjust("6 seconds");
198
+ // Its year-10000+ next run sorts first as text; it must not hide the task below.
199
+ yield* service.upsert(
200
+ yield* decodeUpsertInput({
201
+ title: "Far future",
202
+ prompt: "Review the open pull requests.",
203
+ enabled: true,
204
+ schedule: { type: "interval", everyMs: 3e14 },
205
+ projectId: "project-idle-poll",
206
+ workspaceStrategy: { type: "root" },
207
+ modelSelection: { instanceId: "codex", model: "gpt-5.4" },
208
+ runtimeMode: "full-access",
209
+ interactionMode: "default",
210
+ }),
211
+ );
212
+ const created = yield* service.upsert(
213
+ yield* decodeUpsertInput({
214
+ title: "Idle poll",
215
+ prompt: "Review the open pull requests.",
216
+ enabled: true,
217
+ schedule: { type: "interval", everyMs: 60_000 },
218
+ projectId: "project-idle-poll",
219
+ workspaceStrategy: { type: "root" },
220
+ modelSelection: { instanceId: "codex", model: "gpt-5.4" },
221
+ runtimeMode: "full-access",
222
+ interactionMode: "default",
223
+ }),
224
+ );
225
+ // This poll records the new task's due time; the later ones must still fire it.
226
+ yield* TestClock.adjust("6 seconds");
227
+ yield* TestClock.adjust("60 seconds");
228
+ expect(yield* Deferred.await(launched)).toBe(created.task.title);
229
+ }).pipe(Effect.provide(ScheduledTaskService.layer.pipe(Layer.provide(layerDependencies))));
230
+ }).pipe(Effect.provide(SqlitePersistence.layerMemory)),
231
+);
apps/server/src/scheduledTasks/ScheduledTaskService.ts
+69
-1
@@ -396,6 +396,35 @@ export const listDueTasks = Effect.fn("ScheduledTaskService.listDueTasks")(funct
396
return tasks;
397
});
398
399
+// Backstop for a write path that forgets to invalidate; the tick still runs every few seconds.
400
+const DUE_HINT_MAX_AGE_MS = 10 * 60 * 1_000;
401
+
402
+/**
403
+ * Remembers when a poller next has work so idle scheduler ticks skip SQL.
404
+ * `invalidate` after every write that can move work earlier; a tick records
405
+ * only if no write happened since it read `version`.
406
+ */
407
+const makeDueHint = () => {
408
+ let version = 0;
409
+ let hint: { readonly dueAtMs: number; readonly checkedAtMs: number } | null = null;
410
+ return {
411
+ invalidate: Effect.sync(() => {
412
+ version++;
413
+ hint = null;
414
+ }),
415
+ version: Effect.sync(() => version),
416
+ isIdle: (nowMs: number) =>
417
+ Effect.sync(
418
+ () =>
419
+ hint !== null && hint.dueAtMs > nowMs && nowMs - hint.checkedAtMs < DUE_HINT_MAX_AGE_MS,
420
+ ),
421
+ record: (readVersion: number, dueAtMs: number, nowMs: number) =>
422
+ Effect.sync(() => {
423
+ if (readVersion === version) hint = { dueAtMs, checkedAtMs: nowMs };
424
+ }),
425
+ };
426
+};
427
+
428
export const layer = Layer.effect(
429
ScheduledTaskService,
430
Effect.gen(function* () {
@@ -426,7 +455,13 @@ export const layer = Layer.effect(
455
const activeRuns = yield* Ref.make<ReadonlySet<ScheduledTaskId>>(new Set());
456
// Notifications re-emit the full list; slow subscribers need only the newest dirty signal.
457
const changesPubSub = yield* PubSub.sliding<void>(1);
429
- const notifyChanged = PubSub.publish(changesPubSub, undefined).pipe(Effect.asVoid);
458
+ const dueTasks = makeDueHint();
459
+ const pendingWebhooks = makeDueHint();
460
+ // Every scheduled_tasks write that can change next_run_at, enabled or run state notifies.
461
+ const notifyChanged = dueTasks.invalidate.pipe(
462
+ Effect.andThen(PubSub.publish(changesPubSub, undefined)),
463
+ Effect.asVoid,
464
+ );
465
466
const selectAllRows = () =>
467
sql<ScheduledTaskRow>`
@@ -972,6 +1007,28 @@ export const layer = Layer.effect(
1007
1008
const runDueTasks = Effect.fn("ScheduledTaskService.runDueTasks")(function* () {
1009
const now = yield* localNow;
1010
+ const nowMs = DateTime.toEpochMillis(now);
1011
+ if (yield* dueTasks.isIdle(nowMs)) return;
1012
+ const readVersion = yield* dueTasks.version;
1013
+ const [next] = yield* sql<{ next_run_at: string | null }>`
1014
+ SELECT MIN(next_run_at) AS next_run_at FROM scheduled_tasks
1015
+ WHERE enabled = 1 AND next_run_at IS NOT NULL AND last_run_status <> 'running'
1016
+ `.pipe(
1017
+ NodeSqliteClient.readOnly,
1018
+ Effect.mapError((cause) => taskError("Could not list schedule tasks.", { cause })),
1019
+ );
1020
+ // Compare as text like listDueTasks: expanded years (`+011533-…`) sort first but parse far ahead.
1021
+ const nextRunAt = next?.next_run_at ?? null;
1022
+ if (nextRunAt === null || nextRunAt > iso(now)) {
1023
+ const nextDueAt =
1024
+ nextRunAt === null
1025
+ ? Option.some(Infinity)
1026
+ : DateTime.make(nextRunAt).pipe(Option.map(DateTime.toEpochMillis));
1027
+ if (Option.isSome(nextDueAt) && nextDueAt.value > nowMs) {
1028
+ yield* dueTasks.record(readVersion, nextDueAt.value, nowMs);
1029
+ }
1030
+ return;
1031
+ }
1032
const tasks = yield* listDueTasks(now).pipe(
1033
Effect.mapError((cause) => taskError("Could not list schedule tasks.", { cause })),
1034
);
@@ -1695,6 +1752,8 @@ export const layer = Layer.effect(
1752
1753
const triggerWebhook: ScheduledTaskService["Service"]["triggerWebhook"] = (request) =>
1754
triggerWebhookUnobserved(request).pipe(
1755
+ // Accepted deliveries may have queued a coalescing window.
1756
+ Effect.ensuring(pendingWebhooks.invalidate),
1757
Effect.tapError(() => observeDelivery(request, "error", undefined, undefined)),
1758
Metrics.withMetrics({ timer: Metrics.webhookDeliveryDuration }),
1759
Effect.withSpan("ScheduledTaskService.triggerWebhook", {
@@ -2090,10 +2149,19 @@ export const layer = Layer.effect(
2149
2150
const dispatchPending = Effect.fn("ScheduledTaskService.dispatchPendingWebhooks")(function* () {
2151
const now = DateTime.toEpochMillis(yield* localNow);
2152
+ if (yield* pendingWebhooks.isIdle(now)) return;
2153
+ const readVersion = yield* pendingWebhooks.version;
2154
yield* sql`UPDATE scheduled_task_webhook_pending SET state = 'dispatching' WHERE state = 'pending' AND due_at <= ${now}`;
2155
const tasks = yield* sql<{ task_id: ScheduledTaskId }>`
2156
SELECT DISTINCT task_id FROM scheduled_task_webhook_pending WHERE state IN ('dispatching', 'running', 'completed', 'failed')
2157
`.pipe(NodeSqliteClient.readOnly);
2158
+ // Workers can leave rows behind to retry, so keep polling while any are claimed.
2159
+ if (tasks.length === 0) {
2160
+ const [next] = yield* sql<{ due_at: number | null }>`
2161
+ SELECT MIN(due_at) AS due_at FROM scheduled_task_webhook_pending WHERE state = 'pending'
2162
+ `.pipe(NodeSqliteClient.readOnly);
2163
+ yield* pendingWebhooks.record(readVersion, next?.due_at ?? Infinity, now);
2164
+ }
2165
yield* Effect.forEach(
2166
tasks,
2167
({ task_id: taskId }) =>
apps/server/src/serverSettings.test.ts
+51
-1
@@ -1047,6 +1047,54 @@ it.layer(NodeServices.layer)("server settings", (it) => {
1047
}).pipe(Effect.provide(layerServerSettings())),
1048
);
1049
1050
+ it.effect("reads settings secrets once per settings change", () => {
1051
+ let reads = 0;
1052
+ const layerSecret = Layer.effect(
1053
+ ServerSecretStore.ServerSecretStore,
1054
+ Effect.map(ServerSecretStore.ServerSecretStore, (store) => ({
1055
+ ...store,
1056
+ get: (name: string) =>
1057
+ Effect.suspend(() => {
1058
+ reads++;
1059
+ return store.get(name);
1060
+ }),
1061
+ })),
1062
+ ).pipe(Layer.provide(ServerSecretStore.layer));
1063
+ const layerSettings = ServerSettingsModule.layer.pipe(
1064
+ Layer.provide(layerSecret),
1065
+ Layer.provideMerge(Layer.fresh(SqlitePersistence.layerMemory)),
1066
+ Layer.provideMerge(
1067
+ Layer.fresh(
1068
+ ServerConfig.layerTest(process.cwd(), { prefix: "t3code-settings-secret-cache-test-" }),
1069
+ ),
1070
+ ),
1071
+ );
1072
+ return Effect.gen(function* () {
1073
+ const instanceId = ProviderInstanceId.make("codex_cached");
1074
+ const service = yield* ServerSettingsModule.ServerSettingsService;
1075
+ const save = (value: string) =>
1076
+ service.updateSettings({
1077
+ providerInstances: {
1078
+ [instanceId]: {
1079
+ driver: ProviderDriverKind.make("codex"),
1080
+ environment: [{ name: "API_TOKEN", value, sensitive: true }],
1081
+ config: {},
1082
+ },
1083
+ },
1084
+ });
1085
+ const token = service.getSettings.pipe(
1086
+ Effect.map((settings) => settings.providerInstances[instanceId]?.environment?.[0]?.value),
1087
+ );
1088
+ yield* save("sk-first");
1089
+ reads = 0;
1090
+ assert.equal(yield* token, "sk-first");
1091
+ assert.equal(yield* token, "sk-first");
1092
+ assert.equal(reads, 0);
1093
+ yield* save("sk-second");
1094
+ assert.equal(yield* token, "sk-second");
1095
+ }).pipe(Effect.provide(layerSettings));
1096
+ });
1097
+
1098
it.effect("keeps the inline value on disk when secret migration fails", () => {
1099
const cause = new ServerSecretStore.SecretStorePersistError({
1100
resource: "provider environment secret",
@@ -1578,6 +1626,7 @@ it.layer(NodeServices.layer)("server settings", (it) => {
1626
const fileSystem = yield* FileSystem.FileSystem;
1627
let failRename = false;
1628
let settingsPathToFail: string | undefined;
1629
+ let readDuringWrite: Effect.Effect<void> = Effect.void;
1630
const writeFailure = PlatformError.systemError({
1631
_tag: "PermissionDenied",
1632
module: "FileSystem",
@@ -1588,7 +1637,7 @@ it.layer(NodeServices.layer)("server settings", (it) => {
1637
...fileSystem,
1638
rename: (fromPath, toPath) =>
1639
failRename && toPath === settingsPathToFail
1591
- ? Effect.fail(writeFailure)
1640
+ ? readDuringWrite.pipe(Effect.andThen(Effect.fail(writeFailure)))
1641
: fileSystem.rename(fromPath, toPath),
1642
});
1643
const instanceId = ProviderInstanceId.make("antigravity_write_failure");
@@ -1610,6 +1659,7 @@ it.layer(NodeServices.layer)("server settings", (it) => {
1659
});
1660
1661
failRename = true;
1662
+ readDuringWrite = Effect.ignore(serverSettings.getSettings);
1663
const failedUpdate = yield* serverSettings
1664
.updateProviderInstance({
1665
operation: "upsert",
apps/server/src/serverSettings.ts
+33
-14
@@ -851,6 +851,31 @@ const make = Effect.gen(function* () {
851
},
852
);
853
854
+ // Background sweeps read settings every few seconds; reading secret files each time is wasted.
855
+ // Every secret write rewrites the settings file, so a new cached object implies fresh secrets.
856
+ // A read that overlaps a secret write may see files a rollback then restores, so it is not kept.
857
+ let materialized: { readonly source: ServerSettings; readonly value: ServerSettings } | null =
858
+ null;
859
+ let secretWriteGeneration = 0;
860
+ const invalidateMaterialized = Effect.sync(() => {
861
+ secretWriteGeneration++;
862
+ materialized = null;
863
+ });
864
+ const getMaterializedSettings = getSettingsFromCache.pipe(
865
+ Effect.flatMap((source) => {
866
+ if (materialized?.source === source) return Effect.succeed(materialized.value);
867
+ const generation = secretWriteGeneration;
868
+ return materializeSettingsSecrets(source).pipe(
869
+ Effect.map(resolveTextGenerationProvider),
870
+ Effect.tap((value) =>
871
+ Effect.sync(() => {
872
+ if (generation === secretWriteGeneration) materialized = { source, value };
873
+ }),
874
+ ),
875
+ );
876
+ }),
877
+ );
878
+
879
const materializeChanges = (changes: Stream.Stream<ServerSettings>) =>
880
changes.pipe(
881
Stream.mapEffect((settings) =>
@@ -1026,8 +1051,9 @@ const make = Effect.gen(function* () {
1051
const updated = yield* update(current);
1052
const persisted = yield* persistSettingsSecrets(current, updated);
1053
const next = yield* normalizeServerSettings(persisted.settings);
1029
- const materialized = yield* Effect.uninterruptibleMask(() =>
1054
+ const materializedNext = yield* Effect.uninterruptibleMask(() =>
1055
Effect.gen(function* () {
1056
+ yield* invalidateMaterialized;
1057
const rollbackSecretChanges = yield* applySettingsSecretChanges(persisted.changes);
1058
const materializedExit = yield* Effect.exit(materializeSettingsSecrets(next));
1059
if (Exit.isFailure(materializedExit)) {
@@ -1040,22 +1066,18 @@ const make = Effect.gen(function* () {
1066
return yield* Effect.failCause(writeExit.cause);
1067
}
1068
return materializedExit.value;
1043
- }),
1069
+ }).pipe(Effect.ensuring(invalidateMaterialized)),
1070
);
1071
yield* Cache.set(settingsCache, cacheKey, { settings: next, parseFailure: Option.none() });
1072
+ const resolved = resolveTextGenerationProvider(materializedNext);
1073
+ materialized = { source: next, value: resolved };
1074
yield* emitChange(next);
1047
- return resolveTextGenerationProvider(materialized);
1075
+ return resolved;
1076
}),
1077
);
1078
1079
const withSettingsSnapshot: ServerSettingsService["Service"]["withSettingsSnapshot"] = (use) =>
1052
- writeSemaphore.withPermits(1)(
1053
- getSettingsFromCache.pipe(
1054
- Effect.flatMap(materializeSettingsSecrets),
1055
- Effect.map(resolveTextGenerationProvider),
1056
- Effect.flatMap(use),
1057
- ),
1058
- );
1080
+ writeSemaphore.withPermits(1)(getMaterializedSettings.pipe(Effect.flatMap(use)));
1081
1082
const revalidateAndEmit = writeSemaphore.withPermits(1)(
1083
Effect.gen(function* () {
@@ -1160,10 +1182,7 @@ const make = Effect.gen(function* () {
1182
return {
1183
start,
1184
ready: Deferred.await(startedDeferred),
1163
- getSettings: getSettingsFromCache.pipe(
1164
- Effect.flatMap(materializeSettingsSecrets),
1165
- Effect.map(resolveTextGenerationProvider),
1166
- ),
1185
+ getSettings: getMaterializedSettings,
1186
updateSettings: (patch) =>
1187
updateAndPersistSettings((current) =>
1188
Effect.succeed(applyServerSettingsPatch(current, patch)),