The best way to control Claude Code, Codex and more from any computer or phone
fix(server): meter checkpoint git subprocess attempts (#24218)
Signed-off-by: Yordis Prieto <yordis.prieto@gmail.com> Co-authored-by: Yordis Prieto <yordis.prieto@gmail.com>
Julius Marminge committed
Oct 23, 2026 at 12:47 UTC
e340aaeeb98bc6ac623c2e70ac8eca8bcbddfe1b
3 files changed
+307
-35
apps/server/src/vcs/GitVcsDriver.test.ts
+224
@@ -12,12 +12,15 @@ import * as Layer from "effect/Layer";
12
import * as Option from "effect/Option";
13
import * as Path from "effect/Path";
14
import * as PlatformError from "effect/PlatformError";
15
+import { ChildProcessSpawner } from "effect/process";
16
+import * as Metric from "effect/Metric";
17
import { assert, it } from "@effect/vitest";
18
19
import { CheckpointRef, GitCommandError, VcsProcessExitError } from "@t3tools/contracts";
20
import * as HostProcess from "@t3tools/shared/HostProcess";
21
import { symlinksSupported } from "@t3tools/shared/testing/symlinks";
22
import * as ServerConfig from "../config.ts";
23
+import { gitCommandDuration, gitCommandsTotal } from "../observability/Metrics.ts";
24
import * as CheckpointStore from "../checkpointing/CheckpointStore.ts";
25
import * as ProcessRunner from "../processRunner.ts";
26
import * as GitVcsDriver from "./GitVcsDriver.ts";
@@ -1017,3 +1020,224 @@ it.effect.skipIf(HostProcess.Platform.defaultValue() === "win32")(
1020
assert.strictEqual(detectionError._tag, "VcsProcessSpawnError");
1021
}).pipe(Effect.scoped, Effect.provide(NodeServices.layer)),
1022
);
1023
+
1024
+const gitCommandCount = (operation: string, outcome: string) =>
1025
+ Metric.value(
1026
+ Metric.withAttributes(gitCommandsTotal, [
1027
+ ["operation", operation],
1028
+ ["outcome", outcome],
1029
+ ]),
1030
+ ).pipe(Effect.map((state) => state.count));
1031
+
1032
+const gitCommandTiming = (operation: string) =>
1033
+ Metric.value(Metric.withAttributes(gitCommandDuration, [["operation", operation]]));
1034
+
1035
+// Keep VcsProcess's exit classification; replace only the subprocess and its clock.
1036
+const makeMeteredDriver = Effect.fn("test.makeMeteredDriver")(function* (exitCode: number) {
1037
+ const process = yield* VcsProcess.make.pipe(
1038
+ Effect.provideService(ProcessRunner.ProcessRunner, {
1039
+ run: () =>
1040
+ TestClock.adjust("250 millis").pipe(
1041
+ Effect.as({
1042
+ code: ChildProcessSpawner.ExitCode(exitCode),
1043
+ stdout: "git output",
1044
+ stderr: exitCode === 0 ? "" : "fatal: test failure",
1045
+ timedOut: false,
1046
+ stdoutTruncated: false,
1047
+ stderrTruncated: false,
1048
+ stdoutInvalidUtf8: false,
1049
+ stderrInvalidUtf8: false,
1050
+ }),
1051
+ ),
1052
+ }),
1053
+ );
1054
+ return yield* GitVcsDriver.makeVcsDriverShape().pipe(
1055
+ Effect.provideService(VcsProcess.VcsProcess, process),
1056
+ );
1057
+});
1058
+
1059
+it.effect("GitVcsDriver meters each git command once under its operation", () =>
1060
+ Effect.gen(function* () {
1061
+ const operation = "GitVcsDriver.checkpoints.resolveHeadCommit";
1062
+ const driver = yield* makeMeteredDriver(0);
1063
+
1064
+ const result = yield* driver.execute({ operation, cwd: "/repo", args: ["rev-parse", "HEAD"] });
1065
+ yield* driver.execute({ operation, cwd: "/another-repo", args: ["rev-parse", "other-ref"] });
1066
+
1067
+ assert.strictEqual(result.stdout, "git output");
1068
+ assert.strictEqual(yield* gitCommandCount(operation, "success"), 2);
1069
+ assert.strictEqual(yield* gitCommandCount(operation, "failure"), 0);
1070
+ const timing = yield* gitCommandTiming(operation);
1071
+ assert.strictEqual(timing.count, 2);
1072
+ assert.strictEqual(timing.sum, 500);
1073
+ }).pipe(
1074
+ Effect.provide(NodeServices.layer),
1075
+ Effect.provideService(Metric.MetricRegistry, new Map()),
1076
+ ),
1077
+);
1078
+
1079
+it.effect("GitVcsDriver counts an allowed non-zero exit as a success", () =>
1080
+ Effect.gen(function* () {
1081
+ const operation = "GitVcsDriver.checkpoints.hasHeadCommit";
1082
+ const driver = yield* makeMeteredDriver(1);
1083
+
1084
+ const result = yield* driver.execute({
1085
+ operation,
1086
+ cwd: "/repo",
1087
+ args: ["rev-parse", "--verify", "HEAD"],
1088
+ allowNonZeroExit: true,
1089
+ });
1090
+
1091
+ assert.strictEqual(result.exitCode, 1);
1092
+ assert.strictEqual(yield* gitCommandCount(operation, "success"), 1);
1093
+ assert.strictEqual(yield* gitCommandCount(operation, "failure"), 0);
1094
+ const timing = yield* gitCommandTiming(operation);
1095
+ assert.strictEqual(timing.count, 1);
1096
+ assert.strictEqual(timing.sum, 250);
1097
+ }).pipe(
1098
+ Effect.provide(NodeServices.layer),
1099
+ Effect.provideService(Metric.MetricRegistry, new Map()),
1100
+ ),
1101
+);
1102
+
1103
+it.effect("GitVcsDriver counts a failed git command as a failure", () =>
1104
+ Effect.gen(function* () {
1105
+ const operation = "GitVcsDriver.checkpoints.resolveHeadCommit";
1106
+ const driver = yield* makeMeteredDriver(128);
1107
+
1108
+ const error = yield* driver
1109
+ .execute({ operation, cwd: "/repo", args: ["rev-parse", "HEAD"] })
1110
+ .pipe(Effect.flip);
1111
+
1112
+ assert.strictEqual(error._tag, "VcsProcessExitError");
1113
+ assert.strictEqual(yield* gitCommandCount(operation, "failure"), 1);
1114
+ assert.strictEqual(yield* gitCommandCount(operation, "success"), 0);
1115
+ const timing = yield* gitCommandTiming(operation);
1116
+ assert.strictEqual(timing.count, 1);
1117
+ assert.strictEqual(timing.sum, 250);
1118
+ }).pipe(
1119
+ Effect.provide(NodeServices.layer),
1120
+ Effect.provideService(Metric.MetricRegistry, new Map()),
1121
+ ),
1122
+);
1123
+
1124
+it.effect("checkpoint capture meters every git process it starts, including streamed ones", () =>
1125
+ Effect.gen(function* () {
1126
+ const fileSystem = yield* FileSystem.FileSystem;
1127
+ const liveProcess = yield* VcsProcess.VcsProcess;
1128
+ const operation = VcsProcess.CHECKPOINT_CAPTURE_OPERATION;
1129
+ let started = 0;
1130
+ let streamed = 0;
1131
+ const driver = yield* GitVcsDriver.makeVcsDriverShape();
1132
+ const captureDriver = yield* GitVcsDriver.makeVcsDriverShape().pipe(
1133
+ Effect.provideService(VcsProcess.VcsProcess, {
1134
+ run: (input) => {
1135
+ if (input.operation === operation) {
1136
+ started++;
1137
+ if (input.onStdoutChunk !== undefined) streamed++;
1138
+ }
1139
+ return liveProcess.run(input);
1140
+ },
1141
+ }),
1142
+ );
1143
+ const cwd = yield* fileSystem.makeTempDirectoryScoped({ prefix: "t3-checkpoint-metrics-" });
1144
+ const { checkpointRef } = yield* makeCheckpointFixture(driver, cwd);
1145
+
1146
+ yield* captureDriver.checkpoints.captureCheckpoint({ cwd, checkpointRef });
1147
+
1148
+ assert.isAbove(streamed, 0);
1149
+ assert.strictEqual(yield* gitCommandCount(operation, "success"), started);
1150
+ assert.strictEqual(yield* gitCommandCount(operation, "failure"), 0);
1151
+ assert.strictEqual((yield* gitCommandTiming(operation)).count, started);
1152
+ }).pipe(
1153
+ Effect.scoped,
1154
+ Effect.provide(layerGitContract),
1155
+ Effect.provideService(Metric.MetricRegistry, new Map()),
1156
+ ),
1157
+);
1158
+
1159
+it.effect("git metrics finalize cancellation without exposing arbitrary operation labels", () =>
1160
+ Effect.gen(function* () {
1161
+ const started = yield* Deferred.make<void>();
1162
+ const process = yield* VcsProcess.make.pipe(
1163
+ Effect.provideService(ProcessRunner.ProcessRunner, {
1164
+ run: () => Deferred.succeed(started, undefined).pipe(Effect.andThen(Effect.never)),
1165
+ }),
1166
+ );
1167
+ const driver = yield* GitVcsDriver.makeVcsDriverShape().pipe(
1168
+ Effect.provideService(VcsProcess.VcsProcess, process),
1169
+ );
1170
+ const operation = "arbitrary operation /private/path";
1171
+ const fiber = yield* driver
1172
+ .execute({ operation, cwd: "/repo", args: ["rev-parse", "HEAD"] })
1173
+ .pipe(Effect.forkChild);
1174
+ yield* Deferred.await(started);
1175
+ yield* TestClock.adjust("250 millis");
1176
+ yield* Fiber.interrupt(fiber);
1177
+
1178
+ assert.strictEqual(yield* gitCommandCount("other", "interrupt"), 1);
1179
+ assert.strictEqual(yield* gitCommandCount(operation, "interrupt"), 0);
1180
+ const timing = yield* gitCommandTiming("other");
1181
+ assert.strictEqual(timing.count, 1);
1182
+ assert.strictEqual(timing.sum, 250);
1183
+ }).pipe(
1184
+ Effect.provide(NodeServices.layer),
1185
+ Effect.provideService(Metric.MetricRegistry, new Map()),
1186
+ ),
1187
+);
1188
+
1189
+it.effect("git metrics count fsync fallback and transient retry subprocesses separately", () =>
1190
+ Effect.gen(function* () {
1191
+ const retryReady = yield* Deferred.make<void>();
1192
+ const operation = VcsProcess.CHECKPOINT_CAPTURE_OPERATION;
1193
+ let attempts = 0;
1194
+ const process = yield* VcsProcess.make.pipe(
1195
+ Effect.provideService(HostProcess.Platform, "darwin"),
1196
+ Effect.provideService(ProcessRunner.ProcessRunner, {
1197
+ run: (input) =>
1198
+ Effect.gen(function* () {
1199
+ attempts++;
1200
+ yield* TestClock.adjust("250 millis");
1201
+ if (attempts === 2) {
1202
+ assert.include(input.args, "core.fsyncMethod=writeout-only");
1203
+ yield* Deferred.succeed(retryReady, undefined);
1204
+ }
1205
+ return {
1206
+ code: ChildProcessSpawner.ExitCode(attempts < 3 ? 128 : 0),
1207
+ stdout: "",
1208
+ stderr:
1209
+ attempts === 1
1210
+ ? "fatal: fsync error on '.git/objects/test': Operation not supported"
1211
+ : attempts === 2
1212
+ ? "fatal: Unable to create '/repo/.git/index.lock': File exists."
1213
+ : "",
1214
+ timedOut: false,
1215
+ stdoutTruncated: false,
1216
+ stderrTruncated: false,
1217
+ stdoutInvalidUtf8: false,
1218
+ stderrInvalidUtf8: false,
1219
+ };
1220
+ }),
1221
+ }),
1222
+ );
1223
+ const driver = yield* GitVcsDriver.makeVcsDriverShape().pipe(
1224
+ Effect.provideService(VcsProcess.VcsProcess, process),
1225
+ );
1226
+ const fiber = yield* driver
1227
+ .execute({ operation, cwd: "/repo", args: ["-c", "core.fsyncMethod=batch", "add"] })
1228
+ .pipe(Effect.forkChild);
1229
+ yield* Deferred.await(retryReady);
1230
+ yield* TestClock.adjust("75 millis");
1231
+ yield* Fiber.join(fiber);
1232
+
1233
+ assert.strictEqual(attempts, 3);
1234
+ assert.strictEqual(yield* gitCommandCount(operation, "failure"), 2);
1235
+ assert.strictEqual(yield* gitCommandCount(operation, "success"), 1);
1236
+ const timing = yield* gitCommandTiming(operation);
1237
+ assert.strictEqual(timing.count, 3);
1238
+ assert.strictEqual(timing.sum, 750);
1239
+ }).pipe(
1240
+ Effect.provide(NodeServices.layer),
1241
+ Effect.provideService(Metric.MetricRegistry, new Map()),
1242
+ ),
1243
+);
apps/server/src/vcs/GitVcsDriver.ts
+35
-33
@@ -503,39 +503,41 @@ function parseGitRemoteVerboseOutput(
503
return remotes;
504
}
505
506
-const gitCommand = (
507
- process: VcsProcess.VcsProcess["Service"],
508
- operation: string,
509
- cwd: string,
510
- args: ReadonlyArray<string>,
511
- options?: {
512
- readonly stdin?: string;
513
- readonly env?: NodeJS.ProcessEnv;
514
- readonly allowNonZeroExit?: boolean;
515
- readonly timeoutMs?: number;
516
- readonly maxOutputBytes?: number;
517
- readonly outputMode?: VcsProcess.VcsProcessInput["outputMode"];
518
- readonly appendTruncationMarker?: boolean;
519
- },
520
-) =>
521
- process.run({
522
- operation,
523
- command: "git",
524
- args: ["-C", cwd, ...args],
525
- cwd,
526
- spawnCwd: globalThis.process.cwd(),
527
- ...(options?.stdin !== undefined ? { stdin: options.stdin } : {}),
528
- ...(options?.env !== undefined ? { env: options.env } : {}),
529
- ...(options?.allowNonZeroExit !== undefined
530
- ? { allowNonZeroExit: options.allowNonZeroExit }
531
- : {}),
532
- ...(options?.timeoutMs !== undefined ? { timeoutMs: options.timeoutMs } : {}),
533
- ...(options?.maxOutputBytes !== undefined ? { maxOutputBytes: options.maxOutputBytes } : {}),
534
- ...(options?.outputMode !== undefined ? { outputMode: options.outputMode } : {}),
535
- ...(options?.appendTruncationMarker !== undefined
536
- ? { appendTruncationMarker: options.appendTruncationMarker }
537
- : {}),
538
- });
506
+const gitCommand = Effect.fn("GitVcsDriver.gitCommand")(
507
+ (
508
+ process: VcsProcess.VcsProcess["Service"],
509
+ operation: string,
510
+ cwd: string,
511
+ args: ReadonlyArray<string>,
512
+ options?: {
513
+ readonly stdin?: string;
514
+ readonly env?: NodeJS.ProcessEnv;
515
+ readonly allowNonZeroExit?: boolean;
516
+ readonly timeoutMs?: number;
517
+ readonly maxOutputBytes?: number;
518
+ readonly outputMode?: VcsProcess.VcsProcessInput["outputMode"];
519
+ readonly appendTruncationMarker?: boolean;
520
+ },
521
+ ) =>
522
+ process.run({
523
+ operation,
524
+ command: "git",
525
+ args: ["-C", cwd, ...args],
526
+ cwd,
527
+ spawnCwd: globalThis.process.cwd(),
528
+ ...(options?.stdin !== undefined ? { stdin: options.stdin } : {}),
529
+ ...(options?.env !== undefined ? { env: options.env } : {}),
530
+ ...(options?.allowNonZeroExit !== undefined
531
+ ? { allowNonZeroExit: options.allowNonZeroExit }
532
+ : {}),
533
+ ...(options?.timeoutMs !== undefined ? { timeoutMs: options.timeoutMs } : {}),
534
+ ...(options?.maxOutputBytes !== undefined ? { maxOutputBytes: options.maxOutputBytes } : {}),
535
+ ...(options?.outputMode !== undefined ? { outputMode: options.outputMode } : {}),
536
+ ...(options?.appendTruncationMarker !== undefined
537
+ ? { appendTruncationMarker: options.appendTruncationMarker }
538
+ : {}),
539
+ }),
540
+);
541
542
export const makeVcsDriverShape = Effect.fn("makeGitVcsDriverShape")(function* () {
543
const fileSystem = yield* FileSystem.FileSystem;
apps/server/src/vcs/VcsProcess.ts
+48
-2
@@ -21,6 +21,7 @@ import {
21
} from "@t3tools/contracts";
22
import * as HostProcess from "@t3tools/shared/HostProcess";
23
24
+import { gitCommandDuration, gitCommandsTotal, withMetrics } from "../observability/Metrics.ts";
25
import * as ProcessRunner from "../processRunner.ts";
26
27
export interface VcsProcessInput {
@@ -68,6 +69,27 @@ export const GIT_INDEX_LOCKED_DETAIL =
69
70
export const CHECKPOINT_CAPTURE_OPERATION = "GitVcsDriver.checkpoints.captureCheckpoint";
71
72
+// Caller operations are diagnostic strings; only known labels may become metric attributes.
73
+const gitMetricOperations = new Set([
74
+ CHECKPOINT_CAPTURE_OPERATION,
75
+ "GitVcsDriver.checkpoints.deleteCheckpointRefs",
76
+ "GitVcsDriver.checkpoints.diffCheckpoints",
77
+ "GitVcsDriver.checkpoints.hasHeadCommit",
78
+ "GitVcsDriver.checkpoints.listAuthoredPaths",
79
+ "GitVcsDriver.checkpoints.resolveCheckpointCommit",
80
+ "GitVcsDriver.checkpoints.resolveGitCommonDir",
81
+ "GitVcsDriver.checkpoints.resolveHeadCommit",
82
+ "GitVcsDriver.checkpoints.restoreCheckpoint",
83
+ "GitVcsDriver.detectRepository.commonDir",
84
+ "GitVcsDriver.detectRepository.root",
85
+ "GitVcsDriver.filterIgnoredPaths",
86
+ "GitVcsDriver.initRepository",
87
+ "GitVcsDriver.isInsideWorkTree",
88
+ "GitVcsDriver.listRemotes",
89
+ "GitVcsDriver.listWorkspaceFiles",
90
+ "GitVcsDriver.resolveGitCommonDir",
91
+]);
92
+
93
/**
94
* Appends `key=value` to the `GIT_CONFIG_*` entries git reads as command-line
95
* config, so argv stays the same on every platform and nothing is written to
@@ -184,8 +206,12 @@ export const make = Effect.gen(function* () {
206
const vcsProcesses = yield* Semaphore.make(VCS_PROCESS_CONCURRENCY);
207
const githubProcesses = yield* Semaphore.make(GITHUB_PROCESS_CONCURRENCY);
208
187
- const spawn = (input: VcsProcessInput, baseError: VcsProcessErrorContext) =>
188
- processRunner
209
+ const spawn = Effect.fn("VcsProcess.spawn")(function* (
210
+ input: VcsProcessInput,
211
+ baseError: VcsProcessErrorContext,
212
+ ) {
213
+ let failedExit = false;
214
+ const execution = processRunner
215
.run({
216
command: input.command,
217
args: input.args,
@@ -204,6 +230,10 @@ export const make = Effect.gen(function* () {
230
timeoutBehavior: "error",
231
})
232
.pipe(
233
+ Effect.tap((result) => {
234
+ failedExit = result.code === null || (!input.allowNonZeroExit && result.code !== 0);
235
+ return Effect.void;
236
+ }),
237
Effect.mapError(
238
Match.valueTags({
239
ProcessSpawnError: (error) =>
@@ -233,6 +263,22 @@ export const make = Effect.gen(function* () {
263
),
264
);
265
266
+ return yield* input.command === "git"
267
+ ? execution.pipe(
268
+ withMetrics({
269
+ counter: gitCommandsTotal,
270
+ timer: gitCommandDuration,
271
+ attributes: {
272
+ operation: gitMetricOperations.has(input.operation) ? input.operation : "other",
273
+ },
274
+ outcomeAttributes: (outcome) => ({
275
+ outcome: outcome === "success" && failedExit ? "failure" : outcome,
276
+ }),
277
+ }),
278
+ )
279
+ : execution;
280
+ });
281
+
282
const runUnbounded = Effect.fn("VcsProcess.runUnbounded")(function* (input: VcsProcessInput) {
283
const baseError = {
284
operation: input.operation,