@devries-cameron20 / t3code Mirror · read-only / commits / e340aaeeb9
Fork 0

The best way to control Claude Code, Codex and more from any computer or phone

Mirrored from https://github.com/Camo67/t3code.git

Synced Nov 22, 18:15 UTC

fix(server): meter checkpoint git subprocess attempts (#24218)

Signed-off-by: Yordis Prieto <yordis.prieto@gmail.com> Co-authored-by: Yordis Prieto <yordis.prieto@gmail.com>

Julius Marminge committed Oct 23, 2026 at 12:47 UTC e340aaeeb98bc6ac623c2e70ac8eca8bcbddfe1b
3 files changed +307 -35
apps/server/src/vcs/GitVcsDriver.test.ts
+224
@@ -12,12 +12,15 @@ import * as Layer from "effect/Layer";
12 import * as Option from "effect/Option";
13 import * as Path from "effect/Path";
14 import * as PlatformError from "effect/PlatformError";
15 +import { ChildProcessSpawner } from "effect/process";
16 +import * as Metric from "effect/Metric";
17 import { assert, it } from "@effect/vitest";
18
19 import { CheckpointRef, GitCommandError, VcsProcessExitError } from "@t3tools/contracts";
20 import * as HostProcess from "@t3tools/shared/HostProcess";
21 import { symlinksSupported } from "@t3tools/shared/testing/symlinks";
22 import * as ServerConfig from "../config.ts";
23 +import { gitCommandDuration, gitCommandsTotal } from "../observability/Metrics.ts";
24 import * as CheckpointStore from "../checkpointing/CheckpointStore.ts";
25 import * as ProcessRunner from "../processRunner.ts";
26 import * as GitVcsDriver from "./GitVcsDriver.ts";
@@ -1017,3 +1020,224 @@ it.effect.skipIf(HostProcess.Platform.defaultValue() === "win32")(
1020 assert.strictEqual(detectionError._tag, "VcsProcessSpawnError");
1021 }).pipe(Effect.scoped, Effect.provide(NodeServices.layer)),
1022 );
1023 +
1024 +const gitCommandCount = (operation: string, outcome: string) =>
1025 + Metric.value(
1026 + Metric.withAttributes(gitCommandsTotal, [
1027 + ["operation", operation],
1028 + ["outcome", outcome],
1029 + ]),
1030 + ).pipe(Effect.map((state) => state.count));
1031 +
1032 +const gitCommandTiming = (operation: string) =>
1033 + Metric.value(Metric.withAttributes(gitCommandDuration, [["operation", operation]]));
1034 +
1035 +// Keep VcsProcess's exit classification; replace only the subprocess and its clock.
1036 +const makeMeteredDriver = Effect.fn("test.makeMeteredDriver")(function* (exitCode: number) {
1037 + const process = yield* VcsProcess.make.pipe(
1038 + Effect.provideService(ProcessRunner.ProcessRunner, {
1039 + run: () =>
1040 + TestClock.adjust("250 millis").pipe(
1041 + Effect.as({
1042 + code: ChildProcessSpawner.ExitCode(exitCode),
1043 + stdout: "git output",
1044 + stderr: exitCode === 0 ? "" : "fatal: test failure",
1045 + timedOut: false,
1046 + stdoutTruncated: false,
1047 + stderrTruncated: false,
1048 + stdoutInvalidUtf8: false,
1049 + stderrInvalidUtf8: false,
1050 + }),
1051 + ),
1052 + }),
1053 + );
1054 + return yield* GitVcsDriver.makeVcsDriverShape().pipe(
1055 + Effect.provideService(VcsProcess.VcsProcess, process),
1056 + );
1057 +});
1058 +
1059 +it.effect("GitVcsDriver meters each git command once under its operation", () =>
1060 + Effect.gen(function* () {
1061 + const operation = "GitVcsDriver.checkpoints.resolveHeadCommit";
1062 + const driver = yield* makeMeteredDriver(0);
1063 +
1064 + const result = yield* driver.execute({ operation, cwd: "/repo", args: ["rev-parse", "HEAD"] });
1065 + yield* driver.execute({ operation, cwd: "/another-repo", args: ["rev-parse", "other-ref"] });
1066 +
1067 + assert.strictEqual(result.stdout, "git output");
1068 + assert.strictEqual(yield* gitCommandCount(operation, "success"), 2);
1069 + assert.strictEqual(yield* gitCommandCount(operation, "failure"), 0);
1070 + const timing = yield* gitCommandTiming(operation);
1071 + assert.strictEqual(timing.count, 2);
1072 + assert.strictEqual(timing.sum, 500);
1073 + }).pipe(
1074 + Effect.provide(NodeServices.layer),
1075 + Effect.provideService(Metric.MetricRegistry, new Map()),
1076 + ),
1077 +);
1078 +
1079 +it.effect("GitVcsDriver counts an allowed non-zero exit as a success", () =>
1080 + Effect.gen(function* () {
1081 + const operation = "GitVcsDriver.checkpoints.hasHeadCommit";
1082 + const driver = yield* makeMeteredDriver(1);
1083 +
1084 + const result = yield* driver.execute({
1085 + operation,
1086 + cwd: "/repo",
1087 + args: ["rev-parse", "--verify", "HEAD"],
1088 + allowNonZeroExit: true,
1089 + });
1090 +
1091 + assert.strictEqual(result.exitCode, 1);
1092 + assert.strictEqual(yield* gitCommandCount(operation, "success"), 1);
1093 + assert.strictEqual(yield* gitCommandCount(operation, "failure"), 0);
1094 + const timing = yield* gitCommandTiming(operation);
1095 + assert.strictEqual(timing.count, 1);
1096 + assert.strictEqual(timing.sum, 250);
1097 + }).pipe(
1098 + Effect.provide(NodeServices.layer),
1099 + Effect.provideService(Metric.MetricRegistry, new Map()),
1100 + ),
1101 +);
1102 +
1103 +it.effect("GitVcsDriver counts a failed git command as a failure", () =>
1104 + Effect.gen(function* () {
1105 + const operation = "GitVcsDriver.checkpoints.resolveHeadCommit";
1106 + const driver = yield* makeMeteredDriver(128);
1107 +
1108 + const error = yield* driver
1109 + .execute({ operation, cwd: "/repo", args: ["rev-parse", "HEAD"] })
1110 + .pipe(Effect.flip);
1111 +
1112 + assert.strictEqual(error._tag, "VcsProcessExitError");
1113 + assert.strictEqual(yield* gitCommandCount(operation, "failure"), 1);
1114 + assert.strictEqual(yield* gitCommandCount(operation, "success"), 0);
1115 + const timing = yield* gitCommandTiming(operation);
1116 + assert.strictEqual(timing.count, 1);
1117 + assert.strictEqual(timing.sum, 250);
1118 + }).pipe(
1119 + Effect.provide(NodeServices.layer),
1120 + Effect.provideService(Metric.MetricRegistry, new Map()),
1121 + ),
1122 +);
1123 +
1124 +it.effect("checkpoint capture meters every git process it starts, including streamed ones", () =>
1125 + Effect.gen(function* () {
1126 + const fileSystem = yield* FileSystem.FileSystem;
1127 + const liveProcess = yield* VcsProcess.VcsProcess;
1128 + const operation = VcsProcess.CHECKPOINT_CAPTURE_OPERATION;
1129 + let started = 0;
1130 + let streamed = 0;
1131 + const driver = yield* GitVcsDriver.makeVcsDriverShape();
1132 + const captureDriver = yield* GitVcsDriver.makeVcsDriverShape().pipe(
1133 + Effect.provideService(VcsProcess.VcsProcess, {
1134 + run: (input) => {
1135 + if (input.operation === operation) {
1136 + started++;
1137 + if (input.onStdoutChunk !== undefined) streamed++;
1138 + }
1139 + return liveProcess.run(input);
1140 + },
1141 + }),
1142 + );
1143 + const cwd = yield* fileSystem.makeTempDirectoryScoped({ prefix: "t3-checkpoint-metrics-" });
1144 + const { checkpointRef } = yield* makeCheckpointFixture(driver, cwd);
1145 +
1146 + yield* captureDriver.checkpoints.captureCheckpoint({ cwd, checkpointRef });
1147 +
1148 + assert.isAbove(streamed, 0);
1149 + assert.strictEqual(yield* gitCommandCount(operation, "success"), started);
1150 + assert.strictEqual(yield* gitCommandCount(operation, "failure"), 0);
1151 + assert.strictEqual((yield* gitCommandTiming(operation)).count, started);
1152 + }).pipe(
1153 + Effect.scoped,
1154 + Effect.provide(layerGitContract),
1155 + Effect.provideService(Metric.MetricRegistry, new Map()),
1156 + ),
1157 +);
1158 +
1159 +it.effect("git metrics finalize cancellation without exposing arbitrary operation labels", () =>
1160 + Effect.gen(function* () {
1161 + const started = yield* Deferred.make<void>();
1162 + const process = yield* VcsProcess.make.pipe(
1163 + Effect.provideService(ProcessRunner.ProcessRunner, {
1164 + run: () => Deferred.succeed(started, undefined).pipe(Effect.andThen(Effect.never)),
1165 + }),
1166 + );
1167 + const driver = yield* GitVcsDriver.makeVcsDriverShape().pipe(
1168 + Effect.provideService(VcsProcess.VcsProcess, process),
1169 + );
1170 + const operation = "arbitrary operation /private/path";
1171 + const fiber = yield* driver
1172 + .execute({ operation, cwd: "/repo", args: ["rev-parse", "HEAD"] })
1173 + .pipe(Effect.forkChild);
1174 + yield* Deferred.await(started);
1175 + yield* TestClock.adjust("250 millis");
1176 + yield* Fiber.interrupt(fiber);
1177 +
1178 + assert.strictEqual(yield* gitCommandCount("other", "interrupt"), 1);
1179 + assert.strictEqual(yield* gitCommandCount(operation, "interrupt"), 0);
1180 + const timing = yield* gitCommandTiming("other");
1181 + assert.strictEqual(timing.count, 1);
1182 + assert.strictEqual(timing.sum, 250);
1183 + }).pipe(
1184 + Effect.provide(NodeServices.layer),
1185 + Effect.provideService(Metric.MetricRegistry, new Map()),
1186 + ),
1187 +);
1188 +
1189 +it.effect("git metrics count fsync fallback and transient retry subprocesses separately", () =>
1190 + Effect.gen(function* () {
1191 + const retryReady = yield* Deferred.make<void>();
1192 + const operation = VcsProcess.CHECKPOINT_CAPTURE_OPERATION;
1193 + let attempts = 0;
1194 + const process = yield* VcsProcess.make.pipe(
1195 + Effect.provideService(HostProcess.Platform, "darwin"),
1196 + Effect.provideService(ProcessRunner.ProcessRunner, {
1197 + run: (input) =>
1198 + Effect.gen(function* () {
1199 + attempts++;
1200 + yield* TestClock.adjust("250 millis");
1201 + if (attempts === 2) {
1202 + assert.include(input.args, "core.fsyncMethod=writeout-only");
1203 + yield* Deferred.succeed(retryReady, undefined);
1204 + }
1205 + return {
1206 + code: ChildProcessSpawner.ExitCode(attempts < 3 ? 128 : 0),
1207 + stdout: "",
1208 + stderr:
1209 + attempts === 1
1210 + ? "fatal: fsync error on '.git/objects/test': Operation not supported"
1211 + : attempts === 2
1212 + ? "fatal: Unable to create '/repo/.git/index.lock': File exists."
1213 + : "",
1214 + timedOut: false,
1215 + stdoutTruncated: false,
1216 + stderrTruncated: false,
1217 + stdoutInvalidUtf8: false,
1218 + stderrInvalidUtf8: false,
1219 + };
1220 + }),
1221 + }),
1222 + );
1223 + const driver = yield* GitVcsDriver.makeVcsDriverShape().pipe(
1224 + Effect.provideService(VcsProcess.VcsProcess, process),
1225 + );
1226 + const fiber = yield* driver
1227 + .execute({ operation, cwd: "/repo", args: ["-c", "core.fsyncMethod=batch", "add"] })
1228 + .pipe(Effect.forkChild);
1229 + yield* Deferred.await(retryReady);
1230 + yield* TestClock.adjust("75 millis");
1231 + yield* Fiber.join(fiber);
1232 +
1233 + assert.strictEqual(attempts, 3);
1234 + assert.strictEqual(yield* gitCommandCount(operation, "failure"), 2);
1235 + assert.strictEqual(yield* gitCommandCount(operation, "success"), 1);
1236 + const timing = yield* gitCommandTiming(operation);
1237 + assert.strictEqual(timing.count, 3);
1238 + assert.strictEqual(timing.sum, 750);
1239 + }).pipe(
1240 + Effect.provide(NodeServices.layer),
1241 + Effect.provideService(Metric.MetricRegistry, new Map()),
1242 + ),
1243 +);
apps/server/src/vcs/GitVcsDriver.ts
+35 -33
@@ -503,39 +503,41 @@ function parseGitRemoteVerboseOutput(
503 return remotes;
504 }
505
506 -const gitCommand = (
507 - process: VcsProcess.VcsProcess["Service"],
508 - operation: string,
509 - cwd: string,
510 - args: ReadonlyArray<string>,
511 - options?: {
512 - readonly stdin?: string;
513 - readonly env?: NodeJS.ProcessEnv;
514 - readonly allowNonZeroExit?: boolean;
515 - readonly timeoutMs?: number;
516 - readonly maxOutputBytes?: number;
517 - readonly outputMode?: VcsProcess.VcsProcessInput["outputMode"];
518 - readonly appendTruncationMarker?: boolean;
519 - },
520 -) =>
521 - process.run({
522 - operation,
523 - command: "git",
524 - args: ["-C", cwd, ...args],
525 - cwd,
526 - spawnCwd: globalThis.process.cwd(),
527 - ...(options?.stdin !== undefined ? { stdin: options.stdin } : {}),
528 - ...(options?.env !== undefined ? { env: options.env } : {}),
529 - ...(options?.allowNonZeroExit !== undefined
530 - ? { allowNonZeroExit: options.allowNonZeroExit }
531 - : {}),
532 - ...(options?.timeoutMs !== undefined ? { timeoutMs: options.timeoutMs } : {}),
533 - ...(options?.maxOutputBytes !== undefined ? { maxOutputBytes: options.maxOutputBytes } : {}),
534 - ...(options?.outputMode !== undefined ? { outputMode: options.outputMode } : {}),
535 - ...(options?.appendTruncationMarker !== undefined
536 - ? { appendTruncationMarker: options.appendTruncationMarker }
537 - : {}),
538 - });
506 +const gitCommand = Effect.fn("GitVcsDriver.gitCommand")(
507 + (
508 + process: VcsProcess.VcsProcess["Service"],
509 + operation: string,
510 + cwd: string,
511 + args: ReadonlyArray<string>,
512 + options?: {
513 + readonly stdin?: string;
514 + readonly env?: NodeJS.ProcessEnv;
515 + readonly allowNonZeroExit?: boolean;
516 + readonly timeoutMs?: number;
517 + readonly maxOutputBytes?: number;
518 + readonly outputMode?: VcsProcess.VcsProcessInput["outputMode"];
519 + readonly appendTruncationMarker?: boolean;
520 + },
521 + ) =>
522 + process.run({
523 + operation,
524 + command: "git",
525 + args: ["-C", cwd, ...args],
526 + cwd,
527 + spawnCwd: globalThis.process.cwd(),
528 + ...(options?.stdin !== undefined ? { stdin: options.stdin } : {}),
529 + ...(options?.env !== undefined ? { env: options.env } : {}),
530 + ...(options?.allowNonZeroExit !== undefined
531 + ? { allowNonZeroExit: options.allowNonZeroExit }
532 + : {}),
533 + ...(options?.timeoutMs !== undefined ? { timeoutMs: options.timeoutMs } : {}),
534 + ...(options?.maxOutputBytes !== undefined ? { maxOutputBytes: options.maxOutputBytes } : {}),
535 + ...(options?.outputMode !== undefined ? { outputMode: options.outputMode } : {}),
536 + ...(options?.appendTruncationMarker !== undefined
537 + ? { appendTruncationMarker: options.appendTruncationMarker }
538 + : {}),
539 + }),
540 +);
541
542 export const makeVcsDriverShape = Effect.fn("makeGitVcsDriverShape")(function* () {
543 const fileSystem = yield* FileSystem.FileSystem;
apps/server/src/vcs/VcsProcess.ts
+48 -2
@@ -21,6 +21,7 @@ import {
21 } from "@t3tools/contracts";
22 import * as HostProcess from "@t3tools/shared/HostProcess";
23
24 +import { gitCommandDuration, gitCommandsTotal, withMetrics } from "../observability/Metrics.ts";
25 import * as ProcessRunner from "../processRunner.ts";
26
27 export interface VcsProcessInput {
@@ -68,6 +69,27 @@ export const GIT_INDEX_LOCKED_DETAIL =
69
70 export const CHECKPOINT_CAPTURE_OPERATION = "GitVcsDriver.checkpoints.captureCheckpoint";
71
72 +// Caller operations are diagnostic strings; only known labels may become metric attributes.
73 +const gitMetricOperations = new Set([
74 + CHECKPOINT_CAPTURE_OPERATION,
75 + "GitVcsDriver.checkpoints.deleteCheckpointRefs",
76 + "GitVcsDriver.checkpoints.diffCheckpoints",
77 + "GitVcsDriver.checkpoints.hasHeadCommit",
78 + "GitVcsDriver.checkpoints.listAuthoredPaths",
79 + "GitVcsDriver.checkpoints.resolveCheckpointCommit",
80 + "GitVcsDriver.checkpoints.resolveGitCommonDir",
81 + "GitVcsDriver.checkpoints.resolveHeadCommit",
82 + "GitVcsDriver.checkpoints.restoreCheckpoint",
83 + "GitVcsDriver.detectRepository.commonDir",
84 + "GitVcsDriver.detectRepository.root",
85 + "GitVcsDriver.filterIgnoredPaths",
86 + "GitVcsDriver.initRepository",
87 + "GitVcsDriver.isInsideWorkTree",
88 + "GitVcsDriver.listRemotes",
89 + "GitVcsDriver.listWorkspaceFiles",
90 + "GitVcsDriver.resolveGitCommonDir",
91 +]);
92 +
93 /**
94 * Appends `key=value` to the `GIT_CONFIG_*` entries git reads as command-line
95 * config, so argv stays the same on every platform and nothing is written to
@@ -184,8 +206,12 @@ export const make = Effect.gen(function* () {
206 const vcsProcesses = yield* Semaphore.make(VCS_PROCESS_CONCURRENCY);
207 const githubProcesses = yield* Semaphore.make(GITHUB_PROCESS_CONCURRENCY);
208
187 - const spawn = (input: VcsProcessInput, baseError: VcsProcessErrorContext) =>
188 - processRunner
209 + const spawn = Effect.fn("VcsProcess.spawn")(function* (
210 + input: VcsProcessInput,
211 + baseError: VcsProcessErrorContext,
212 + ) {
213 + let failedExit = false;
214 + const execution = processRunner
215 .run({
216 command: input.command,
217 args: input.args,
@@ -204,6 +230,10 @@ export const make = Effect.gen(function* () {
230 timeoutBehavior: "error",
231 })
232 .pipe(
233 + Effect.tap((result) => {
234 + failedExit = result.code === null || (!input.allowNonZeroExit && result.code !== 0);
235 + return Effect.void;
236 + }),
237 Effect.mapError(
238 Match.valueTags({
239 ProcessSpawnError: (error) =>
@@ -233,6 +263,22 @@ export const make = Effect.gen(function* () {
263 ),
264 );
265
266 + return yield* input.command === "git"
267 + ? execution.pipe(
268 + withMetrics({
269 + counter: gitCommandsTotal,
270 + timer: gitCommandDuration,
271 + attributes: {
272 + operation: gitMetricOperations.has(input.operation) ? input.operation : "other",
273 + },
274 + outcomeAttributes: (outcome) => ({
275 + outcome: outcome === "success" && failedExit ? "failure" : outcome,
276 + }),
277 + }),
278 + )
279 + : execution;
280 + });
281 +
282 const runUnbounded = Effect.fn("VcsProcess.runUnbounded")(function* (input: VcsProcessInput) {
283 const baseError = {
284 operation: input.operation,