The best way to control Claude Code, Codex and more from any computer or phone

main
ts 414 lines 14.9 KB
Raw
1 /**
2 * The import alarm job's git client (DESIGN.md §2.3 Import).
3 *
4 * A minimal **smart-HTTP protocol v0 client**: fetches and parses the
5 * `info/refs?service=git-upload-pack` advertisement, then POSTs a
6 * `git-upload-pack` request (wants + optional `deepen`, no haves,
7 * immediate `done`, **no side-band** — so the pack follows the `NAK`
8 * verbatim) and returns the raw pack bytes for the Repo DO to feed
9 * through the same `ingestPack` path as a push, with the same 50 MiB cap
10 * (documented v1 limit: small / depth-limited imports only).
11 *
12 * Network I/O goes through the Effect `HttpClient` service; callers
13 * provide `FetchHttpClient.layer` (see `RepoObject.ts`'s alarm handler).
14 */
15 import * as Effect from "effect/Effect";
16 import * as HttpClient from "effect/http/HttpClient";
17 import * as HttpClientRequest from "effect/http/HttpClientRequest";
18 import * as Schema from "effect/Schema";
19 import { concatBytes, isOid, ZERO_OID } from "../Protocol/ObjectCodec.ts";
20 import { flushPkt, pktPayloadText, pktText, readPktLineAt } from "../Protocol/Pkt.ts";
21
22 // ─────────────────────────────────────────────────────────────────────────────
23 // Errors & shapes
24 // ─────────────────────────────────────────────────────────────────────────────
25
26 /**
27 * Raised when the source remote misbehaves: HTTP failures, non-200s,
28 * malformed advertisements, oversize packs, or an `ERR` pkt from the
29 * remote. The Repo DO maps this onto the repo's import job state.
30 */
31 export class ImportClientError extends Schema.TaggedError<ImportClientError>()(
32 "ImportClientError",
33 { reason: Schema.String },
34 ) {}
35
36 /** The import source, as accepted by `POST /api/v1/repos/import`. */
37 export interface ImportSource {
38 /** Smart-HTTP base URL of the source repository (with or without `.git`). */
39 readonly url: string;
40 /** Restrict the import to a single ref (full or short name). */
41 readonly ref?: string | undefined;
42 /** Depth-limit the imported history (`deepen <n>`). */
43 readonly depth?: number | undefined;
44 }
45
46 /** One ref parsed from the remote advertisement. */
47 export interface RemoteRef {
48 readonly name: string;
49 readonly oid: string;
50 /** Peeled target from a `<name>^{}` line (annotated tags). */
51 readonly peeled?: string | undefined;
52 }
53
54 /** A parsed v0 `info/refs` advertisement. */
55 export interface RemoteAdvertisement {
56 readonly refs: ReadonlyArray<RemoteRef>;
57 /**
58 * The remote HEAD's symref target (e.g. `refs/heads/main`) from the
59 * `symref=HEAD:...` capability, or `null` when not advertised.
60 */
61 readonly head: string | null;
62 /** The remote's advertised capability set. */
63 readonly capabilities: ReadonlySet<string>;
64 }
65
66 /** The result of {@link runImport}: everything the Repo DO needs to ingest. */
67 export interface ImportResult {
68 /** The refs to create locally (already filtered by `source.ref`). */
69 readonly refs: ReadonlyArray<RemoteRef>;
70 /** Short default-branch name derived from the remote HEAD, or `null`. */
71 readonly defaultBranch: string | null;
72 /**
73 * The raw pack bytes, or `undefined` when the remote is empty (nothing
74 * to ingest).
75 */
76 readonly pack: Uint8Array | undefined;
77 /** `shallow <oid>` boundary commits reported by the remote. */
78 readonly shallow: ReadonlyArray<string>;
79 }
80
81 /** Options of {@link runImport}. */
82 export interface ImportOptions {
83 readonly source: ImportSource;
84 /**
85 * Hard cap on the fetched pack size (bytes).
86 * @default 52_428_800 (50 MiB, the same cap as push ingest)
87 */
88 readonly maxPackBytes?: number | undefined;
89 }
90
91 const DEFAULT_MAX_PACK = 50 * 1024 * 1024;
92
93 // ─────────────────────────────────────────────────────────────────────────────
94 // Advertisement
95 // ─────────────────────────────────────────────────────────────────────────────
96
97 const normalizeBase = (url: string): string => url.replace(/\/+$/, "");
98
99 /** Decodes all pkt-lines of a buffer into text/flush tokens. */
100 type Token = { readonly _tag: "line"; readonly text: string } | { readonly _tag: "flush" };
101
102 const tokenize = (buf: Uint8Array): Effect.Effect<ReadonlyArray<Token>, ImportClientError> =>
103 Effect.suspend(() => {
104 const out: Array<Token> = [];
105 let pos = 0;
106 while (pos < buf.length) {
107 const r = readPktLineAt(buf, pos);
108 if (r._tag === "incomplete") {
109 return Effect.fail(ImportClientError.make({ reason: "truncated advertisement" }));
110 }
111 if (r._tag === "invalid") {
112 return Effect.fail(ImportClientError.make({ reason: r.reason }));
113 }
114 if (r.pkt._tag === "flush") {
115 out.push({ _tag: "flush" });
116 } else if (r.pkt._tag === "data") {
117 out.push({ _tag: "line", text: pktPayloadText(r.pkt.payload) });
118 }
119 pos = r.next;
120 }
121 return Effect.succeed(out);
122 });
123
124 /**
125 * Parses a v0 `git-upload-pack` advertisement body (including the
126 * `# service=` prelude). Exposed for unit testing against fixture bytes.
127 */
128 export const parseAdvertisement = Effect.fn(function* (body: Uint8Array) {
129 const tokens = yield* tokenize(body);
130 const refs: Array<RemoteRef> = [];
131 const peeled = new Map<string, string>();
132 let capabilities: ReadonlySet<string> = new Set();
133 let head: string | null = null;
134 let sawCaps = false;
135
136 for (const token of tokens) {
137 if (token._tag === "flush") continue;
138 const text = token.text;
139 if (text.startsWith("#")) continue; // "# service=git-upload-pack"
140 if (text.startsWith("ERR ")) {
141 return yield* ImportClientError.make({
142 reason: `remote error: ${text.slice(4)}`,
143 });
144 }
145 let line = text;
146 if (!sawCaps) {
147 // First ref line carries "\0cap1 cap2 ..."
148 const nul = text.indexOf("\0");
149 if (nul !== -1) {
150 const caps = new Set(
151 text
152 .slice(nul + 1)
153 .split(" ")
154 .filter(Boolean),
155 );
156 capabilities = caps;
157 for (const cap of caps) {
158 if (cap.startsWith("symref=HEAD:")) {
159 head = cap.slice("symref=HEAD:".length);
160 }
161 }
162 line = text.slice(0, nul);
163 sawCaps = true;
164 }
165 }
166 const space = line.indexOf(" ");
167 if (space !== 40 || !isOid(line.slice(0, 40))) continue;
168 const oid = line.slice(0, 40);
169 const name = line.slice(space + 1);
170 if (oid === ZERO_OID && name === "capabilities^{}") {
171 continue; // empty repo advertisement
172 }
173 if (name.endsWith("^{}")) {
174 peeled.set(name.slice(0, -3), oid);
175 } else {
176 refs.push({ name, oid });
177 }
178 }
179
180 const withPeeled = refs.map((ref) => {
181 const p = peeled.get(ref.name);
182 return p === undefined ? ref : { ...ref, peeled: p };
183 });
184
185 return {
186 refs: withPeeled,
187 head,
188 capabilities,
189 } satisfies RemoteAdvertisement;
190 });
191
192 /**
193 * Fetches and parses the remote's `info/refs?service=git-upload-pack`
194 * advertisement.
195 */
196 export const fetchAdvertisement = Effect.fn(function* (url: string) {
197 const client = yield* HttpClient.HttpClient;
198 const response = yield* client
199 .get(`${normalizeBase(url)}/info/refs?service=git-upload-pack`)
200 .pipe(
201 Effect.mapError((error) =>
202 ImportClientError.make({ reason: `info/refs failed: ${error.message}` }),
203 ),
204 );
205 if (response.status !== 200) {
206 return yield* ImportClientError.make({
207 reason: `info/refs returned HTTP ${response.status}`,
208 });
209 }
210 const body = yield* response.arrayBuffer.pipe(
211 Effect.mapError((error) =>
212 ImportClientError.make({ reason: `info/refs body read: ${error.message}` }),
213 ),
214 );
215 return yield* parseAdvertisement(new Uint8Array(body));
216 });
217
218 // ─────────────────────────────────────────────────────────────────────────────
219 // upload-pack POST
220 // ─────────────────────────────────────────────────────────────────────────────
221
222 /** Builds the v0 upload-pack request body: wants, optional deepen, done. */
223 export const buildUploadPackRequest = (
224 wants: ReadonlyArray<string>,
225 options?: { readonly depth?: number | undefined },
226 ): Uint8Array => {
227 const parts: Array<Uint8Array> = [];
228 wants.forEach((oid, index) => {
229 parts.push(
230 index === 0
231 ? pktText(`want ${oid} ofs-delta agent=alchemy-git-service/1`)
232 : pktText(`want ${oid}`),
233 );
234 });
235 if (options?.depth !== undefined) {
236 parts.push(pktText(`deepen ${options.depth}`));
237 }
238 parts.push(flushPkt);
239 parts.push(pktText("done"));
240 return concatBytes(parts);
241 };
242
243 /**
244 * The parsed upload-pack response: the shallow boundary lines (when a
245 * `deepen` was requested) and the raw pack bytes that follow the `NAK`.
246 */
247 export interface UploadPackResponse {
248 readonly shallow: ReadonlyArray<string>;
249 readonly pack: Uint8Array;
250 }
251
252 /**
253 * Splits a (non-side-band) upload-pack response into its shallow section /
254 * NAK and the trailing raw pack. Exposed for unit testing.
255 */
256 export const parseUploadPackResponse = Effect.fn(function* (body: Uint8Array) {
257 const shallow: Array<string> = [];
258 let pos = 0;
259 for (;;) {
260 if (pos >= body.length) {
261 return yield* ImportClientError.make({
262 reason: "upload-pack response ended before NAK/ACK",
263 });
264 }
265 const r = readPktLineAt(body, pos);
266 if (r._tag === "incomplete" || r._tag === "invalid") {
267 return yield* ImportClientError.make({
268 reason: r._tag === "invalid" ? r.reason : "truncated upload-pack response",
269 });
270 }
271 pos = r.next;
272 if (r.pkt._tag !== "data") continue; // flush after the shallow section
273 const text = pktPayloadText(r.pkt.payload);
274 if (text.startsWith("shallow ")) {
275 const oid = text.slice("shallow ".length);
276 if (isOid(oid)) shallow.push(oid);
277 continue;
278 }
279 if (text.startsWith("unshallow ")) continue;
280 if (text.startsWith("ERR ")) {
281 return yield* ImportClientError.make({
282 reason: `remote error: ${text.slice(4)}`,
283 });
284 }
285 if (text === "NAK" || text.startsWith("ACK ")) {
286 // Pack bytes follow verbatim (no side-band was requested).
287 return {
288 shallow,
289 pack: body.subarray(pos),
290 } satisfies UploadPackResponse;
291 }
292 // Ignore anything else (progress from misbehaving servers).
293 }
294 });
295
296 /**
297 * POSTs the upload-pack request and returns the shallow lines + raw pack.
298 */
299 export const fetchPack = Effect.fn(function* (
300 url: string,
301 wants: ReadonlyArray<string>,
302 options?: {
303 readonly depth?: number | undefined;
304 readonly maxPackBytes?: number | undefined;
305 },
306 ) {
307 const client = yield* HttpClient.HttpClient;
308 const maxPack = options?.maxPackBytes ?? DEFAULT_MAX_PACK;
309 const request = HttpClientRequest.post(`${normalizeBase(url)}/git-upload-pack`).pipe(
310 HttpClientRequest.setHeaders({
311 "content-type": "application/x-git-upload-pack-request",
312 accept: "application/x-git-upload-pack-result",
313 }),
314 HttpClientRequest.bodyUint8Array(buildUploadPackRequest(wants, { depth: options?.depth })),
315 );
316 const response = yield* client
317 .execute(request)
318 .pipe(
319 Effect.mapError((error) =>
320 ImportClientError.make({ reason: `git-upload-pack failed: ${error.message}` }),
321 ),
322 );
323 if (response.status !== 200) {
324 return yield* ImportClientError.make({
325 reason: `git-upload-pack returned HTTP ${response.status}`,
326 });
327 }
328 const body = yield* response.arrayBuffer.pipe(
329 Effect.mapError((error) =>
330 ImportClientError.make({
331 reason: `git-upload-pack body read: ${error.message}`,
332 }),
333 ),
334 );
335 if (body.byteLength > maxPack + 65536) {
336 return yield* ImportClientError.make({
337 reason: `source pack exceeds the ${maxPack} byte import cap`,
338 });
339 }
340 const parsed = yield* parseUploadPackResponse(new Uint8Array(body));
341 if (parsed.pack.byteLength > maxPack) {
342 return yield* ImportClientError.make({
343 reason: `source pack exceeds the ${maxPack} byte import cap`,
344 });
345 }
346 return parsed;
347 });
348
349 // ─────────────────────────────────────────────────────────────────────────────
350 // Orchestration
351 // ─────────────────────────────────────────────────────────────────────────────
352
353 /** Resolves `source.ref` (full or short name) against advertised refs. */
354 const selectRefs = (
355 advertised: ReadonlyArray<RemoteRef>,
356 ref: string | undefined,
357 ): ReadonlyArray<RemoteRef> | undefined => {
358 if (ref === undefined) {
359 return advertised.filter(
360 (r) => r.name.startsWith("refs/heads/") || r.name.startsWith("refs/tags/"),
361 );
362 }
363 const candidates = [ref, `refs/heads/${ref}`, `refs/tags/${ref}`];
364 const match = advertised.find((r) => candidates.includes(r.name));
365 return match === undefined ? undefined : [match];
366 };
367
368 /**
369 * Runs the full import client flow: advertisement → ref selection →
370 * upload-pack POST. Returns everything the Repo DO's alarm job needs to
371 * ingest the history and create the refs. Requires `HttpClient.HttpClient`
372 * (provide `FetchHttpClient.layer`).
373 */
374 export const runImport = Effect.fn(function* (options: ImportOptions) {
375 const { source } = options;
376 const advertisement = yield* fetchAdvertisement(source.url);
377
378 const selected = selectRefs(advertisement.refs, source.ref);
379 if (selected === undefined) {
380 return yield* ImportClientError.make({
381 reason: `ref '${source.ref}' not found on the remote`,
382 });
383 }
384
385 const defaultBranch =
386 advertisement.head !== null && advertisement.head.startsWith("refs/heads/")
387 ? advertisement.head.slice("refs/heads/".length)
388 : null;
389
390 if (selected.length === 0) {
391 // Empty remote — nothing to fetch.
392 return {
393 refs: [],
394 defaultBranch,
395 pack: undefined,
396 shallow: [],
397 } satisfies ImportResult;
398 }
399
400 // Want the peeled targets too? No — wanting the tag object oid pulls the
401 // whole chain; peeled lines are advisory.
402 const wants = Array.from(new Set(selected.map((r) => r.oid)));
403 const { shallow, pack } = yield* fetchPack(source.url, wants, {
404 depth: source.depth,
405 maxPackBytes: options.maxPackBytes,
406 });
407
408 return {
409 refs: selected,
410 defaultBranch,
411 pack,
412 shallow,
413 } satisfies ImportResult;
414 });