The best way to control Claude Code, Codex and more from any computer or phone

main
ts 327 lines 12.7 KB
Raw
1 /**
2 * The fork alarm job (DESIGN.md §2.3 Fork).
3 *
4 * A fork streams a row snapshot from the parent Repo DO over an RPC
5 * `Stream` and copies it in batches:
6 *
7 * - `config` — only the fork-relevant keys (`default_branch`,
8 * `description`); identity keys (repoId/owner/name/created_at/status)
9 * belong to the fork itself.
10 * - `refs`, `commits`, `commit_parents` — copied verbatim.
11 * - `objects` — `location='row'` rows carry their `zdata` BLOB;
12 * `location='r2'` rows are copied **by reference**: the `r2_key` column
13 * carries the parent's full key, which is immutable and shared
14 * (DESIGN.md §3.2). The parent's R2 prefix is retained while the
15 * Registry reports `fork_count > 0`.
16 *
17 * Every insert is `INSERT OR IGNORE` keyed on the natural primary key, so
18 * the job is idempotent and alarm-re-armable: a crashed run resumes by
19 * re-streaming from the start and skipping already-copied rows.
20 */
21 import * as Effect from "effect/Effect";
22 import { Base64 } from "effect/encoding";
23 import * as Option from "effect/Option";
24 import * as Result from "effect/Result";
25 import * as Stream from "effect/Stream";
26 import { StoreError } from "../Protocol/Store.ts";
27 import { LIVE_OBJECTS } from "../Store/ObjectStore.ts";
28 import type { CommitParentRow, CommitRow, ObjectRow, RefRow, SqlClient } from "../Store/Sql.ts";
29
30 // ─────────────────────────────────────────────────────────────────────────────
31 // Snapshot wire shapes
32 // ─────────────────────────────────────────────────────────────────────────────
33
34 /**
35 * A snapshotted `objects` row. `zdata` is `null` when `location != 'row'`.
36 *
37 * NOTE: `zdata` crosses the RPC boundary as **base64**, not raw bytes. The
38 * snapshot is a `Stream` of plain objects, which the RPC layer serializes
39 * as JSONL — a `Uint8Array` inside one of those objects would arrive as
40 * `{"0":31,"1":139,...}` and silently corrupt every forked object.
41 */
42 export interface SnapshotObjectRow {
43 readonly oid: string;
44 readonly type: number;
45 readonly size: number;
46 readonly zsize: number;
47 readonly location: string;
48 /** base64 of `zlib(content)`, or `null` when the bytes live in R2. */
49 readonly zdata: string | null;
50 /** Full R2 key — may point into the PARENT repo's prefix (shared). */
51 readonly r2_key: string | null;
52 readonly pack_id: string | null;
53 readonly pack_offset: number | null;
54 }
55
56 /**
57 * One batch of snapshotted rows from a single table. The union member is
58 * discriminated by `table`.
59 */
60 export type SnapshotChunk =
61 | {
62 readonly table: "config";
63 readonly rows: ReadonlyArray<{ readonly key: string; readonly value: string }>;
64 }
65 | {
66 readonly table: "refs";
67 readonly rows: ReadonlyArray<{ readonly name: string; readonly oid: string }>;
68 }
69 | { readonly table: "objects"; readonly rows: ReadonlyArray<SnapshotObjectRow> }
70 | {
71 readonly table: "commits";
72 readonly rows: ReadonlyArray<{
73 readonly oid: string;
74 readonly tree: string;
75 readonly gen: number;
76 readonly commit_time: number;
77 }>;
78 }
79 | {
80 readonly table: "commit_parents";
81 readonly rows: ReadonlyArray<{
82 readonly oid: string;
83 readonly parent: string;
84 readonly ord: number;
85 }>;
86 };
87
88 /** Decodes a snapshot row's base64 `zdata` back into bytes. */
89 const decodeZData = (row: SnapshotObjectRow): Effect.Effect<Uint8Array, StoreError> => {
90 const decoded = Base64.decode(row.zdata ?? "");
91 return Result.isSuccess(decoded)
92 ? Effect.succeed(decoded.success)
93 : Effect.fail(StoreError.make({ reason: `fork: corrupt zdata for ${row.oid}` }));
94 };
95
96 /** Config keys that transfer from parent to fork. */
97 const FORKED_CONFIG_KEYS = ["default_branch", "description"] as const;
98
99 /** Rows per page for small-row tables. */
100 const PAGE_SIZE = 400;
101 /** Rows per page for `objects` (each `zdata` BLOB may be up to 1 MiB). */
102 const OBJECTS_PAGE_SIZE = 8;
103
104 // ─────────────────────────────────────────────────────────────────────────────
105 // Parent side: produce the snapshot stream
106 // ─────────────────────────────────────────────────────────────────────────────
107
108 /**
109 * Streams a point-in-time row snapshot of one repo (the parent side of a
110 * fork). Backs the Repo DO's `snapshotRows` RPC method — DO RPC methods may
111 * return `Stream`s, and the caller (the fork's alarm job) sees chunks live.
112 *
113 * Staged (in-flight push) objects are excluded; keyed pagination keeps
114 * per-batch memory bounded. The stream is not transactional across pages —
115 * a concurrent push may add rows mid-snapshot; forks take whatever refs
116 * page they observe, which is the same semantics as `git clone` during a
117 * push.
118 */
119 export const snapshotStream = (sql: SqlClient): Stream.Stream<SnapshotChunk, StoreError> => {
120 const config: Stream.Stream<SnapshotChunk, StoreError> = Stream.fromEffect(
121 Effect.gen(function* () {
122 const rows = yield* sql.all<{ key: string; value: string }>(
123 `SELECT key, value FROM config WHERE key IN (${FORKED_CONFIG_KEYS.map(() => "?").join(", ")})`,
124 ...FORKED_CONFIG_KEYS,
125 );
126 return { table: "config", rows } satisfies SnapshotChunk;
127 }),
128 );
129
130 const refs: Stream.Stream<SnapshotChunk, StoreError> = Stream.paginate("", (after: string) =>
131 Effect.gen(function* () {
132 const rows = yield* sql.all<RefRow>(
133 `SELECT name, oid FROM refs WHERE name > ? ORDER BY name LIMIT ?`,
134 after,
135 PAGE_SIZE,
136 );
137 const chunk: SnapshotChunk = { table: "refs", rows };
138 const next =
139 rows.length < PAGE_SIZE ? Option.none<string>() : Option.some(rows[rows.length - 1]!.name);
140 return [rows.length > 0 ? [chunk] : [], next] as const;
141 }),
142 );
143
144 const objects: Stream.Stream<SnapshotChunk, StoreError> = Stream.paginate("", (after: string) =>
145 Effect.gen(function* () {
146 const rows = yield* sql.all<ObjectRow>(
147 `SELECT oid, type, size, zsize, location, zdata, r2_key, pack_id, pack_offset, staged_push
148 FROM objects WHERE oid > ? AND ${LIVE_OBJECTS} ORDER BY oid LIMIT ?`,
149 after,
150 OBJECTS_PAGE_SIZE,
151 );
152 const chunk: SnapshotChunk = {
153 table: "objects",
154 rows: rows.map((row): SnapshotObjectRow => ({
155 oid: row.oid,
156 type: row.type,
157 size: row.size,
158 zsize: row.zsize,
159 location: row.location,
160 zdata: row.zdata === null ? null : Base64.encode(new Uint8Array(row.zdata)),
161 r2_key: row.r2_key,
162 pack_id: row.pack_id,
163 pack_offset: row.pack_offset,
164 })),
165 };
166 const next =
167 rows.length < OBJECTS_PAGE_SIZE
168 ? Option.none<string>()
169 : Option.some(rows[rows.length - 1]!.oid);
170 return [rows.length > 0 ? [chunk] : [], next] as const;
171 }),
172 );
173
174 const commits: Stream.Stream<SnapshotChunk, StoreError> = Stream.paginate("", (after: string) =>
175 Effect.gen(function* () {
176 const rows = yield* sql.all<CommitRow>(
177 `SELECT oid, tree, gen, commit_time FROM commits WHERE oid > ? ORDER BY oid LIMIT ?`,
178 after,
179 PAGE_SIZE,
180 );
181 const chunk: SnapshotChunk = { table: "commits", rows };
182 const next =
183 rows.length < PAGE_SIZE ? Option.none<string>() : Option.some(rows[rows.length - 1]!.oid);
184 return [rows.length > 0 ? [chunk] : [], next] as const;
185 }),
186 );
187
188 const commitParents: Stream.Stream<SnapshotChunk, StoreError> = Stream.paginate(
189 ["", ""] as readonly [string, string],
190 (after) =>
191 Effect.gen(function* () {
192 const rows = yield* sql.all<CommitParentRow>(
193 `SELECT oid, parent, ord FROM commit_parents
194 WHERE (oid > ? OR (oid = ? AND parent > ?)) ORDER BY oid, parent LIMIT ?`,
195 after[0],
196 after[0],
197 after[1],
198 PAGE_SIZE,
199 );
200 const chunk: SnapshotChunk = { table: "commit_parents", rows };
201 const last = rows[rows.length - 1];
202 const next =
203 rows.length < PAGE_SIZE || last === undefined
204 ? Option.none<readonly [string, string]>()
205 : Option.some([last.oid, last.parent] as const);
206 return [rows.length > 0 ? [chunk] : [], next] as const;
207 }),
208 );
209
210 // Objects before refs: a partially-copied fork must never advertise a ref
211 // whose closure is missing (the job is re-run from scratch anyway, but
212 // this ordering makes even a crashed fork's DB internally consistent).
213 return config.pipe(
214 Stream.concat(objects),
215 Stream.concat(commits),
216 Stream.concat(commitParents),
217 Stream.concat(refs),
218 );
219 };
220
221 // ─────────────────────────────────────────────────────────────────────────────
222 // Fork side: apply the snapshot
223 // ─────────────────────────────────────────────────────────────────────────────
224
225 /** Zero-copy-when-possible Uint8Array → ArrayBuffer for BLOB binding. */
226 const toArrayBuffer = (bytes: Uint8Array): ArrayBuffer =>
227 bytes.byteOffset === 0 &&
228 bytes.byteLength === bytes.buffer.byteLength &&
229 bytes.buffer instanceof ArrayBuffer
230 ? bytes.buffer
231 : (bytes.buffer.slice(bytes.byteOffset, bytes.byteOffset + bytes.byteLength) as ArrayBuffer);
232
233 /**
234 * Applies one snapshot chunk to the fork's own SQLite. All inserts are
235 * `INSERT OR IGNORE` on the natural primary key — re-running the whole
236 * snapshot after a crash converges without duplicating rows.
237 */
238 export const applySnapshotChunk = Effect.fn(function* (sql: SqlClient, chunk: SnapshotChunk) {
239 switch (chunk.table) {
240 case "config": {
241 for (const row of chunk.rows) {
242 yield* sql.run(
243 `INSERT INTO config (key, value) VALUES (?, ?)
244 ON CONFLICT (key) DO UPDATE SET value = excluded.value`,
245 row.key,
246 row.value,
247 );
248 }
249 return;
250 }
251 case "refs": {
252 for (const row of chunk.rows) {
253 yield* sql.run(`INSERT OR IGNORE INTO refs (name, oid) VALUES (?, ?)`, row.name, row.oid);
254 }
255 return;
256 }
257 case "objects": {
258 for (const row of chunk.rows) {
259 yield* sql.run(
260 `INSERT OR IGNORE INTO objects (oid, type, size, zsize, location, zdata, r2_key, pack_id, pack_offset, staged_push)
261 VALUES (?, ?, ?, ?, ?, ?, ?, ?, ?, NULL)`,
262 row.oid,
263 row.type,
264 row.size,
265 row.zsize,
266 row.location,
267 row.zdata === null ? null : toArrayBuffer(yield* decodeZData(row)),
268 row.r2_key,
269 row.pack_id,
270 row.pack_offset,
271 );
272 }
273 return;
274 }
275 case "commits": {
276 for (const row of chunk.rows) {
277 yield* sql.run(
278 `INSERT OR IGNORE INTO commits (oid, tree, gen, commit_time) VALUES (?, ?, ?, ?)`,
279 row.oid,
280 row.tree,
281 row.gen,
282 row.commit_time,
283 );
284 }
285 return;
286 }
287 case "commit_parents": {
288 for (const row of chunk.rows) {
289 yield* sql.run(
290 `INSERT OR IGNORE INTO commit_parents (oid, parent, ord) VALUES (?, ?, ?)`,
291 row.oid,
292 row.parent,
293 row.ord,
294 );
295 }
296 return;
297 }
298 }
299 });
300
301 /** Dependencies of {@link runForkJob}. */
302 export interface ForkJobOptions {
303 /** The fork's own SQL client. */
304 readonly sql: SqlClient;
305 /**
306 * The parent's snapshot stream — `parentStub.snapshotRows()` from the
307 * fork DO's alarm handler. Error type is widened to `unknown` because
308 * the stream crosses the RPC boundary.
309 */
310 readonly snapshot: Stream.Stream<SnapshotChunk, unknown>;
311 }
312
313 /**
314 * Drains the parent snapshot into the fork's SQLite. Returns the number of
315 * chunks applied. Idempotent; the caller flips `status` to `ready` (and
316 * clears the jobs row) on success, or re-arms the alarm on failure.
317 */
318 export const runForkJob = (options: ForkJobOptions): Effect.Effect<number, StoreError> =>
319 options.snapshot.pipe(
320 Stream.mapError((error) =>
321 StoreError.make({ reason: `fork snapshot stream failed: ${String(error)}` }),
322 ),
323 Stream.runFoldEffect(
324 () => 0,
325 (count, chunk) => applySnapshotChunk(options.sql, chunk).pipe(Effect.as(count + 1)),
326 ),
327 );