The best way to control Claude Code, Codex and more from any computer or phone
Mirrored from https://github.com/Camo67/t3code.git
Synced Nov 21, 04:45 UTC
| 1 | /** |
| 2 | * The fork alarm job (DESIGN.md §2.3 Fork). |
| 3 | * |
| 4 | * A fork streams a row snapshot from the parent Repo DO over an RPC |
| 5 | * `Stream` and copies it in batches: |
| 6 | * |
| 7 | * - `config` — only the fork-relevant keys (`default_branch`, |
| 8 | * `description`); identity keys (repoId/owner/name/created_at/status) |
| 9 | * belong to the fork itself. |
| 10 | * - `refs`, `commits`, `commit_parents` — copied verbatim. |
| 11 | * - `objects` — `location='row'` rows carry their `zdata` BLOB; |
| 12 | * `location='r2'` rows are copied **by reference**: the `r2_key` column |
| 13 | * carries the parent's full key, which is immutable and shared |
| 14 | * (DESIGN.md §3.2). The parent's R2 prefix is retained while the |
| 15 | * Registry reports `fork_count > 0`. |
| 16 | * |
| 17 | * Every insert is `INSERT OR IGNORE` keyed on the natural primary key, so |
| 18 | * the job is idempotent and alarm-re-armable: a crashed run resumes by |
| 19 | * re-streaming from the start and skipping already-copied rows. |
| 20 | */ |
| 21 | import * as Effect from "effect/Effect"; |
| 22 | import { Base64 } from "effect/encoding"; |
| 23 | import * as Option from "effect/Option"; |
| 24 | import * as Result from "effect/Result"; |
| 25 | import * as Stream from "effect/Stream"; |
| 26 | import { StoreError } from "../Protocol/Store.ts"; |
| 27 | import { LIVE_OBJECTS } from "../Store/ObjectStore.ts"; |
| 28 | import type { CommitParentRow, CommitRow, ObjectRow, RefRow, SqlClient } from "../Store/Sql.ts"; |
| 29 | |
| 30 | // ───────────────────────────────────────────────────────────────────────────── |
| 31 | // Snapshot wire shapes |
| 32 | // ───────────────────────────────────────────────────────────────────────────── |
| 33 | |
| 34 | /** |
| 35 | * A snapshotted `objects` row. `zdata` is `null` when `location != 'row'`. |
| 36 | * |
| 37 | * NOTE: `zdata` crosses the RPC boundary as **base64**, not raw bytes. The |
| 38 | * snapshot is a `Stream` of plain objects, which the RPC layer serializes |
| 39 | * as JSONL — a `Uint8Array` inside one of those objects would arrive as |
| 40 | * `{"0":31,"1":139,...}` and silently corrupt every forked object. |
| 41 | */ |
| 42 | export interface SnapshotObjectRow { |
| 43 | readonly oid: string; |
| 44 | readonly type: number; |
| 45 | readonly size: number; |
| 46 | readonly zsize: number; |
| 47 | readonly location: string; |
| 48 | /** base64 of `zlib(content)`, or `null` when the bytes live in R2. */ |
| 49 | readonly zdata: string | null; |
| 50 | /** Full R2 key — may point into the PARENT repo's prefix (shared). */ |
| 51 | readonly r2_key: string | null; |
| 52 | readonly pack_id: string | null; |
| 53 | readonly pack_offset: number | null; |
| 54 | } |
| 55 | |
| 56 | /** |
| 57 | * One batch of snapshotted rows from a single table. The union member is |
| 58 | * discriminated by `table`. |
| 59 | */ |
| 60 | export type SnapshotChunk = |
| 61 | | { |
| 62 | readonly table: "config"; |
| 63 | readonly rows: ReadonlyArray<{ readonly key: string; readonly value: string }>; |
| 64 | } |
| 65 | | { |
| 66 | readonly table: "refs"; |
| 67 | readonly rows: ReadonlyArray<{ readonly name: string; readonly oid: string }>; |
| 68 | } |
| 69 | | { readonly table: "objects"; readonly rows: ReadonlyArray<SnapshotObjectRow> } |
| 70 | | { |
| 71 | readonly table: "commits"; |
| 72 | readonly rows: ReadonlyArray<{ |
| 73 | readonly oid: string; |
| 74 | readonly tree: string; |
| 75 | readonly gen: number; |
| 76 | readonly commit_time: number; |
| 77 | }>; |
| 78 | } |
| 79 | | { |
| 80 | readonly table: "commit_parents"; |
| 81 | readonly rows: ReadonlyArray<{ |
| 82 | readonly oid: string; |
| 83 | readonly parent: string; |
| 84 | readonly ord: number; |
| 85 | }>; |
| 86 | }; |
| 87 | |
| 88 | /** Decodes a snapshot row's base64 `zdata` back into bytes. */ |
| 89 | const decodeZData = (row: SnapshotObjectRow): Effect.Effect<Uint8Array, StoreError> => { |
| 90 | const decoded = Base64.decode(row.zdata ?? ""); |
| 91 | return Result.isSuccess(decoded) |
| 92 | ? Effect.succeed(decoded.success) |
| 93 | : Effect.fail(StoreError.make({ reason: `fork: corrupt zdata for ${row.oid}` })); |
| 94 | }; |
| 95 | |
| 96 | /** Config keys that transfer from parent to fork. */ |
| 97 | const FORKED_CONFIG_KEYS = ["default_branch", "description"] as const; |
| 98 | |
| 99 | /** Rows per page for small-row tables. */ |
| 100 | const PAGE_SIZE = 400; |
| 101 | /** Rows per page for `objects` (each `zdata` BLOB may be up to 1 MiB). */ |
| 102 | const OBJECTS_PAGE_SIZE = 8; |
| 103 | |
| 104 | // ───────────────────────────────────────────────────────────────────────────── |
| 105 | // Parent side: produce the snapshot stream |
| 106 | // ───────────────────────────────────────────────────────────────────────────── |
| 107 | |
| 108 | /** |
| 109 | * Streams a point-in-time row snapshot of one repo (the parent side of a |
| 110 | * fork). Backs the Repo DO's `snapshotRows` RPC method — DO RPC methods may |
| 111 | * return `Stream`s, and the caller (the fork's alarm job) sees chunks live. |
| 112 | * |
| 113 | * Staged (in-flight push) objects are excluded; keyed pagination keeps |
| 114 | * per-batch memory bounded. The stream is not transactional across pages — |
| 115 | * a concurrent push may add rows mid-snapshot; forks take whatever refs |
| 116 | * page they observe, which is the same semantics as `git clone` during a |
| 117 | * push. |
| 118 | */ |
| 119 | export const snapshotStream = (sql: SqlClient): Stream.Stream<SnapshotChunk, StoreError> => { |
| 120 | const config: Stream.Stream<SnapshotChunk, StoreError> = Stream.fromEffect( |
| 121 | Effect.gen(function* () { |
| 122 | const rows = yield* sql.all<{ key: string; value: string }>( |
| 123 | `SELECT key, value FROM config WHERE key IN (${FORKED_CONFIG_KEYS.map(() => "?").join(", ")})`, |
| 124 | ...FORKED_CONFIG_KEYS, |
| 125 | ); |
| 126 | return { table: "config", rows } satisfies SnapshotChunk; |
| 127 | }), |
| 128 | ); |
| 129 | |
| 130 | const refs: Stream.Stream<SnapshotChunk, StoreError> = Stream.paginate("", (after: string) => |
| 131 | Effect.gen(function* () { |
| 132 | const rows = yield* sql.all<RefRow>( |
| 133 | `SELECT name, oid FROM refs WHERE name > ? ORDER BY name LIMIT ?`, |
| 134 | after, |
| 135 | PAGE_SIZE, |
| 136 | ); |
| 137 | const chunk: SnapshotChunk = { table: "refs", rows }; |
| 138 | const next = |
| 139 | rows.length < PAGE_SIZE ? Option.none<string>() : Option.some(rows[rows.length - 1]!.name); |
| 140 | return [rows.length > 0 ? [chunk] : [], next] as const; |
| 141 | }), |
| 142 | ); |
| 143 | |
| 144 | const objects: Stream.Stream<SnapshotChunk, StoreError> = Stream.paginate("", (after: string) => |
| 145 | Effect.gen(function* () { |
| 146 | const rows = yield* sql.all<ObjectRow>( |
| 147 | `SELECT oid, type, size, zsize, location, zdata, r2_key, pack_id, pack_offset, staged_push |
| 148 | FROM objects WHERE oid > ? AND ${LIVE_OBJECTS} ORDER BY oid LIMIT ?`, |
| 149 | after, |
| 150 | OBJECTS_PAGE_SIZE, |
| 151 | ); |
| 152 | const chunk: SnapshotChunk = { |
| 153 | table: "objects", |
| 154 | rows: rows.map((row): SnapshotObjectRow => ({ |
| 155 | oid: row.oid, |
| 156 | type: row.type, |
| 157 | size: row.size, |
| 158 | zsize: row.zsize, |
| 159 | location: row.location, |
| 160 | zdata: row.zdata === null ? null : Base64.encode(new Uint8Array(row.zdata)), |
| 161 | r2_key: row.r2_key, |
| 162 | pack_id: row.pack_id, |
| 163 | pack_offset: row.pack_offset, |
| 164 | })), |
| 165 | }; |
| 166 | const next = |
| 167 | rows.length < OBJECTS_PAGE_SIZE |
| 168 | ? Option.none<string>() |
| 169 | : Option.some(rows[rows.length - 1]!.oid); |
| 170 | return [rows.length > 0 ? [chunk] : [], next] as const; |
| 171 | }), |
| 172 | ); |
| 173 | |
| 174 | const commits: Stream.Stream<SnapshotChunk, StoreError> = Stream.paginate("", (after: string) => |
| 175 | Effect.gen(function* () { |
| 176 | const rows = yield* sql.all<CommitRow>( |
| 177 | `SELECT oid, tree, gen, commit_time FROM commits WHERE oid > ? ORDER BY oid LIMIT ?`, |
| 178 | after, |
| 179 | PAGE_SIZE, |
| 180 | ); |
| 181 | const chunk: SnapshotChunk = { table: "commits", rows }; |
| 182 | const next = |
| 183 | rows.length < PAGE_SIZE ? Option.none<string>() : Option.some(rows[rows.length - 1]!.oid); |
| 184 | return [rows.length > 0 ? [chunk] : [], next] as const; |
| 185 | }), |
| 186 | ); |
| 187 | |
| 188 | const commitParents: Stream.Stream<SnapshotChunk, StoreError> = Stream.paginate( |
| 189 | ["", ""] as readonly [string, string], |
| 190 | (after) => |
| 191 | Effect.gen(function* () { |
| 192 | const rows = yield* sql.all<CommitParentRow>( |
| 193 | `SELECT oid, parent, ord FROM commit_parents |
| 194 | WHERE (oid > ? OR (oid = ? AND parent > ?)) ORDER BY oid, parent LIMIT ?`, |
| 195 | after[0], |
| 196 | after[0], |
| 197 | after[1], |
| 198 | PAGE_SIZE, |
| 199 | ); |
| 200 | const chunk: SnapshotChunk = { table: "commit_parents", rows }; |
| 201 | const last = rows[rows.length - 1]; |
| 202 | const next = |
| 203 | rows.length < PAGE_SIZE || last === undefined |
| 204 | ? Option.none<readonly [string, string]>() |
| 205 | : Option.some([last.oid, last.parent] as const); |
| 206 | return [rows.length > 0 ? [chunk] : [], next] as const; |
| 207 | }), |
| 208 | ); |
| 209 | |
| 210 | // Objects before refs: a partially-copied fork must never advertise a ref |
| 211 | // whose closure is missing (the job is re-run from scratch anyway, but |
| 212 | // this ordering makes even a crashed fork's DB internally consistent). |
| 213 | return config.pipe( |
| 214 | Stream.concat(objects), |
| 215 | Stream.concat(commits), |
| 216 | Stream.concat(commitParents), |
| 217 | Stream.concat(refs), |
| 218 | ); |
| 219 | }; |
| 220 | |
| 221 | // ───────────────────────────────────────────────────────────────────────────── |
| 222 | // Fork side: apply the snapshot |
| 223 | // ───────────────────────────────────────────────────────────────────────────── |
| 224 | |
| 225 | /** Zero-copy-when-possible Uint8Array → ArrayBuffer for BLOB binding. */ |
| 226 | const toArrayBuffer = (bytes: Uint8Array): ArrayBuffer => |
| 227 | bytes.byteOffset === 0 && |
| 228 | bytes.byteLength === bytes.buffer.byteLength && |
| 229 | bytes.buffer instanceof ArrayBuffer |
| 230 | ? bytes.buffer |
| 231 | : (bytes.buffer.slice(bytes.byteOffset, bytes.byteOffset + bytes.byteLength) as ArrayBuffer); |
| 232 | |
| 233 | /** |
| 234 | * Applies one snapshot chunk to the fork's own SQLite. All inserts are |
| 235 | * `INSERT OR IGNORE` on the natural primary key — re-running the whole |
| 236 | * snapshot after a crash converges without duplicating rows. |
| 237 | */ |
| 238 | export const applySnapshotChunk = Effect.fn(function* (sql: SqlClient, chunk: SnapshotChunk) { |
| 239 | switch (chunk.table) { |
| 240 | case "config": { |
| 241 | for (const row of chunk.rows) { |
| 242 | yield* sql.run( |
| 243 | `INSERT INTO config (key, value) VALUES (?, ?) |
| 244 | ON CONFLICT (key) DO UPDATE SET value = excluded.value`, |
| 245 | row.key, |
| 246 | row.value, |
| 247 | ); |
| 248 | } |
| 249 | return; |
| 250 | } |
| 251 | case "refs": { |
| 252 | for (const row of chunk.rows) { |
| 253 | yield* sql.run(`INSERT OR IGNORE INTO refs (name, oid) VALUES (?, ?)`, row.name, row.oid); |
| 254 | } |
| 255 | return; |
| 256 | } |
| 257 | case "objects": { |
| 258 | for (const row of chunk.rows) { |
| 259 | yield* sql.run( |
| 260 | `INSERT OR IGNORE INTO objects (oid, type, size, zsize, location, zdata, r2_key, pack_id, pack_offset, staged_push) |
| 261 | VALUES (?, ?, ?, ?, ?, ?, ?, ?, ?, NULL)`, |
| 262 | row.oid, |
| 263 | row.type, |
| 264 | row.size, |
| 265 | row.zsize, |
| 266 | row.location, |
| 267 | row.zdata === null ? null : toArrayBuffer(yield* decodeZData(row)), |
| 268 | row.r2_key, |
| 269 | row.pack_id, |
| 270 | row.pack_offset, |
| 271 | ); |
| 272 | } |
| 273 | return; |
| 274 | } |
| 275 | case "commits": { |
| 276 | for (const row of chunk.rows) { |
| 277 | yield* sql.run( |
| 278 | `INSERT OR IGNORE INTO commits (oid, tree, gen, commit_time) VALUES (?, ?, ?, ?)`, |
| 279 | row.oid, |
| 280 | row.tree, |
| 281 | row.gen, |
| 282 | row.commit_time, |
| 283 | ); |
| 284 | } |
| 285 | return; |
| 286 | } |
| 287 | case "commit_parents": { |
| 288 | for (const row of chunk.rows) { |
| 289 | yield* sql.run( |
| 290 | `INSERT OR IGNORE INTO commit_parents (oid, parent, ord) VALUES (?, ?, ?)`, |
| 291 | row.oid, |
| 292 | row.parent, |
| 293 | row.ord, |
| 294 | ); |
| 295 | } |
| 296 | return; |
| 297 | } |
| 298 | } |
| 299 | }); |
| 300 | |
| 301 | /** Dependencies of {@link runForkJob}. */ |
| 302 | export interface ForkJobOptions { |
| 303 | /** The fork's own SQL client. */ |
| 304 | readonly sql: SqlClient; |
| 305 | /** |
| 306 | * The parent's snapshot stream — `parentStub.snapshotRows()` from the |
| 307 | * fork DO's alarm handler. Error type is widened to `unknown` because |
| 308 | * the stream crosses the RPC boundary. |
| 309 | */ |
| 310 | readonly snapshot: Stream.Stream<SnapshotChunk, unknown>; |
| 311 | } |
| 312 | |
| 313 | /** |
| 314 | * Drains the parent snapshot into the fork's SQLite. Returns the number of |
| 315 | * chunks applied. Idempotent; the caller flips `status` to `ready` (and |
| 316 | * clears the jobs row) on success, or re-arms the alarm on failure. |
| 317 | */ |
| 318 | export const runForkJob = (options: ForkJobOptions): Effect.Effect<number, StoreError> => |
| 319 | options.snapshot.pipe( |
| 320 | Stream.mapError((error) => |
| 321 | StoreError.make({ reason: `fork snapshot stream failed: ${String(error)}` }), |
| 322 | ), |
| 323 | Stream.runFoldEffect( |
| 324 | () => 0, |
| 325 | (count, chunk) => applySnapshotChunk(options.sql, chunk).pipe(Effect.as(count + 1)), |
| 326 | ), |
| 327 | ); |