The best way to control Claude Code, Codex and more from any computer or phone

main
ts 975 lines 38.8 KB
Raw
1 import { RuntimeContext } from "alchemy/RuntimeContext";
2 import * as Effect from "effect/Effect";
3 import * as Schema from "effect/Schema";
4 import * as Stream from "effect/Stream";
5 /**
6 * The concrete object store: DO SQLite rows + R2 overflow (DESIGN.md §3.3).
7 *
8 * Implements the git layer's `ObjectSource` contract over the `objects`
9 * table, dispatching byte reads on the `location` column:
10 *
11 * - `'row'` — the `zdata` BLOB (zlib(content), no loose header),
12 * - `'r2'` — a streamed R2 `get` of `r2_key` (which may point into a fork
13 * parent's prefix — immutable, shared),
14 * - `'pack'` — v1.1 compaction; reads die with {@link NotImplementedError}
15 * until the compaction alarm lands (schema-ready, DESIGN.md §10).
16 *
17 * Plus the push-side surface the protocol layer needs: staged inserts with
18 * the row/R2 placement policy, batched metadata reads, and the
19 * staged-∪-live membership oracle behind the receive-pack connectivity
20 * check (DESIGN.md §3.6 step 5).
21 *
22 * All reads on the `ObjectSource` surface see **live objects only**
23 * (`staged_push IS NULL`); staged rows become visible to fetches only after
24 * the final `transactionSync` flips them live.
25 */
26 import type { BlobStoreError, BlobStoreShape } from "../BlobStore.ts";
27 import type { ObjectType, Oid } from "../Protocol/ObjectCodec.ts";
28 import { concatBytes, encodeTypeSize, type PackEntryType } from "../Protocol/ObjectCodec.ts";
29 import type { ManifestEntry } from "../Protocol/Store.ts";
30 import {
31 StoreError,
32 type ObjectLocation,
33 type ObjectMeta,
34 type ObjectSource,
35 } from "../Protocol/Store.ts";
36 import * as Zlib from "../Protocol/Zlib.ts";
37 import { objectKey, packKeyOf } from "./Keys.ts";
38 import type { ObjectMetaRow, SqlClient } from "./Sql.ts";
39
40 // ─────────────────────────────────────────────────────────────────────────────
41 // Constants & errors
42 // ─────────────────────────────────────────────────────────────────────────────
43
44 /**
45 * Compressed-size threshold above which an object's bytes are offloaded to
46 * R2 (`location = 'r2'`) instead of a row BLOB — 1 MiB, a safety margin
47 * under the 2 MB SQLite row cap (DESIGN.md §3.3).
48 */
49 export const R2_OFFLOAD_THRESHOLD = 1_048_576;
50
51 /**
52 * Per-object uncompressed size cap for v1 — 64 MiB. Delta application needs
53 * base + result in memory; this keeps worst-case ingest memory bounded.
54 * Enforced by `PackParser` (rejected with `unpack object too large`);
55 * exported here as the storage layer's authoritative constant.
56 */
57 export const MAX_OBJECT_SIZE = 67_108_864;
58
59 /**
60 * Window size for coalesced reads out of compacted packs (DESIGN §15
61 * bottleneck 2). Touching one object in a window fetches the whole window,
62 * so a fetch that walks thousands of objects costs a handful of R2 GETs
63 * instead of thousands.
64 */
65 export const WINDOW_BYTES = 4 * 1024 * 1024;
66
67 /**
68 * Objects per batch on the emission/closure paths — one SQL statement and
69 * one emitted chunk per batch instead of per object (DESIGN §22).
70 */
71 export const EMIT_BATCH = 256;
72
73 /**
74 * Rows per multi-row `INSERT` while staging a push: 6 bindings per row, and
75 * Durable Object SQLite allows at most 100 bound parameters per statement
76 * (`MAX_IN_PARAMS`) — 16 rows is 96. Still a 16x cut in statement
77 * executions over one row per statement (see `insertStagedBatch`).
78 */
79 export const STAGE_INSERT_ROWS = 16;
80
81 const chunkArray = <T>(items: ReadonlyArray<T>, size: number): Array<Array<T>> => {
82 const out: Array<Array<T>> = [];
83 for (let i = 0; i < items.length; i += size) {
84 out.push(items.slice(i, i + size) as Array<T>);
85 }
86 return out;
87 };
88
89 /**
90 * Isolate-wide budget for retained pack windows — 32 MiB (8 windows).
91 *
92 * The cache is a module-level LRU shared by every repo instance in the
93 * isolate, not a per-instance map: Durable Object instances of one class
94 * share an isolate and its 128 MiB, so per-instance caches multiply with
95 * the number of active repos. Measured (DESIGN §22.4): a 36 MiB push into a
96 * second repo was rejected with "isolate exceeded its memory limit" while
97 * the first repo's caches were resident. Keys carry the repo id
98 * (`packKey`), so sharing is safe.
99 */
100 export const WINDOW_CACHE_BYTES = 32 * 1024 * 1024;
101
102 /**
103 * Windows fetched ahead of the one being emitted (DESIGN §22.4). Blob
104 * storage GETs cost ~150–250 ms each regardless of size, and a 40 MiB
105 * pack is ten of them; fetched one at a time they were half of the
106 * dynamic transfer time. 3 ahead + 1 current stays within the cache.
107 */
108 export const PACK_LOOKAHEAD = 3;
109
110 const windows = new Map<string, { start: number; bytes: Uint8Array }>();
111 let windowBytes = 0;
112 const retainWindow = (cacheKey: string, slab: { start: number; bytes: Uint8Array }) => {
113 windows.set(cacheKey, slab);
114 windowBytes += slab.bytes.byteLength;
115 while (windowBytes > WINDOW_CACHE_BYTES && windows.size > 1) {
116 const oldest = windows.keys().next().value;
117 if (oldest === undefined) break;
118 windowBytes -= windows.get(oldest)!.bytes.byteLength;
119 windows.delete(oldest);
120 }
121 };
122
123 /**
124 * Raised (as a defect, via `Effect.die`) when a v1 code path reaches a
125 * feature the design explicitly defers — currently only reads of
126 * `location = 'pack'` objects, which exist starting with v1.1 compaction.
127 */
128 export class NotImplementedError extends Schema.TaggedError<NotImplementedError>()(
129 "NotImplementedError",
130 { feature: Schema.String },
131 ) {}
132
133 // ─────────────────────────────────────────────────────────────────────────────
134 // Contract
135 // ─────────────────────────────────────────────────────────────────────────────
136
137 /**
138 * A pack entry resolved by `PackParser`, ready to stage: the object id, its
139 * numeric type, uncompressed size, and the zlib-compressed bytes to store
140 * (the pack's compressed span verbatim for non-delta entries; a fresh
141 * `deflate(content)` for delta-resolved ones).
142 */
143 /**
144 * The live-object predicate (DESIGN §22.14). A push's rows are live the
145 * moment its `pushes` row is committed; the per-row `staged_push` column
146 * is nulled afterwards, off the response path (`flipPush`), and by GC as a
147 * safety net. Reads never wait for that rewrite.
148 */
149 export const LIVE_OBJECTS =
150 "(staged_push IS NULL OR staged_push IN (SELECT push_id FROM pushes WHERE state = 'committed'))";
151
152 export interface StagedObject {
153 readonly oid: Oid;
154 /**
155 * Set when the bytes already live in a pack in blob storage (a promoted
156 * wire pack, DESIGN §22.5): the row is staged as `location='pack'` with
157 * these coordinates and `zdata` is NOT stored.
158 */
159 readonly pack?: { readonly packId: string; readonly offset: number } | undefined;
160 /**
161 * Compressed span when `zdata` is not carried (a promoted row whose
162 * bytes live in the pack): the insert must record the real size.
163 */
164 readonly zsize?: number | undefined;
165 readonly type: ObjectType;
166 /** Uncompressed content size in bytes. */
167 readonly size: number;
168 /** `zlib(content)` without the loose header. */
169 readonly zdata: Uint8Array;
170 }
171
172 /** Outcome of {@link ObjectStore.insertStaged}. */
173 export type InsertOutcome = "inserted" | "exists";
174
175 /**
176 * The full storage surface of a repo's objects: the read-only
177 * `ObjectSource` the protocol layer consumes, plus batched metadata reads,
178 * staged inserts, and the connectivity-check membership oracle.
179 */
180 export interface ObjectStore extends ObjectSource {
181 /**
182 * Inflated content for many objects, one statement per ~256 oids — the
183 * closure walk reads whole tree levels this way instead of one SQL
184 * round trip per tree (DESIGN §22). Missing oids are simply absent.
185 */
186 readonly readContentBatch: (
187 oids: ReadonlyArray<Oid>,
188 ) => Effect.Effect<ReadonlyMap<Oid, Uint8Array>, StoreError>;
189 readonly packEntries: (
190 entries: ReadonlyArray<ManifestEntry>,
191 ) => Stream.Stream<Uint8Array, StoreError>;
192 /**
193 * Batched metadata for many oids (live objects only). Chunked `IN`
194 * queries, ≤ 100 params each. Missing oids are simply absent from the map.
195 */
196 readonly getMetaBatch: (
197 oids: ReadonlyArray<Oid>,
198 ) => Effect.Effect<ReadonlyMap<Oid, ObjectMeta>, StoreError>;
199 /**
200 * Stages one resolved object for `pushId` (DESIGN.md §3.6 step 4).
201 *
202 * Placement policy: compressed size > 1 MiB ⇒ the bytes are written to R2
203 * **first** (`{repoId}/objects/{oid}`, content-addressed so re-runs are
204 * idempotent), then the metadata row; otherwise the row carries the BLOB.
205 *
206 * Existing oids are skipped (idempotent re-push). A row left staged by a
207 * **different** (crashed) push is adopted — re-stamped with this `pushId`
208 * so the connectivity check and the final live-flip see it.
209 */
210 readonly insertStaged: (
211 pushId: string,
212 object: StagedObject,
213 ) => Effect.Effect<InsertOutcome, StoreError>;
214 /**
215 * Batched sibling of {@link ObjectStore.insertStaged} — the ingest hot
216 * path (DESIGN.md §16.6).
217 *
218 * Staging objects one at a time costs two statements each (an existence
219 * probe and an insert), so a 13.7k-object push ran ~27k statements and
220 * ~13.7k transactions inside a single-threaded Durable Object. This
221 * inserts a whole batch in **one** `transactionSync` with
222 * `INSERT OR IGNORE` (no probe), then adopts any rows left staged by a
223 * crashed push with one chunked `UPDATE`.
224 *
225 * Oversize objects (> 1 MiB compressed) still go through the per-object
226 * path, since each needs its own R2 write first.
227 */
228 readonly insertStagedBatch: (
229 pushId: string,
230 objects: ReadonlyArray<StagedObject>,
231 ) => Effect.Effect<void, StoreError>;
232 /** A live object's type and content for thin-delta resolution, or `undefined`. */
233 /** Read only live objects or objects staged by this push, with a caller-selected size cap. */
234 readonly readPrepared: (
235 pushId: string,
236 oid: Oid,
237 maxBytes: number,
238 ) => Effect.Effect<
239 { readonly type: ObjectType; readonly content: Uint8Array } | undefined,
240 StoreError
241 >;
242 readonly readBase: (
243 oid: Oid,
244 ) => Effect.Effect<
245 { readonly type: ObjectType; readonly content: Uint8Array } | undefined,
246 StoreError
247 >;
248 /**
249 * The connectivity-check membership oracle: returns the subset of `oids`
250 * that exist in **neither** the live store **nor** this push's staged
251 * rows. An empty result means every referenced object is present and the
252 * push is safe to commit.
253 */
254 readonly missingObjects: (
255 oids: ReadonlyArray<Oid>,
256 pushId: string,
257 ) => Effect.Effect<ReadonlyArray<Oid>, StoreError>;
258 }
259
260 /**
261 * Dependencies of {@link makeObjectStore}. Plain values (not Layers) so
262 * `RepoObject.ts` wires the store inside its inner per-instance effect.
263 */
264 export interface ObjectStoreOptions {
265 /** The repo DO's SQL client (see `makeSqlClient`). */
266 readonly sql: SqlClient;
267 /** The swappable bulk-byte store (R2 by default — see `BlobStore.ts`). */
268 readonly blobs: BlobStoreShape;
269 /** The repo's ULID — the R2 key prefix (DESIGN.md §3.2). */
270 readonly repoId: string;
271 }
272
273 // ─────────────────────────────────────────────────────────────────────────────
274 // Implementation
275 // ─────────────────────────────────────────────────────────────────────────────
276
277 const VALID_LOCATIONS: ReadonlySet<string> = new Set(["row", "r2", "pack"]);
278
279 const isObjectType = (type: number): type is ObjectType =>
280 type === 1 || type === 2 || type === 3 || type === 4;
281
282 const metaFromRow = (row: ObjectMetaRow): Effect.Effect<ObjectMeta, StoreError> => {
283 if (!isObjectType(row.type) || !VALID_LOCATIONS.has(row.location)) {
284 return Effect.fail(
285 StoreError.make({
286 reason: `corrupt objects row for ${row.oid}: type=${row.type} location=${row.location}`,
287 }),
288 );
289 }
290 return Effect.succeed({
291 oid: row.oid,
292 type: row.type,
293 size: row.size,
294 zsize: row.zsize,
295 location: row.location as ObjectLocation,
296 });
297 };
298
299 /**
300 * Returns a plain `ArrayBuffer` view of the bytes for SQLite BLOB binding
301 * (workerd binds `ArrayBuffer`, not typed arrays). Zero-copy when the view
302 * spans its whole buffer.
303 */
304 const toArrayBuffer = (bytes: Uint8Array): ArrayBuffer =>
305 bytes.byteOffset === 0 &&
306 bytes.byteLength === bytes.buffer.byteLength &&
307 bytes.buffer instanceof ArrayBuffer
308 ? bytes.buffer
309 : (bytes.buffer.slice(bytes.byteOffset, bytes.byteOffset + bytes.byteLength) as ArrayBuffer);
310
311 /** Row projection used by the byte-read dispatch. */
312 interface ZDataRow extends Record<string, string | number | ArrayBuffer | null> {
313 readonly oid: string;
314 readonly location: string;
315 readonly zdata: ArrayBuffer | null;
316 readonly r2_key: string | null;
317 /** sha1 of the containing pack when `location='pack'` (DESIGN §12.1). */
318 readonly pack_id: string | null;
319 /** Byte offset of this object's zdata span inside that pack. */
320 readonly pack_offset: number | null;
321 /** Length of the zdata span (mirrors `zsize`). */
322 readonly zsize: number;
323 }
324
325 /**
326 * Creates the {@link ObjectStore} for one repo. Plain factory — no Layers —
327 * so `RepoObject.ts` can wire it in its inner effect with the ambient
328 * `state`/bucket handles.
329 */
330 export const makeObjectStore = (options: ObjectStoreOptions): ObjectStore => {
331 const { sql, blobs, repoId } = options;
332
333 /** Discharges the blob store's `RuntimeContext` coloring and closes the
334 * error union into `StoreError` (we run inside the Repo DO). */
335 const runBlob = <A>(
336 effect: Effect.Effect<A, BlobStoreError, RuntimeContext>,
337 what: string,
338 ): Effect.Effect<A, StoreError> =>
339 effect.pipe(
340 Effect.mapError((error) => StoreError.make({ reason: `${what}: ${error.reason}` })),
341 Effect.provide(RuntimeContext.phantom),
342 );
343
344 const zlibToStore = (error: Zlib.ZlibError): StoreError =>
345 StoreError.make({ reason: error.reason });
346
347 const readRow = (
348 oid: Oid,
349 pushId: string | null = null,
350 ): Effect.Effect<ZDataRow | undefined, StoreError> =>
351 sql.first<ZDataRow>(
352 `SELECT oid, location, zdata, r2_key, pack_id, pack_offset, zsize
353 FROM objects WHERE oid = ? AND (${LIVE_OBJECTS} OR staged_push = ?)`,
354 oid,
355 pushId,
356 );
357
358 const requireRow = Effect.fn(function* (oid: Oid, pushId: string | null = null) {
359 const row = yield* readRow(oid, pushId);
360 if (row === undefined) {
361 return yield* StoreError.make({ reason: `object not found: ${oid}` });
362 }
363 return row;
364 });
365
366 /** Fetches the R2 body for an `'r2'`-located row. */
367 const r2Body = Effect.fn(function* (oid: Oid, r2Key: string | null) {
368 if (r2Key === null) {
369 return yield* StoreError.make({
370 reason: `object ${oid} has location='r2' but no r2_key`,
371 });
372 }
373 const body = yield* runBlob(blobs.get(r2Key), `blob get ${r2Key}`);
374 if (body === null) {
375 return yield* StoreError.make({ reason: `R2 object missing: ${r2Key}` });
376 }
377 return body;
378 });
379
380 /**
381 * Coalesced window cache over compacted packs (DESIGN §15 bottleneck 2).
382 *
383 * A pack read is one ranged R2 GET per object if done naively — and a
384 * fetch that walks a few thousand objects then costs a few thousand
385 * serialized round trips. Measured: **68 s to serve a 53 KiB fetch** over
386 * a 1202-object pack. Instead, reads are served out of window-aligned
387 * slabs: touching any object in a window fetches the whole window once,
388 * and every other object inside it is then free. A small pack collapses
389 * to a single GET.
390 *
391 * Bounded by construction: at most {@link MAX_CACHED_WINDOWS} windows of
392 * {@link WINDOW_BYTES} are retained (insertion-ordered LRU), so the cache
393 * cannot outgrow the isolate's memory budget.
394 */
395
396 const readWindow = Effect.fn(function* (key: string, windowIndex: number) {
397 const cacheKey = `${key}:${windowIndex}`;
398 const hit = windows.get(cacheKey);
399 if (hit !== undefined) {
400 // Refresh recency (Map iterates in insertion order).
401 windows.delete(cacheKey);
402 windows.set(cacheKey, hit);
403 return hit;
404 }
405 const start = windowIndex * WINDOW_BYTES;
406 const body = yield* runBlob(
407 blobs.get(key, { offset: start, length: WINDOW_BYTES }),
408 `blob window get ${key}`,
409 );
410 if (body === null) {
411 return yield* StoreError.make({ reason: `pack missing: ${key}` });
412 }
413 const bytes = yield* body.bytes.pipe(
414 Effect.mapError((error) => StoreError.make({ reason: `R2 read ${key}: ${error.message}` })),
415 );
416 const slab = { start, bytes };
417 retainWindow(cacheKey, slab);
418 return slab;
419 });
420
421 /**
422 * Reads an object's zdata span out of a compacted R2 pack (DESIGN §12.1),
423 * via the window cache above.
424 *
425 * `pack_offset` addresses the object's **zdata** directly (not the pack
426 * entry header), so the slice is exactly the stored compressed bytes —
427 * the same bytes a `location='row'` object holds in its BLOB. Entry
428 * headers are regenerated from `(type, size)` at emission time.
429 */
430 const packBytes = Effect.fn(function* (oid: Oid, row: ZDataRow) {
431 if (row.pack_id === null || row.pack_offset === null) {
432 return yield* StoreError.make({
433 reason: `object ${oid} has location='pack' but no pack coordinates`,
434 });
435 }
436 const key = packKeyOf(repoId, row.pack_id);
437 const offset = row.pack_offset;
438 const windowIndex = Math.floor(offset / WINDOW_BYTES);
439 const slab = yield* readWindow(key, windowIndex);
440 const relative = offset - slab.start;
441 if (relative >= 0 && relative + row.zsize <= slab.bytes.length) {
442 // Copy out: the slab is retained by the cache and must not be aliased.
443 return slab.bytes.slice(relative, relative + row.zsize);
444 }
445 // The object straddles the window boundary (or the window was short) —
446 // fall back to an exact ranged read for this one object.
447 const body = yield* runBlob(
448 blobs.get(key, { offset, length: row.zsize }),
449 `blob ranged get ${key}`,
450 );
451 if (body === null) {
452 return yield* StoreError.make({ reason: `pack missing: ${key}` });
453 }
454 return yield* body.bytes.pipe(
455 Effect.mapError((error) => StoreError.make({ reason: `R2 read ${key}: ${error.message}` })),
456 );
457 });
458
459 /** Reads the stored compressed bytes fully into memory. */
460 const readZBytes = Effect.fn(function* (oid: Oid, pushId: string | null = null) {
461 const row = yield* requireRow(oid, pushId);
462 switch (row.location) {
463 case "row": {
464 if (row.zdata === null) {
465 return yield* StoreError.make({
466 reason: `object ${oid} has location='row' but NULL zdata`,
467 });
468 }
469 return new Uint8Array(row.zdata);
470 }
471 case "r2": {
472 const body = yield* r2Body(oid, row.r2_key);
473 return yield* body.bytes.pipe(
474 Effect.mapError((error) =>
475 StoreError.make({
476 reason: `R2 read ${row.r2_key}: ${error.message}`,
477 }),
478 ),
479 );
480 }
481 case "pack":
482 return yield* packBytes(oid, row);
483 default:
484 return yield* StoreError.make({
485 reason: `object ${oid} has unknown location '${row.location}'`,
486 });
487 }
488 });
489
490 const getMetaBatch = (
491 oids: ReadonlyArray<Oid>,
492 ): Effect.Effect<ReadonlyMap<Oid, ObjectMeta>, StoreError> =>
493 Effect.gen(function* () {
494 const unique = Array.from(new Set(oids));
495 const map = new Map<Oid, ObjectMeta>();
496 if (unique.length === 0) {
497 return map;
498 }
499 const rows = yield* sql.inChunks<ObjectMetaRow>(
500 (ph) =>
501 `SELECT oid, type, size, zsize, location FROM objects WHERE oid IN (${ph}) AND ${LIVE_OBJECTS}`,
502 unique,
503 );
504 for (const row of rows) {
505 map.set(row.oid, yield* metaFromRow(row));
506 }
507 return map;
508 });
509
510 const insertStagedOne = Effect.fn(function* (pushId: string, object: StagedObject) {
511 const existing = yield* sql.first<{ staged_push: string | null }>(
512 `SELECT staged_push FROM objects WHERE oid = ?`,
513 object.oid,
514 );
515 if (existing !== undefined) {
516 if (existing.staged_push !== null && existing.staged_push !== pushId) {
517 // Adopt a row left staged by a crashed push so this push's
518 // connectivity check and live-flip see it.
519 yield* sql.run(
520 `UPDATE objects SET staged_push = ? WHERE oid = ? AND staged_push IS NOT NULL`,
521 pushId,
522 object.oid,
523 );
524 }
525 return "exists" as const;
526 }
527 if (object.zdata.byteLength > R2_OFFLOAD_THRESHOLD) {
528 // R2 write FIRST, then the metadata row — a crash between the two
529 // leaves only a harmless content-addressed R2 orphan.
530 const key = objectKey(repoId, object.oid);
531 yield* runBlob(blobs.put(key, object.zdata), `blob put ${key}`);
532 yield* sql.run(
533 `INSERT OR IGNORE INTO objects (oid, type, size, zsize, location, zdata, r2_key, staged_push)
534 VALUES (?, ?, ?, ?, 'r2', NULL, ?, ?)`,
535 object.oid,
536 object.type,
537 object.size,
538 object.zdata.byteLength,
539 key,
540 pushId,
541 );
542 } else {
543 yield* sql.run(
544 `INSERT OR IGNORE INTO objects (oid, type, size, zsize, location, zdata, staged_push)
545 VALUES (?, ?, ?, ?, 'row', ?, ?)`,
546 object.oid,
547 object.type,
548 object.size,
549 object.zdata.byteLength,
550 toArrayBuffer(object.zdata),
551 pushId,
552 );
553 }
554 return "inserted" as const;
555 });
556
557 const readZDataOf = (oid: Oid): Stream.Stream<Uint8Array, StoreError> =>
558 Stream.unwrap(
559 Effect.gen(function* () {
560 const row = yield* requireRow(oid);
561 switch (row.location) {
562 case "row": {
563 if (row.zdata === null) {
564 return yield* StoreError.make({
565 reason: `object ${oid} has location='row' but NULL zdata`,
566 });
567 }
568 return Stream.succeed(new Uint8Array(row.zdata));
569 }
570 case "r2": {
571 const body = yield* r2Body(oid, row.r2_key);
572 return body.stream.pipe(
573 Stream.mapError((error) =>
574 StoreError.make({
575 reason: `R2 stream ${row.r2_key}: ${error.message}`,
576 }),
577 ),
578 );
579 }
580 case "pack":
581 return Stream.succeed(yield* packBytes(oid, row));
582 default:
583 return yield* StoreError.make({
584 reason: `object ${oid} has unknown location '${row.location}'`,
585 });
586 }
587 }),
588 );
589
590 const self: ObjectStore = {
591 // ── ObjectSource (live objects only) ────────────────────────────────────
592 has: (oid) =>
593 sql
594 .first<{ one: number }>(
595 `SELECT 1 AS one FROM objects WHERE oid = ? AND ${LIVE_OBJECTS}`,
596 oid,
597 )
598 .pipe(Effect.map((row) => row !== undefined)),
599
600 filterExisting: (oids) =>
601 Effect.gen(function* () {
602 const unique = Array.from(new Set(oids));
603 if (unique.length === 0) {
604 return [];
605 }
606 const rows = yield* sql.inChunks<{ oid: string }>(
607 (ph) => `SELECT oid FROM objects WHERE oid IN (${ph}) AND ${LIVE_OBJECTS}`,
608 unique,
609 );
610 return rows.map((row) => row.oid);
611 }),
612
613 getMeta: (oid) =>
614 Effect.gen(function* () {
615 const row = yield* sql.first<ObjectMetaRow>(
616 `SELECT oid, type, size, zsize, location FROM objects WHERE oid = ? AND ${LIVE_OBJECTS}`,
617 oid,
618 );
619 if (row === undefined) {
620 return undefined;
621 }
622 return yield* metaFromRow(row);
623 }),
624
625 readZData: readZDataOf,
626
627 readContent: (oid) =>
628 readZBytes(oid).pipe(
629 Effect.flatMap((zdata) => Zlib.inflate(zdata).pipe(Effect.mapError(zlibToStore))),
630 ),
631
632 readContentBatch: (oids) =>
633 Effect.gen(function* () {
634 const out = new Map<Oid, Uint8Array>();
635 const unique = Array.from(new Set(oids));
636 if (unique.length === 0) return out;
637 const rows = yield* sql.inChunks<ZDataRow>(
638 (ph) =>
639 `SELECT oid, location, zdata, r2_key, pack_id, pack_offset, zsize
640 FROM objects WHERE oid IN (${ph}) AND ${LIVE_OBJECTS}`,
641 unique,
642 );
643 const inflate = (z: Uint8Array) => Zlib.inflate(z).pipe(Effect.mapError(zlibToStore));
644 // Pack-resident rows in pack order so consecutive window reads hit.
645 const packRows = rows
646 .filter((row) => row.location === "pack")
647 .sort(
648 (a, b) =>
649 (a.pack_id ?? "").localeCompare(b.pack_id ?? "") ||
650 (a.pack_offset ?? 0) - (b.pack_offset ?? 0),
651 );
652 for (const row of rows) {
653 if (row.location === "row" && row.zdata !== null) {
654 out.set(row.oid, yield* inflate(new Uint8Array(row.zdata)));
655 } else if (row.location === "r2") {
656 out.set(row.oid, yield* readZBytes(row.oid).pipe(Effect.flatMap(inflate)));
657 }
658 }
659 for (const row of packRows) {
660 out.set(row.oid, yield* inflate(yield* packBytes(row.oid, row)));
661 }
662 return out;
663 }),
664
665 packEntries: (entries) =>
666 Stream.suspend(() => {
667 // Locality order (DESIGN §22): a non-delta pack is order-free, so
668 // emit SQLite rows in manifest order, then every pack-resident
669 // object sorted by (pack, offset) so the window cache streams each
670 // 4 MiB window exactly once, then the rare R2-offloaded objects.
671 const byOid = new Map(entries.map((entry) => [entry.oid, entry]));
672 const rowEntries: Array<ManifestEntry> = [];
673 const packList: Array<ManifestEntry> = [];
674 const r2Entries: Array<ManifestEntry> = [];
675 for (const entry of entries) {
676 (entry.location === "row"
677 ? rowEntries
678 : entry.location === "pack"
679 ? packList
680 : r2Entries
681 ).push(entry);
682 }
683 const header = (entry: ManifestEntry) =>
684 encodeTypeSize(entry.type as PackEntryType, entry.size);
685
686 const rowStream = Stream.fromIterable(chunkArray(rowEntries, EMIT_BATCH)).pipe(
687 Stream.mapEffect((run) =>
688 Effect.gen(function* () {
689 const rows = yield* sql.inChunks<{
690 oid: string;
691 zdata: ArrayBuffer | null;
692 }>(
693 (ph) => `SELECT oid, zdata FROM objects WHERE oid IN (${ph}) AND ${LIVE_OBJECTS}`,
694 run.map((entry) => entry.oid),
695 );
696 const zdataOf = new Map(rows.map((row) => [row.oid, row.zdata]));
697 const pieces: Array<Uint8Array> = [];
698 for (const entry of run) {
699 const zdata = zdataOf.get(entry.oid);
700 if (zdata === undefined || zdata === null) {
701 // The manifest said 'row', but compaction may have moved
702 // the bytes to a pack between closure and emission (the
703 // alarm runs concurrently with fetches). Re-dispatch on
704 // the CURRENT location; only a truly absent object fails.
705 pieces.push(header(entry), yield* readZBytes(entry.oid));
706 continue;
707 }
708 pieces.push(header(entry), new Uint8Array(zdata));
709 }
710 return concatBytes(pieces);
711 }),
712 ),
713 );
714
715 // Pack runs (DESIGN §22.4): coords sorted by (pack, offset) and
716 // grouped by 4 MiB window. Each group is ONE window fetch (with the
717 // next window fetched concurrently — a cache hit on the following
718 // step) and one synchronous slice loop: no per-object Effect call,
719 // no per-object cache lookup. Objects that straddle a window edge
720 // (rare) fall back to the ranged read in `packBytes`; pack order
721 // is free, so they simply follow their window's run.
722 const packStream = Stream.unwrap(
723 Effect.gen(function* () {
724 if (packList.length === 0) return Stream.empty;
725 const coords = yield* sql.inChunks<{
726 oid: string;
727 pack_id: string | null;
728 pack_offset: number | null;
729 zsize: number;
730 }>(
731 (ph) =>
732 `SELECT oid, pack_id, pack_offset, zsize FROM objects WHERE oid IN (${ph}) AND ${LIVE_OBJECTS}`,
733 packList.map((entry) => entry.oid),
734 );
735 coords.sort(
736 (a, b) =>
737 (a.pack_id ?? "").localeCompare(b.pack_id ?? "") ||
738 (a.pack_offset ?? 0) - (b.pack_offset ?? 0),
739 );
740 type Coord = (typeof coords)[number];
741 type Run = {
742 readonly key: string;
743 readonly window: number;
744 readonly inside: Array<Coord>;
745 readonly straddling: Array<Coord>;
746 };
747 const runs: Array<Run> = [];
748 for (const coord of coords) {
749 if (coord.pack_id === null || coord.pack_offset === null) {
750 return yield* StoreError.make({
751 reason: `object ${coord.oid} is location='pack' without coordinates`,
752 });
753 }
754 const key = packKeyOf(repoId, coord.pack_id);
755 const window = Math.floor(coord.pack_offset / WINDOW_BYTES);
756 let run = runs[runs.length - 1];
757 if (run === undefined || run.key !== key || run.window !== window) {
758 run = { key, window, inside: [], straddling: [] };
759 runs.push(run);
760 }
761 (coord.pack_offset + coord.zsize <= (window + 1) * WINDOW_BYTES
762 ? run.inside
763 : run.straddling
764 ).push(coord);
765 }
766 const rowOf = (coord: Coord): ZDataRow => ({
767 oid: coord.oid,
768 location: "pack",
769 zdata: null,
770 r2_key: null,
771 pack_id: coord.pack_id,
772 pack_offset: coord.pack_offset,
773 zsize: coord.zsize,
774 });
775 // Lookahead: the next PACK_LOOKAHEAD windows are fetched
776 // concurrently with this one; each is a cache hit when its
777 // turn comes. Bounded by WINDOW_CACHE_BYTES (8 windows).
778 return Stream.fromIterable(
779 runs.map((run, i) => [run, runs.slice(i + 1, i + 1 + PACK_LOOKAHEAD)] as const),
780 ).pipe(
781 Stream.mapEffect(([run, ahead]) =>
782 Effect.gen(function* () {
783 const [slab] = yield* Effect.all(
784 [
785 readWindow(run.key, run.window),
786 ...ahead.map((next) => readWindow(next.key, next.window).pipe(Effect.ignore)),
787 ],
788 { concurrency: 1 + PACK_LOOKAHEAD },
789 );
790 const pieces = yield* Effect.sync(() => {
791 const out: Array<Uint8Array> = [];
792 for (const coord of run.inside) {
793 const entry = byOid.get(coord.oid);
794 if (entry === undefined) continue;
795 const at = coord.pack_offset! - slab.start;
796 out.push(header(entry), slab.bytes.subarray(at, at + coord.zsize));
797 }
798 return out;
799 });
800 for (const coord of run.straddling) {
801 const entry = byOid.get(coord.oid);
802 if (entry === undefined) continue;
803 pieces.push(header(entry), yield* packBytes(coord.oid, rowOf(coord)));
804 }
805 return concatBytes(pieces);
806 }),
807 ),
808 );
809 }),
810 );
811
812 const r2Stream = Stream.fromIterable(r2Entries).pipe(
813 Stream.flatMap((entry) =>
814 Stream.succeed(header(entry)).pipe(Stream.concat(readZDataOf(entry.oid))),
815 ),
816 );
817
818 return rowStream.pipe(Stream.concat(packStream), Stream.concat(r2Stream));
819 }),
820
821 // ── Store-side surface ──────────────────────────────────────────────────
822 getMetaBatch,
823
824 insertStagedBatch: Effect.fn(function* (pushId: string, objects: ReadonlyArray<StagedObject>) {
825 if (objects.length === 0) return;
826 const inline: Array<StagedObject> = [];
827 const promoted: Array<StagedObject> = [];
828 for (const object of objects) {
829 if (object.pack !== undefined) {
830 promoted.push(object);
831 } else if (object.zdata.byteLength > R2_OFFLOAD_THRESHOLD) {
832 yield* insertStagedOne(pushId, object);
833 } else {
834 inline.push(object);
835 }
836 }
837 if (promoted.length > 0) {
838 // Rows that point into a pack already in blob storage: no BLOB
839 // written to SQLite at all. 7 bindings per row → 14 rows per
840 // statement under the 100-parameter cap.
841 const sorted = [...promoted].sort((a, b) => (a.oid < b.oid ? -1 : a.oid > b.oid ? 1 : 0));
842 const written = yield* sql.transactionSync((raw) => {
843 let changed = 0;
844 for (let at = 0; at < sorted.length; at += 14) {
845 const part = sorted.slice(at, at + 14);
846 const values = part.map(() => "(?, ?, ?, ?, 'pack', NULL, ?, ?, ?)").join(", ");
847 const bindings: Array<string | number> = [];
848 for (const object of part) {
849 bindings.push(
850 object.oid,
851 object.type,
852 object.size,
853 object.zsize ?? object.zdata.byteLength,
854 object.pack!.packId,
855 object.pack!.offset,
856 pushId,
857 );
858 }
859 changed += raw.exec(
860 `INSERT OR IGNORE INTO objects (oid, type, size, zsize, location, zdata, pack_id, pack_offset, staged_push) VALUES ${values}`,
861 ...bindings,
862 ).rowsWritten;
863 }
864 return changed;
865 });
866 if (written < sorted.length) {
867 yield* sql.inChunks(
868 (ph) =>
869 `UPDATE objects SET staged_push = ? WHERE oid IN (${ph})
870 AND staged_push IS NOT NULL AND staged_push != ?
871 AND staged_push NOT IN (SELECT push_id FROM pushes WHERE state = 'committed')`,
872 sorted.map((object) => object.oid),
873 { prefix: [pushId], suffix: [pushId] },
874 );
875 }
876 }
877 if (inline.length === 0) return;
878
879 // One transaction for the whole batch: `INSERT OR IGNORE` needs no
880 // existence probe, and a single commit replaces one per object.
881 // Sorted by oid: the table is WITHOUT ROWID keyed by oid, so inserting
882 // in key order turns a batch into sequential B-tree appends instead of
883 // random page splits.
884 const rows = inline
885 .map((object) => ({
886 oid: object.oid,
887 type: object.type,
888 size: object.size,
889 zsize: object.zdata.byteLength,
890 zdata: toArrayBuffer(object.zdata),
891 }))
892 .sort((a, b) => (a.oid < b.oid ? -1 : a.oid > b.oid ? 1 : 0));
893 // Multi-row INSERTs (DESIGN §22.4): one statement per STAGE_INSERT_ROWS
894 // rows instead of one per row. On production CPU each statement
895 // execution costs far more than binding six more parameters, and
896 // staging was the largest slice of a push's ingest (7 s of 30 s).
897 const written = yield* sql.transactionSync((raw) => {
898 let changed = 0;
899 for (let at = 0; at < rows.length; at += STAGE_INSERT_ROWS) {
900 const part = rows.slice(at, at + STAGE_INSERT_ROWS);
901 const values = part.map(() => "(?, ?, ?, ?, 'row', ?, ?)").join(", ");
902 const bindings: Array<string | number | ArrayBuffer> = [];
903 for (const row of part) {
904 bindings.push(row.oid, row.type, row.size, row.zsize, row.zdata, pushId);
905 }
906 const cursor = raw.exec(
907 `INSERT OR IGNORE INTO objects (oid, type, size, zsize, location, zdata, staged_push) VALUES ${values}`,
908 ...bindings,
909 );
910 changed += cursor.rowsWritten;
911 }
912 return changed;
913 });
914 // Every row was new: nothing to adopt. Only when INSERT OR IGNORE
915 // skipped rows can some belong to a crashed push and need re-stamping.
916 if (written >= rows.length) return;
917 yield* sql.inChunks(
918 (ph) =>
919 `UPDATE objects SET staged_push = ? WHERE oid IN (${ph})
920 AND staged_push IS NOT NULL AND staged_push != ?
921 AND staged_push NOT IN (SELECT push_id FROM pushes WHERE state = 'committed')`,
922 inline.map((object) => object.oid),
923 { prefix: [pushId], suffix: [pushId] },
924 );
925 }),
926 readPrepared: (pushId, oid, maxBytes) =>
927 Effect.gen(function* () {
928 if (!Number.isSafeInteger(maxBytes) || maxBytes < 0)
929 return yield* StoreError.make({ reason: "invalid object size limit" });
930 const row = yield* sql.first<ObjectMetaRow>(
931 `SELECT oid, type, size, zsize, location FROM objects WHERE oid = ? AND (${LIVE_OBJECTS} OR staged_push = ?)`,
932 oid,
933 pushId,
934 );
935 if (row === undefined) return undefined;
936 const meta = yield* metaFromRow(row);
937 if (meta.size > maxBytes)
938 return yield* StoreError.make({
939 reason: `object exceeds ${maxBytes} byte read limit`,
940 });
941 const zdata = yield* readZBytes(oid, pushId);
942 const content = yield* Zlib.inflate(zdata).pipe(Effect.mapError(zlibToStore));
943 return { type: meta.type, content };
944 }),
945
946 readBase: (oid) =>
947 self
948 .getMeta(oid)
949 .pipe(
950 Effect.flatMap((meta) =>
951 meta === undefined
952 ? Effect.undefined
953 : self.readContent(oid).pipe(Effect.map((content) => ({ type: meta.type, content }))),
954 ),
955 ),
956 insertStaged: insertStagedOne,
957
958 missingObjects: (oids, pushId) =>
959 Effect.gen(function* () {
960 const unique = Array.from(new Set(oids));
961 if (unique.length === 0) {
962 return [];
963 }
964 const rows = yield* sql.inChunks<{ oid: string }>(
965 (ph) =>
966 `SELECT oid FROM objects WHERE oid IN (${ph}) AND (${LIVE_OBJECTS} OR staged_push = ?)`,
967 unique,
968 { suffix: [pushId] },
969 );
970 const present = new Set(rows.map((row) => row.oid));
971 return unique.filter((oid) => !present.has(oid));
972 }),
973 };
974 return self;
975 };