@devries-cameron20 / t3code Mirror · read-only
Fork 0

The best way to control Claude Code, Codex and more from any computer or phone

Mirrored from https://github.com/Camo67/t3code.git

Synced Nov 21, 04:45 UTC

main
ts 861 lines 33.9 KB
Raw
1 /**
2 * Pack ingestion (DESIGN §3.6): a single pass over a pack held behind a
3 * {@link RandomAccess} source, under the push semaphore.
4 *
5 * 1. Validate `PACK`, version 2, count.
6 * 2. Per entry: parse the type/size varint header. Non-delta entries are
7 * streaming-inflated (`Zlib.inflateEntry` — `bytesWritten` gives the exact
8 * compressed span), hashed as `sha1("<type> <size>\0" + content)`, and the
9 * **compressed span is kept verbatim as `zdata`** — no recompression.
10 * Deltas (OFS with the +1-bias offset decoding, REF with a 20-byte base
11 * id) resolve their base from (a) already-ingested entries via a bounded
12 * LRU of resolved contents, (b) re-inflation from the pack on cache miss,
13 * or (c) the object store for thin bases (REF_DELTA to a prior push — the
14 * normal case). The copy/insert instruction stream is applied
15 * (`size==0 ⇒ 0x10000`), the result size verified, the result hashed and
16 * `deflate`d (level 6) for storage.
17 * 3. The trailing pack SHA-1 is verified.
18 *
19 * Resolved entries are emitted to a caller-provided sink as they are
20 * produced; the caller (the Repo DO) stages them with `staged_push = pushId`.
21 *
22 * The parser is written against {@link RandomAccess} from day one so the
23 * v1.x streaming upgrade (R2-multipart tee + ranged reads) swaps one
24 * implementation, not the protocol code (DESIGN §3.6 upgrade seam).
25 */
26 import * as Effect from "effect/Effect";
27 import * as Result from "effect/Result";
28 import * as Schema from "effect/Schema";
29 import { applyDelta, DeltaFormatError } from "./Delta.ts";
30 import {
31 bytesToHex,
32 decodeOfsDeltaOffset,
33 decodeTypeSize,
34 hashObject,
35 hashObjectSync,
36 isDeltaType,
37 makeSha1,
38 ObjectParseError,
39 type ObjectType,
40 type Oid,
41 type PackEntryType,
42 } from "./ObjectCodec.ts";
43 import { type StoreError } from "./Store.ts";
44 import { deflate, inflate, inflateEntry, inflateEntrySync, ZlibError } from "./Zlib.ts";
45
46 // ─────────────────────────────────────────────────────────────────────────────
47 // RandomAccess
48 // ─────────────────────────────────────────────────────────────────────────────
49
50 /**
51 * Random access over a pack's raw bytes. v1 backs this with the buffered
52 * request body ({@link bufferRandomAccess}); the v1.x streaming upgrade backs
53 * it with R2 ranged reads without touching the parser.
54 */
55 export interface RandomAccess {
56 /**
57 * Synchronous read when the range needs no I/O — a view into an
58 * in-memory buffer or an already-cached window — else `undefined`. The
59 * parser's inner loop uses it to stay synchronous between I/O points
60 * (DESIGN §22.5).
61 */
62 readonly readSync?: ((offset: number, length: number) => Uint8Array | undefined) | undefined;
63 /**
64 * For sources whose total size is not known up front (`size` is
65 * `Infinity` while a body is still arriving — DESIGN §22.6): resolves
66 * with the final size once the body has ended.
67 */
68 readonly awaitEnd?: Effect.Effect<number, StoreError> | undefined;
69 /** The parser is done with every byte below `offset` (retention hint). */
70 readonly release?: ((offset: number) => void) | undefined;
71 /**
72 * True when `offset` has been dropped from memory and the body has NOT
73 * ended yet, i.e. reading it would block on the spill completing. The
74 * parser defers such delta entries until the end (DESIGN §22.6).
75 */
76 readonly evictedBeforeEnd?: ((offset: number) => boolean) | undefined;
77 /** Total size of the pack in bytes (`Infinity` while streaming). */
78 readonly size: number;
79 /**
80 * Reads up to `length` bytes at `offset`. May return fewer bytes only when
81 * the range extends past the end of the pack. Implementations should return
82 * views cheaply (no copy) where possible; callers copy what they retain.
83 */
84 readonly read: (offset: number, length: number) => Effect.Effect<Uint8Array, StoreError>;
85 }
86
87 /**
88 * A {@link RandomAccess} over an in-memory buffer. Reads return subarray
89 * views (zero copy).
90 */
91 export const bufferRandomAccess = (buf: Uint8Array): RandomAccess => ({
92 size: buf.length,
93 read: (offset, length) =>
94 Effect.sync(() => buf.subarray(offset, Math.min(offset + length, buf.length))),
95 readSync: (offset, length) => buf.subarray(offset, Math.min(offset + length, buf.length)),
96 awaitEnd: Effect.succeed(buf.length),
97 });
98
99 // ─────────────────────────────────────────────────────────────────────────────
100 // Errors
101 // ─────────────────────────────────────────────────────────────────────────────
102
103 /**
104 * Error raised on malformed pack structure: bad magic/version, truncated
105 * entries, entry-count mismatch, size mismatches, or an OFS_DELTA offset that
106 * does not land on an entry boundary.
107 */
108 export class PackFormatError extends Schema.TaggedError<PackFormatError>()("PackFormatError", {
109 reason: Schema.String,
110 }) {}
111
112 /**
113 * Error raised when the trailing pack SHA-1 does not match the hash of the
114 * preceding bytes (`unpack pack checksum mismatch` on the wire).
115 */
116 export class PackChecksumMismatch extends Schema.TaggedError<PackChecksumMismatch>()(
117 "PackChecksumMismatch",
118 { expected: Schema.String, actual: Schema.String },
119 ) {}
120
121 /**
122 * Error raised when an object (or a delta result) exceeds the per-object
123 * uncompressed size cap (64 MiB in v1 — DESIGN §3.3).
124 */
125 export class ObjectTooLargeError extends Schema.TaggedError<ObjectTooLargeError>()(
126 "ObjectTooLargeError",
127 {
128 size: Schema.Number,
129 limit: Schema.Number,
130 oid: Schema.optional(Schema.String),
131 },
132 ) {}
133
134 /**
135 * Error raised when a REF_DELTA's base cannot be found in the pack or in the
136 * object store (a thin pack whose base we do not have).
137 */
138 /**
139 * A delta's base was evicted from a streaming source before the body ended
140 * (DESIGN §22.6). Retried after the end, when the spilled object is
141 * readable; never reaches callers.
142 */
143 export class BaseEvictedError extends Schema.TaggedError<BaseEvictedError>()("BaseEvictedError", {
144 offset: Schema.Number,
145 }) {}
146
147 export class MissingDeltaBaseError extends Schema.TaggedError<MissingDeltaBaseError>()(
148 "MissingDeltaBaseError",
149 { baseOid: Schema.String },
150 ) {}
151
152 /**
153 * The full error union pack ingestion can produce.
154 */
155 export type PackIngestError =
156 | PackFormatError
157 | PackChecksumMismatch
158 | ObjectTooLargeError
159 | MissingDeltaBaseError
160 | ZlibError
161 | DeltaFormatError
162 | ObjectParseError
163 | StoreError
164 | BaseEvictedError;
165
166 // ─────────────────────────────────────────────────────────────────────────────
167 // Resolved entries / options
168 // ─────────────────────────────────────────────────────────────────────────────
169
170 /**
171 * One fully resolved object produced by pack ingestion.
172 *
173 * `zdata` is `zlib(content)` **without** the loose header — for non-delta
174 * entries it is the pack's compressed span verbatim; for delta-resolved
175 * entries it is a fresh `deflate(content)` at level 6.
176 */
177 export interface ResolvedEntry {
178 readonly oid: Oid;
179 readonly type: ObjectType;
180 /** Uncompressed size in bytes. */
181 readonly size: number;
182 /** `zlib(content)`, no loose header. Safe to retain (always a copy). */
183 readonly zdata: Uint8Array;
184 /** `true` when the entry arrived as a delta and was re-deflated. */
185 readonly fromDelta: boolean;
186 /**
187 * Byte offset of `zdata` within the pack SOURCE for non-delta entries
188 * (the compressed span is the pack's bytes verbatim, so a store may
189 * reference it in place — DESIGN §22.5); `-1` for delta-resolved entries,
190 * whose `zdata` is a fresh deflate that exists nowhere in the pack.
191 */
192 readonly dataOffset: number;
193 /**
194 * The inflated bytes. The parser already holds them at this point, so
195 * passing them through spares every consumer a second inflate just to
196 * read a commit or tree.
197 */
198 readonly content: Uint8Array;
199 }
200
201 /**
202 * Summary returned by {@link ingestPack} after the trailer verifies.
203 */
204 export interface IngestSummary {
205 /** Number of entries in the pack (the header count). */
206 readonly count: number;
207 /** The oids of every resolved entry, in emission order. */
208 readonly oids: ReadonlyArray<Oid>;
209 }
210
211 /** Default per-object uncompressed size cap: 64 MiB (DESIGN §3.3). */
212 export const DEFAULT_MAX_OBJECT_SIZE = 64 * 1024 * 1024;
213
214 /** Default resolved-content LRU budget: 20 MiB (DESIGN §3.6). */
215 export const DEFAULT_CACHE_BYTES = 20 * 1024 * 1024;
216
217 /** Per-entry cache admission cap: entries larger than this are never cached. */
218 const MAX_CACHE_ENTRY_BYTES = 10 * 1024 * 1024;
219
220 /**
221 * Bytes read per entry before growing (see the read in the main pass).
222 * Sized so virtually every git object — a few KiB — is covered by one read,
223 * which is what keeps ingest from a remote source proportional to the pack
224 * size rather than to entries × pack size.
225 */
226 /**
227 * Bytes probed per entry before inflating. In-window reads are views on
228 * both sources, so the probe's size costs nothing — except when it crosses
229 * a window edge on the spilled (blob-backed) source, where the reader must
230 * assemble a copy. 64 KiB keeps such crossings rare (only entries within
231 * 64 KiB of a 4 MiB edge) and small; the rare larger entry grows the
232 * window on demand (DESIGN §22.4 — 512 KiB here was ~1 GB of copies per
233 * 15k-object spilled push).
234 */
235 const ENTRY_WINDOW_BYTES = 64 * 1024;
236
237 /** Non-delta entries handed to `sinkBatch` per fiber hop (DESIGN §22.5). */
238 export const SINK_BATCH = 256;
239
240 /**
241 * Options for {@link ingestPack}.
242 */
243 /** What the parser needs from the live store: thin-delta bases. */
244 export interface ThinBaseSource {
245 /** A live object's type and content, or `undefined` when absent. */
246 readonly readBase: (
247 oid: Oid,
248 ) => Effect.Effect<
249 { readonly type: ObjectType; readonly content: Uint8Array } | undefined,
250 StoreError
251 >;
252 }
253
254 export interface IngestPackOptions<E, R> {
255 /** The pack bytes. */
256 readonly source: RandomAccess;
257 /** Live objects, used to resolve thin REF_DELTA bases. */
258 readonly store: ThinBaseSource;
259 /** Receives each resolved entry as it is produced (staging inserts). */
260 readonly sink: (entry: ResolvedEntry) => Effect.Effect<void, E, R>;
261 /**
262 * Optional batched sink: receives runs of entries the synchronous inner
263 * loop produced without any I/O between them (DESIGN §22.5). When set,
264 * non-delta entries go here in batches of up to `SINK_BATCH` and `sink`
265 * receives only delta-resolved entries.
266 */
267 readonly sinkBatch?:
268 | ((entries: ReadonlyArray<ResolvedEntry>) => Effect.Effect<void, E, R>)
269 | undefined;
270 /** Per-object uncompressed cap; default {@link DEFAULT_MAX_OBJECT_SIZE}. */
271 readonly maxObjectSize?: number | undefined;
272 /**
273 * Optional per-phase CPU accounting (ms), accumulated in place:
274 * `inflate`, `hash`, `delta`, `deflate`, `copy`, `sink`. Cheap enough
275 * to leave on: two `performance.now()` calls per phase per object.
276 */
277 readonly phases?: Record<string, number> | undefined;
278 /** Resolved-content LRU budget; default {@link DEFAULT_CACHE_BYTES}. */
279 readonly cacheBytes?: number | undefined;
280 }
281
282 // ─────────────────────────────────────────────────────────────────────────────
283 // Internals
284 // ─────────────────────────────────────────────────────────────────────────────
285
286 interface CachedContent {
287 readonly type: ObjectType;
288 readonly content: Uint8Array;
289 }
290
291 /** Insertion-order LRU bounded by total content bytes. */
292 class ByteLru {
293 private readonly map = new Map<string, CachedContent>();
294 private bytes = 0;
295 constructor(private readonly capacity: number) {}
296
297 get(key: string): CachedContent | undefined {
298 const hit = this.map.get(key);
299 if (hit !== undefined) {
300 // refresh recency
301 this.map.delete(key);
302 this.map.set(key, hit);
303 }
304 return hit;
305 }
306
307 set(key: string, entry: CachedContent): void {
308 if (entry.content.length > MAX_CACHE_ENTRY_BYTES) return;
309 if (this.map.has(key)) return;
310 this.map.set(key, entry);
311 this.bytes += entry.content.length;
312 while (this.bytes > this.capacity && this.map.size > 1) {
313 const oldest = this.map.keys().next().value as string;
314 const evicted = this.map.get(oldest)!;
315 this.map.delete(oldest);
316 this.bytes -= evicted.content.length;
317 }
318 }
319 }
320
321 interface IndexedEntry {
322 /** Absolute offset of the entry header in the pack. */
323 readonly offset: number;
324 readonly entryType: PackEntryType;
325 /** Declared uncompressed size (for deltas: of the delta payload). */
326 readonly declaredSize: number;
327 /** Absolute offset of the zlib stream. */
328 readonly dataOffset: number;
329 /** Exact compressed span in bytes. */
330 readonly span: number;
331 /** OFS_DELTA: absolute offset of the base entry. */
332 readonly baseOffset: number | undefined;
333 /** REF_DELTA: base object id. */
334 readonly baseOid: Oid | undefined;
335 /** Set once the entry has been resolved and emitted. */
336 resolved?: { readonly oid: Oid; readonly type: ObjectType };
337 }
338
339 const readU32BE = (buf: Uint8Array, offset: number): number =>
340 ((buf[offset]! << 24) | (buf[offset + 1]! << 16) | (buf[offset + 2]! << 8) | buf[offset + 3]!) >>>
341 0;
342
343 /**
344 * Reads and validates the 12-byte pack header (`PACK`, version 2, entry
345 * count).
346 */
347 export const readPackHeader = Effect.fn(function* (source: RandomAccess) {
348 if (Number.isFinite(source.size) && source.size < 12 + 20) {
349 return yield* PackFormatError.make({
350 reason: `pack too small: ${source.size} bytes`,
351 });
352 }
353 const header = yield* source.read(0, 12);
354 if (header.length < 12) {
355 return yield* PackFormatError.make({
356 reason: `pack too small: ${header.length} bytes`,
357 });
358 }
359 if (
360 header.length < 12 ||
361 header[0] !== 0x50 || // P
362 header[1] !== 0x41 || // A
363 header[2] !== 0x43 || // C
364 header[3] !== 0x4b // K
365 ) {
366 return yield* PackFormatError.make({ reason: "bad pack magic" });
367 }
368 const version = readU32BE(header, 4);
369 if (version !== 2) {
370 return yield* PackFormatError.make({
371 reason: `unsupported pack version ${version}`,
372 });
373 }
374 const count = readU32BE(header, 8);
375 return { version, count };
376 });
377
378 // ─────────────────────────────────────────────────────────────────────────────
379 // Ingest
380 // ─────────────────────────────────────────────────────────────────────────────
381
382 /**
383 * Ingests a complete pack: parses every entry, resolves OFS/REF deltas
384 * (including thin bases from the object store), verifies the trailing SHA-1,
385 * and emits each resolved object to `options.sink`.
386 *
387 * Entries are emitted in pack order, except REF_DELTA entries whose base
388 * appears *later* in the pack (legal but rare) — those are deferred and
389 * emitted after the main pass.
390 *
391 * Memory: the resolved-content LRU is bounded by `cacheBytes`; cache misses
392 * re-inflate from the source. Only one resolved object (plus its base) is
393 * otherwise live at a time.
394 */
395 export const ingestPack = <E, R>(
396 options: IngestPackOptions<E, R>,
397 ): Effect.Effect<IngestSummary, PackIngestError | E, R> =>
398 Effect.gen(function* () {
399 const { sink, source, store } = options;
400 const phases = options.phases;
401 const timed = <A, E2, R2>(
402 phase: string,
403 effect: Effect.Effect<A, E2, R2>,
404 ): Effect.Effect<A, E2, R2> =>
405 phases === undefined
406 ? effect
407 : Effect.suspend(() => {
408 const at = performance.now();
409 return Effect.ensuring(
410 effect,
411 Effect.sync(() => {
412 phases[phase] = (phases[phase] ?? 0) + (performance.now() - at);
413 }),
414 );
415 });
416 const maxObjectSize = options.maxObjectSize ?? DEFAULT_MAX_OBJECT_SIZE;
417 const cache = new ByteLru(options.cacheBytes ?? DEFAULT_CACHE_BYTES);
418 const { count } = yield* readPackHeader(source);
419 // Unknown while a streaming body is still arriving (DESIGN §22.6); the
420 // loop is driven by `count`, and the true end is learned from
421 // `awaitEnd` before the trailer is checked.
422 let dataEnd = Number.isFinite(source.size) ? source.size - 20 : Number.POSITIVE_INFINITY;
423 // The trailer SHA-1 is accumulated as entries are consumed instead of
424 // re-reading the whole pack afterwards (on a spilled source that was
425 // another full pass of window reads).
426 const trailerSha = makeSha1();
427 trailerSha.update(yield* source.read(0, 12));
428
429 const byOffset = new Map<number, IndexedEntry>();
430 /** oid → entry offset, for in-pack REF_DELTA base lookup. */
431 const offsetByOid = new Map<Oid, number>();
432 const oids: Array<Oid> = [];
433 const deferred: Array<IndexedEntry> = [];
434
435 /** Inflates an already-indexed entry's own zlib stream (exact span). */
436 const inflateOwn = (entry: IndexedEntry): Effect.Effect<Uint8Array, PackIngestError> =>
437 Effect.suspend((): Effect.Effect<Uint8Array, PackIngestError> =>
438 source.evictedBeforeEnd?.(entry.dataOffset) === true
439 ? Effect.fail(BaseEvictedError.make({ offset: entry.dataOffset }))
440 : source.read(entry.dataOffset, entry.span).pipe(Effect.flatMap((z) => inflate(z))),
441 );
442
443 /**
444 * Resolves an entry to its full content, walking delta chains. Bases come
445 * from the LRU, from re-inflation of earlier pack entries, or (thin
446 * REF_DELTA) from the object store.
447 */
448 const resolveContent = (entry: IndexedEntry): Effect.Effect<CachedContent, PackIngestError> =>
449 Effect.gen(function* () {
450 const key = `ofs:${entry.offset}`;
451 const hit = cache.get(key);
452 if (hit !== undefined) return hit;
453 let result: CachedContent;
454 if (!isDeltaType(entry.entryType)) {
455 const content = yield* inflateOwn(entry);
456 result = { type: entry.entryType as ObjectType, content };
457 } else {
458 const base = yield* resolveBase(entry);
459 const payload = yield* timed("inflate", inflateOwn(entry));
460 const content = yield* timed("delta", applyDelta(base.content, payload));
461 if (content.length > maxObjectSize) {
462 return yield* ObjectTooLargeError.make({
463 size: content.length,
464 limit: maxObjectSize,
465 });
466 }
467 result = { type: base.type, content };
468 }
469 cache.set(key, result);
470 return result;
471 });
472
473 /** Resolves a delta entry's base content. */
474 const resolveBase = (entry: IndexedEntry): Effect.Effect<CachedContent, PackIngestError> =>
475 Effect.gen(function* () {
476 if (entry.baseOffset !== undefined) {
477 const base = byOffset.get(entry.baseOffset);
478 if (base === undefined) {
479 return yield* PackFormatError.make({
480 reason: `ofs-delta at ${entry.offset} points at ${entry.baseOffset}, not an entry boundary`,
481 });
482 }
483 return yield* resolveContent(base);
484 }
485 const baseOid = entry.baseOid!;
486 const inPack = offsetByOid.get(baseOid);
487 if (inPack !== undefined) {
488 return yield* resolveContent(byOffset.get(inPack)!);
489 }
490 const cachedThin = cache.get(`oid:${baseOid}`);
491 if (cachedThin !== undefined) return cachedThin;
492 const live = yield* store.readBase(baseOid);
493 if (live === undefined) {
494 return yield* MissingDeltaBaseError.make({ baseOid });
495 }
496 if (live.content.length > maxObjectSize) {
497 return yield* ObjectTooLargeError.make({
498 size: live.content.length,
499 limit: maxObjectSize,
500 oid: baseOid,
501 });
502 }
503 const thin: CachedContent = { type: live.type, content: live.content };
504 cache.set(`oid:${baseOid}`, thin);
505 return thin;
506 });
507
508 /** Resolves, hashes, deflates, and emits one delta entry. */
509 const emitDelta = (entry: IndexedEntry) =>
510 Effect.gen(function* () {
511 const { content, type } = yield* resolveContent(entry);
512 const oid = yield* timed("hash", hashObject(type, content));
513 const zdata = yield* timed("deflate", deflate(content));
514 entry.resolved = { oid, type };
515 offsetByOid.set(oid, entry.offset);
516 oids.push(oid);
517 yield* timed(
518 "sink",
519 sink({
520 oid,
521 type,
522 size: content.length,
523 zdata,
524 fromDelta: true,
525 dataOffset: -1,
526 content,
527 }),
528 );
529 });
530
531 // ── main pass: index + resolve in pack order ─────────────────────────────
532 const sinkBatch = options.sinkBatch;
533 let pendingSink: Array<ResolvedEntry> = [];
534 // On a streaming source the body arrives through the event loop, and
535 // a synchronous batch monopolizes the isolate: without a real
536 // (macrotask) yield between batches, receive and parse alternate
537 // instead of overlapping (DESIGN §22.6). Fiber yields are not enough —
538 // they stay within the same task.
539 const streaming = !Number.isFinite(source.size);
540 const breathe = streaming
541 ? Effect.promise(() => new Promise<void>((resolve) => setTimeout(resolve, 0)))
542 : Effect.void;
543 const flushPending = Effect.suspend(() => {
544 if (pendingSink.length === 0 || sinkBatch === undefined) return breathe;
545 const batch = pendingSink;
546 pendingSink = [];
547 return timed("sink", sinkBatch(batch)).pipe(Effect.andThen(breathe));
548 });
549 /**
550 * The synchronous fast path (DESIGN §22.5): a non-delta entry whose
551 * bytes are already in memory is decoded, inflated, hashed and copied
552 * without a single fiber hop. Returns the next offset, `undefined` to
553 * fall through to the general path (a delta, an entry outrunning the
554 * window, or no sync inflate), or an error to fail with.
555 */
556 const syncEntry = (
557 window: Uint8Array,
558 offset: number,
559 i: number,
560 ): number | undefined | PackFormatError | ObjectTooLargeError => {
561 let header;
562 try {
563 header = decodeTypeSize(window, 0);
564 } catch (error) {
565 return PackFormatError.make({
566 reason: Schema.is(ObjectParseError)(error)
567 ? `entry ${i}: ${error.reason}`
568 : `entry ${i}: ${String(error)}`,
569 });
570 }
571 if (isDeltaType(header.type)) return undefined;
572 if (header.size > maxObjectSize) {
573 return ObjectTooLargeError.make({
574 size: header.size,
575 limit: maxObjectSize,
576 });
577 }
578 const pos = header.next;
579 const inflated = inflateEntrySync(window, pos, {
580 maxOutput: maxObjectSize,
581 expectedSize: header.size,
582 });
583 if (inflated === undefined) return undefined;
584 const { bytesConsumed, content } = inflated;
585 if (content.length !== header.size) {
586 return PackFormatError.make({
587 reason: `entry ${i}: inflated ${content.length} bytes, header declared ${header.size}`,
588 });
589 }
590 const type = header.type as ObjectType;
591 const oid = hashObjectSync(type, content);
592 const dataOffset = offset + pos;
593 const zdata = Uint8Array.from(window.subarray(pos, pos + bytesConsumed));
594 const entry: IndexedEntry = {
595 offset,
596 entryType: header.type,
597 declaredSize: header.size,
598 dataOffset,
599 span: bytesConsumed,
600 baseOffset: undefined,
601 baseOid: undefined,
602 resolved: { oid, type },
603 };
604 byOffset.set(offset, entry);
605 offsetByOid.set(oid, offset);
606 cache.set(`ofs:${offset}`, { type, content });
607 oids.push(oid);
608 pendingSink.push({
609 oid,
610 type,
611 size: content.length,
612 zdata,
613 fromDelta: false,
614 dataOffset,
615 content,
616 });
617 trailerSha.update(window.subarray(0, pos + bytesConsumed));
618 source.release?.(offset);
619 return dataOffset + bytesConsumed;
620 };
621
622 let offset = 12;
623 for (let i = 0; i < count; i++) {
624 if (offset >= dataEnd) {
625 return yield* PackFormatError.make({
626 reason: `truncated pack: entry ${i} of ${count} starts past the trailer`,
627 });
628 }
629 // Read a BOUNDED window, not the rest of the pack.
630 //
631 // `source.read(offset, dataEnd - offset)` is free on a buffer (a
632 // subarray view) but catastrophic on a remote source: it copies the
633 // whole remainder once per entry, so a 13.7k-object pack in R2 moved
634 // hundreds of GB and took minutes instead of seconds. An entry is
635 // almost always a few KiB, so one small window covers it; the rare
636 // large object grows the window on demand (below).
637 const probeLength = Math.min(ENTRY_WINDOW_BYTES, dataEnd - offset);
638 const direct = source.readSync?.(offset, probeLength);
639 if (direct !== undefined) {
640 const at = phases === undefined ? 0 : performance.now();
641 const next = syncEntry(direct, offset, i);
642 if (phases !== undefined) {
643 phases["sync"] = (phases["sync"] ?? 0) + (performance.now() - at);
644 }
645 if (typeof next === "number") {
646 offset = next;
647 if (sinkBatch === undefined) {
648 const entry = pendingSink.pop()!;
649 yield* timed("sink", sink(entry));
650 } else if (pendingSink.length >= SINK_BATCH) {
651 yield* flushPending;
652 }
653 continue;
654 }
655 if (next !== undefined) return yield* next;
656 }
657 let window = yield* source.read(offset, probeLength);
658 if (window.length === 0) {
659 return yield* PackFormatError.make({
660 reason: `truncated pack: entry ${i} of ${count} starts past the end`,
661 });
662 }
663 /**
664 * Re-reads a bigger window when an entry's compressed stream runs past
665 * the current one. Doubling from the entry window reaches the whole
666 * remainder in a handful of steps, and only for objects that need it.
667 */
668 const growWindow = Effect.fn(function* (needed: number) {
669 const size = Math.min(Math.max(needed, window.length * 2), dataEnd - offset);
670 if (size <= window.length) return false;
671 window = yield* source.read(offset, size);
672 return true;
673 });
674 const header = yield* Effect.try({
675 try: () => decodeTypeSize(window, 0),
676 catch: (error) =>
677 PackFormatError.make({
678 reason: Schema.is(ObjectParseError)(error)
679 ? `entry ${i}: ${error.reason}`
680 : `entry ${i}: ${String(error)}`,
681 }),
682 });
683 let pos = header.next;
684 let baseOffset: number | undefined;
685 let baseOid: Oid | undefined;
686 if (header.type === 6) {
687 const ofs = yield* Effect.try({
688 try: () => decodeOfsDeltaOffset(window, pos),
689 catch: (error) =>
690 PackFormatError.make({
691 reason: Schema.is(ObjectParseError)(error)
692 ? `entry ${i}: ${error.reason}`
693 : `entry ${i}: ${String(error)}`,
694 }),
695 });
696 pos = ofs.next;
697 baseOffset = offset - ofs.value;
698 if (baseOffset < 12) {
699 return yield* PackFormatError.make({
700 reason: `entry ${i}: ofs-delta offset ${ofs.value} points before the first entry`,
701 });
702 }
703 } else if (header.type === 7) {
704 if (pos + 20 > window.length) {
705 return yield* PackFormatError.make({
706 reason: `entry ${i}: truncated ref-delta base id`,
707 });
708 }
709 baseOid = bytesToHex(window.subarray(pos, pos + 20));
710 pos += 20;
711 } else if (header.size > maxObjectSize) {
712 return yield* ObjectTooLargeError.make({
713 size: header.size,
714 limit: maxObjectSize,
715 });
716 }
717
718 // An entry whose compressed stream runs past the window grows it and
719 // retries; virtually every object fits the first read.
720 const inflateOptions = {
721 maxOutput: maxObjectSize,
722 // Lets the synchronous fast path verify itself (see Zlib.ts).
723 expectedSize: header.size,
724 };
725 let attempt = yield* Effect.result(
726 timed("inflate", inflateEntry(window, pos, inflateOptions)),
727 );
728 // The header declares the uncompressed size; the compressed span is
729 // at most that plus deflate's own overhead, so one grow reaches an
730 // entry that outran the probe instead of doubling toward it (each
731 // failed attempt below costs up to three inflate passes).
732 while (
733 Result.isFailure(attempt) &&
734 (yield* growWindow(Math.max(window.length * 2, pos + header.size + 64 * 1024)))
735 ) {
736 attempt = yield* Effect.result(inflateEntry(window, pos, inflateOptions));
737 }
738 if (Result.isFailure(attempt)) {
739 return yield* attempt.failure;
740 }
741 const { bytesConsumed, content } = attempt.success;
742 if (content.length !== header.size) {
743 return yield* PackFormatError.make({
744 reason: `entry ${i}: inflated ${content.length} bytes, header declared ${header.size}`,
745 });
746 }
747 const dataOffset = offset + pos;
748 const entry: IndexedEntry = {
749 offset,
750 entryType: header.type,
751 declaredSize: header.size,
752 dataOffset,
753 span: bytesConsumed,
754 baseOffset,
755 baseOid,
756 };
757 byOffset.set(offset, entry);
758
759 if (!isDeltaType(header.type)) {
760 const type = header.type as ObjectType;
761 const oid = yield* timed("hash", hashObject(type, content));
762 // keep the compressed span verbatim — copy it out of the shared buffer
763 const zview = yield* source.read(dataOffset, bytesConsumed);
764 const zdata = yield* timed(
765 "copy",
766 Effect.sync(() => Uint8Array.from(zview)),
767 );
768 entry.resolved = { oid, type };
769 offsetByOid.set(oid, entry.offset);
770 cache.set(`ofs:${offset}`, { type, content });
771 oids.push(oid);
772 yield* timed(
773 "sink",
774 sink({
775 oid,
776 type,
777 size: content.length,
778 zdata,
779 fromDelta: false,
780 dataOffset,
781 content,
782 }),
783 );
784 } else {
785 const result = yield* Effect.result(emitDelta(entry));
786 if (Result.isFailure(result)) {
787 if (
788 Schema.is(MissingDeltaBaseError)(result.failure) ||
789 Schema.is(BaseEvictedError)(result.failure)
790 ) {
791 // The base may appear later in the pack (thin ref-delta), or
792 // it was evicted from a streaming source and is readable once
793 // the body has ended: retry in the deferred pass.
794 deferred.push(entry);
795 } else {
796 return yield* Effect.fail(result.failure);
797 }
798 }
799 }
800 trailerSha.update(window.subarray(0, pos + bytesConsumed));
801 source.release?.(offset);
802 offset = dataOffset + bytesConsumed;
803 }
804
805 yield* flushPending;
806 if (!Number.isFinite(dataEnd)) {
807 if (source.awaitEnd === undefined) {
808 return yield* PackFormatError.make({
809 reason: "pack source has unknown size and no awaitEnd",
810 });
811 }
812 dataEnd = (yield* source.awaitEnd) - 20;
813 }
814 // ── deferred deltas: fixpoint passes AFTER the end ───────────────────────
815 // REF_DELTAs whose base came later in the pack, and (streaming) deltas
816 // whose base had been evicted before the body ended — readable now
817 // through the spilled object.
818 let pending = deferred;
819 while (pending.length > 0) {
820 const next: Array<IndexedEntry> = [];
821 let firstMissing: MissingDeltaBaseError | undefined;
822 for (const entry of pending) {
823 const result = yield* Effect.result(emitDelta(entry));
824 if (Result.isFailure(result)) {
825 if (Schema.is(MissingDeltaBaseError)(result.failure)) {
826 firstMissing ??= result.failure;
827 next.push(entry);
828 } else if (Schema.is(BaseEvictedError)(result.failure)) {
829 return yield* PackFormatError.make({
830 reason: `delta base at ${result.failure.offset} unreadable after the body ended`,
831 });
832 } else {
833 return yield* Effect.fail(result.failure);
834 }
835 }
836 }
837 if (next.length === pending.length) {
838 // no progress — the base genuinely does not exist anywhere
839 return yield* firstMissing!;
840 }
841 pending = next;
842 }
843 if (offset !== dataEnd) {
844 return yield* PackFormatError.make({
845 reason: `pack has ${dataEnd - offset} unconsumed bytes after ${count} entries`,
846 });
847 }
848
849 // ── trailer verification ─────────────────────────────────────────────────
850 const trailer = yield* source.read(dataEnd, 20);
851 if (trailer.length < 20) {
852 return yield* PackFormatError.make({ reason: "truncated pack trailer" });
853 }
854 const expected = bytesToHex(trailer);
855 const actual = yield* Effect.sync(() => trailerSha.digestHex());
856 if (expected !== actual) {
857 return yield* PackChecksumMismatch.make({ expected, actual });
858 }
859
860 return { count, oids } satisfies IngestSummary;
861 });