@devries-cameron20 / t3code Mirror · read-only
Fork 0

The best way to control Claude Code, Codex and more from any computer or phone

Mirrored from https://github.com/Camo67/t3code.git

Synced Nov 21, 06:00 UTC

main
ts 268 lines 10.4 KB
Raw
1 /**
2 * The spilled-pack reader (packages/git/src/Store/PackSource.ts): windowed random
3 * access over blob storage. Reads within a window must be VIEWS of the
4 * cached slab (a copy per probe read was the 40 s spilled-push ingest), and
5 * a parse through tiny windows — many window boundaries, straddling
6 * entries — must produce exactly what the in-memory parse produces.
7 */
8 import {
9 hashObject,
10 makeSha1,
11 encodeTypeSize,
12 type Oid,
13 } from "@alchemy.run/git/Protocol/ObjectCodec";
14 import { bufferRandomAccess, ingestPack, SINK_BATCH } from "@alchemy.run/git/Protocol/PackParser";
15 import { packHeader } from "@alchemy.run/git/Protocol/PackWriter";
16 import * as Zlib from "@alchemy.run/git/Protocol/Zlib";
17 import { makeObjectStore } from "@alchemy.run/git/Store/ObjectStore";
18 import { blobRandomAccess, sliceRandomAccess } from "@alchemy.run/git/Store/PackSource";
19 import { makeStreamingSource } from "@alchemy.run/git/Store/StreamingSource";
20 import { describe, expect, test } from "@alchemy.run/test";
21 import { RuntimeContext } from "alchemy/RuntimeContext";
22 import * as Effect from "effect/Effect";
23 import * as Fiber from "effect/Fiber";
24 import { concat } from "./harness/pack.ts";
25 import { makeMemoryBlobStore, makeTestSqlClient } from "./harness/store.ts";
26
27 /** A synthetic non-delta pack of `n` blobs with sizes cycling 100..5000. */
28 const buildPack = (n: number, options?: { readonly incompressible?: boolean }) =>
29 Effect.gen(function* () {
30 const pieces: Array<Uint8Array> = [packHeader(n)];
31 const oids: Array<Oid> = [];
32 for (let i = 0; i < n; i++) {
33 const size = 100 + ((i * 977) % 4900);
34 const content = new Uint8Array(size);
35 if (options?.incompressible) crypto.getRandomValues(content);
36 else for (let j = 0; j < size; j++) content[j] = (j * 7 + i) & 0xff;
37 content[0] = i & 0xff;
38 oids.push(yield* hashObject(3, content));
39 pieces.push(encodeTypeSize(3, size), yield* Zlib.deflate(content));
40 }
41 const body = concat(pieces);
42 const sha = makeSha1();
43 sha.update(body);
44 return { pack: concat([body, sha.digest()]), oids };
45 });
46
47 const parseWith = (source: ReturnType<typeof bufferRandomAccess>) =>
48 Effect.gen(function* () {
49 const store = makeObjectStore({
50 sql: makeTestSqlClient(),
51 blobs: makeMemoryBlobStore(),
52 repoId: "R",
53 });
54 const seen: Array<string> = [];
55 const offsets: Array<{ dataOffset: number; zdata: Uint8Array; fromDelta: boolean }> = [];
56 const summary = yield* ingestPack({
57 source,
58 store,
59 sink: (entry) =>
60 Effect.sync(() => {
61 seen.push(entry.oid);
62 offsets.push({
63 dataOffset: entry.dataOffset,
64 zdata: entry.zdata,
65 fromDelta: entry.fromDelta,
66 });
67 }),
68 });
69 return { count: summary.count, seen, offsets };
70 });
71
72 describe("blobRandomAccess", { tags: ["unit", "local"] }, () => {
73 test("reads inside one window are views of the cached slab, not copies", async () => {
74 await Effect.runPromise(
75 Effect.gen(function* () {
76 const blobs = makeMemoryBlobStore();
77 const bytes = new Uint8Array(10_000);
78 for (let i = 0; i < bytes.length; i++) bytes[i] = i & 0xff;
79 yield* blobs.put("k", bytes);
80 const source = blobRandomAccess({
81 blobs,
82 key: "k",
83 size: bytes.length,
84 windowBytes: 4096,
85 maxWindows: 2,
86 });
87 const a = yield* source.read(10, 100);
88 const b = yield* source.read(200, 1000);
89 expect(a.buffer).toBe(b.buffer); // same slab
90 expect(a[0]).toBe(10);
91 expect(b[0]).toBe(200);
92 expect(blobs.gets.length).toBe(1); // one window fetch served both
93 // Crossing a window boundary assembles a fresh buffer.
94 const c = yield* source.read(4000, 200);
95 expect(c.length).toBe(200);
96 expect(c[96]).toBe(4096 & 0xff);
97 expect(blobs.gets.length).toBe(2);
98 // A read past the end is clipped.
99 const tail = yield* source.read(9_990, 100);
100 expect(tail.length).toBe(10);
101 }).pipe(Effect.provide(RuntimeContext.phantom)),
102 );
103 });
104
105 test("a parse through 1 KiB windows equals the in-memory parse (boundaries, straddles, eviction)", async () => {
106 await Effect.runPromise(
107 Effect.gen(function* () {
108 const { pack, oids } = yield* buildPack(300);
109 expect(pack.length).toBeGreaterThan(50_000);
110 const memory = yield* parseWith(bufferRandomAccess(pack));
111 const blobs = makeMemoryBlobStore();
112 // Emulate the wire body: a request prefix before the pack, then the pack.
113 const prefix = new TextEncoder().encode("0000".repeat(37));
114 yield* blobs.put("incoming", concat([prefix, pack]));
115 const spilled = sliceRandomAccess(
116 blobRandomAccess({
117 blobs,
118 key: "incoming",
119 size: prefix.length + pack.length,
120 windowBytes: 1024,
121 maxWindows: 3,
122 }),
123 prefix.length,
124 );
125 expect(spilled.readSync).toBeDefined();
126 const windowed = yield* parseWith(spilled);
127 expect(windowed.count).toBe(300);
128 // dataOffset addresses the compressed span in the SOURCE (promotion
129 // relies on this): reading it back yields exactly zdata.
130 for (const entry of windowed.offsets.slice(0, 50)) {
131 expect(entry.fromDelta).toBe(false);
132 const span = yield* spilled.read(entry.dataOffset, entry.zdata.length);
133 expect(Array.from(span)).toEqual(Array.from(entry.zdata));
134 }
135 expect(windowed.seen).toEqual(memory.seen);
136 expect(new Set(windowed.seen)).toEqual(new Set(oids));
137 }).pipe(Effect.provide(RuntimeContext.phantom)),
138 );
139 });
140
141 test("with production geometry (large windows) a sequential parse fetches about one window each", async () => {
142 await Effect.runPromise(
143 Effect.gen(function* () {
144 const { pack } = yield* buildPack(600, { incompressible: true });
145 const blobs = makeMemoryBlobStore();
146 yield* blobs.put("incoming", pack);
147 const windowBytes = 64 * 1024;
148 const windows = Math.ceil(pack.length / windowBytes);
149 expect(windows).toBeGreaterThan(3);
150 const parsed = yield* parseWith(
151 blobRandomAccess({
152 blobs,
153 key: "incoming",
154 size: pack.length,
155 windowBytes,
156 maxWindows: 4,
157 }),
158 );
159 expect(parsed.count).toBe(600);
160 // Every window once, plus at most one re-read per window edge an
161 // entry's probe straddles (the LRU holds 4, the parse is sequential).
162 expect(blobs.gets.length).toBeLessThanOrEqual(windows * 2);
163 }).pipe(Effect.provide(RuntimeContext.phantom)),
164 );
165 });
166 });
167
168 describe("synchronous fast path", { tags: ["unit", "local"] }, () => {
169 test("with a batched sink, non-delta entries arrive in runs of ≤ SINK_BATCH and the per-entry sink sees none", async () => {
170 await Effect.runPromise(
171 Effect.gen(function* () {
172 const { pack, oids } = yield* buildPack(700);
173 const store = makeObjectStore({
174 sql: makeTestSqlClient(),
175 blobs: makeMemoryBlobStore(),
176 repoId: "R",
177 });
178 const batches: Array<number> = [];
179 const single: Array<string> = [];
180 const seen = new Set<string>();
181 const summary = yield* ingestPack({
182 source: bufferRandomAccess(pack),
183 store,
184 sink: (entry) =>
185 Effect.sync(() => {
186 single.push(entry.oid);
187 }),
188 sinkBatch: (entries) =>
189 Effect.sync(() => {
190 batches.push(entries.length);
191 for (const e of entries) seen.add(e.oid);
192 }),
193 });
194 expect(summary.count).toBe(700);
195 expect(single).toEqual([]);
196 expect(batches.every((n) => n <= SINK_BATCH && n > 0)).toBe(true);
197 expect(batches.reduce((a, b) => a + b, 0)).toBe(700);
198 expect(seen).toEqual(new Set(oids));
199 }),
200 );
201 });
202 });
203
204 describe("parsing a pack while it streams in (DESIGN §22.6)", { tags: ["unit", "local"] }, () => {
205 test(
206 "parse runs concurrently with the feed, in random chunk sizes, and verifies the trailer",
207 async () => {
208 await Effect.runPromise(
209 Effect.gen(function* () {
210 const { pack, oids } = yield* buildPack(500, { incompressible: true });
211 const feeder = makeStreamingSource({
212 slabBytes: 64 * 1024,
213 retainBytes: 256 * 1024,
214 backpressureBytes: 128 * 1024,
215 });
216 const prefix = new TextEncoder().encode(
217 "0021push refs/heads/main x\n0000".padEnd(37, "\0"),
218 );
219 const parse = yield* Effect.forkChild(
220 parseWith(sliceRandomAccess(feeder.source, prefix.length)),
221 );
222 const body = concat([prefix, pack]);
223 let at = 0;
224 let seed = 7;
225 while (at < body.length) {
226 seed = (seed * 1103515245 + 12345) & 0x7fffffff;
227 const n = 1 + (seed % 20_000);
228 yield* feeder.push(body.subarray(at, at + n));
229 at += n;
230 if (seed % 5 === 0) yield* Effect.yieldNow;
231 }
232 feeder.end();
233 const parsed = yield* Fiber.join(parse);
234 expect(parsed.count).toBe(500);
235 expect(new Set(parsed.seen)).toEqual(new Set(oids));
236 }),
237 );
238 },
239 { timeout: 30_000 },
240 );
241
242 test("a body that ends early is a truncated-pack error, not a hang", async () => {
243 await Effect.runPromise(
244 Effect.gen(function* () {
245 const { pack } = yield* buildPack(50);
246 const feeder = makeStreamingSource({ slabBytes: 4096 });
247 const parse = yield* Effect.forkChild(Effect.result(parseWith(feeder.source)));
248 yield* feeder.push(pack.subarray(0, Math.floor(pack.length / 2)));
249 feeder.end();
250 const r = yield* Fiber.join(parse);
251 expect(r._tag).toBe("Failure");
252 }),
253 );
254 });
255
256 test("a corrupted trailer is rejected even though the hash is accumulated incrementally", async () => {
257 await Effect.runPromise(
258 Effect.gen(function* () {
259 const { pack } = yield* buildPack(20);
260 const bad = Uint8Array.from(pack);
261 bad[bad.length - 1] ^= 0xff;
262 const r = yield* Effect.result(parseWith(bufferRandomAccess(bad)));
263 expect(r._tag).toBe("Failure");
264 if (r._tag === "Failure") expect(String(r.failure._tag)).toBe("PackChecksumMismatch");
265 }),
266 );
267 });
268 });