The best way to control Claude Code, Codex and more from any computer or phone
| 1 | import { |
| 2 | runCompactJob, |
| 3 | runGeometricMergeJob, |
| 4 | shouldCompact, |
| 5 | BLOB_TYPE, |
| 6 | } from "@alchemy.run/git/Jobs/Compact"; |
| 7 | /** |
| 8 | * Object store unit tests (packages/git/src/Store/ObjectStore.ts, Jobs/Compact.ts) |
| 9 | * over the bun:sqlite + in-memory blob harness. Every emitted pack is |
| 10 | * re-parsed by the real pack parser, so the bytes — not just the counts — |
| 11 | * are verified. |
| 12 | */ |
| 13 | import { |
| 14 | encodeTypeSize, |
| 15 | makeSha1, |
| 16 | hashObject, |
| 17 | type Oid, |
| 18 | type ObjectType, |
| 19 | } from "@alchemy.run/git/Protocol/ObjectCodec"; |
| 20 | import { bufferRandomAccess, ingestPack } from "@alchemy.run/git/Protocol/PackParser"; |
| 21 | import { packHeader } from "@alchemy.run/git/Protocol/PackWriter"; |
| 22 | import type { ManifestEntry } from "@alchemy.run/git/Protocol/Store"; |
| 23 | import * as Zlib from "@alchemy.run/git/Protocol/Zlib"; |
| 24 | import { packKey, packKeyOf, wirePackId } from "@alchemy.run/git/Store/Keys"; |
| 25 | import { |
| 26 | makeObjectStore, |
| 27 | STAGE_INSERT_ROWS, |
| 28 | WINDOW_BYTES, |
| 29 | WINDOW_CACHE_BYTES, |
| 30 | } from "@alchemy.run/git/Store/ObjectStore"; |
| 31 | import { describe, expect, test } from "@alchemy.run/test"; |
| 32 | import { RuntimeContext } from "alchemy/RuntimeContext"; |
| 33 | import * as Effect from "effect/Effect"; |
| 34 | import * as Stream from "effect/Stream"; |
| 35 | import { concat, verifyPack } from "./harness/pack.ts"; |
| 36 | import { makeMemoryBlobStore, makeTestSqlClient } from "./harness/store.ts"; |
| 37 | |
| 38 | const REPO = "01TESTREPO0000000000000000"; |
| 39 | |
| 40 | /** Runs an effect, surfacing typed failures with their fields (not just `message`). */ |
| 41 | const run = <A>(effect: Effect.Effect<A, unknown, RuntimeContext>) => |
| 42 | Effect.runPromise( |
| 43 | effect.pipe( |
| 44 | Effect.provide(RuntimeContext.phantom), |
| 45 | Effect.mapError((e) => |
| 46 | e instanceof Error && !("reason" in e) ? e : new Error(JSON.stringify(e)), |
| 47 | ), |
| 48 | ), |
| 49 | ); |
| 50 | |
| 51 | /** Incompressible bytes; `seed` only guarantees distinct content per call. */ |
| 52 | const bytes = (n: number, seed: number) => { |
| 53 | const out = new Uint8Array(n); |
| 54 | for (let at = 0; at < n; at += 65536) { |
| 55 | crypto.getRandomValues(out.subarray(at, Math.min(at + 65536, n))); |
| 56 | } |
| 57 | out[0] = seed & 0xff; |
| 58 | return out; |
| 59 | }; |
| 60 | |
| 61 | interface Fixture { |
| 62 | readonly oid: Oid; |
| 63 | readonly type: ObjectType; |
| 64 | readonly content: Uint8Array; |
| 65 | readonly zdata: Uint8Array; |
| 66 | } |
| 67 | |
| 68 | const makeFixture = (type: ObjectType, content: Uint8Array) => |
| 69 | Effect.gen(function* () { |
| 70 | const oid = yield* hashObject(type, content); |
| 71 | const zdata = yield* Zlib.deflate(content); |
| 72 | return { oid, type, content, zdata } satisfies Fixture; |
| 73 | }); |
| 74 | |
| 75 | /** A repo with `blobs` incompressible blobs of `blobSize` and `trees` small trees. */ |
| 76 | const seedRepo = (options: { blobs: number; blobSize: number; trees: number }) => |
| 77 | Effect.gen(function* () { |
| 78 | const sql = makeTestSqlClient(); |
| 79 | const blobs = makeMemoryBlobStore(); |
| 80 | const store = makeObjectStore({ sql, blobs, repoId: REPO }); |
| 81 | const fixtures: Array<Fixture> = []; |
| 82 | for (let i = 0; i < options.blobs; i++) { |
| 83 | fixtures.push(yield* makeFixture(3, bytes(options.blobSize, i + 1))); |
| 84 | } |
| 85 | for (let i = 0; i < options.trees; i++) { |
| 86 | // Not a real tree encoding; the store never parses tree bytes here. |
| 87 | fixtures.push(yield* makeFixture(2, new TextEncoder().encode(`tree ${i} ${"x".repeat(40)}`))); |
| 88 | } |
| 89 | yield* sql.transactionSync((raw) => { |
| 90 | for (const f of fixtures) { |
| 91 | raw.exec( |
| 92 | `INSERT INTO objects (oid, type, size, zsize, location, zdata) VALUES (?, ?, ?, ?, 'row', ?)`, |
| 93 | f.oid, |
| 94 | f.type, |
| 95 | f.content.length, |
| 96 | f.zdata.length, |
| 97 | f.zdata.buffer.slice(f.zdata.byteOffset, f.zdata.byteOffset + f.zdata.byteLength), |
| 98 | ); |
| 99 | } |
| 100 | }); |
| 101 | return { sql, blobs, store, fixtures }; |
| 102 | }); |
| 103 | |
| 104 | const manifestOf = (fixtures: ReadonlyArray<Fixture>, location: ManifestEntry["location"]) => |
| 105 | fixtures.map((f): ManifestEntry => ({ |
| 106 | oid: f.oid, |
| 107 | type: f.type, |
| 108 | size: f.content.length, |
| 109 | zsize: f.zdata.length, |
| 110 | location, |
| 111 | })); |
| 112 | |
| 113 | /** Emits a full pack (header + entries + trailer) through `packEntries`. */ |
| 114 | const buildPack = ( |
| 115 | store: ReturnType<typeof makeObjectStore>, |
| 116 | entries: ReadonlyArray<ManifestEntry>, |
| 117 | ) => |
| 118 | Effect.gen(function* () { |
| 119 | const body = Array.from(yield* Stream.runCollect(store.packEntries(entries))); |
| 120 | const head = packHeader(entries.length); |
| 121 | const sha = makeSha1(); |
| 122 | sha.update(head); |
| 123 | for (const b of body) sha.update(b); |
| 124 | return concat([head, ...body, sha.digest()]); |
| 125 | }); |
| 126 | |
| 127 | /** Parses a pack with the real parser and returns the oids it yields. */ |
| 128 | const parsePack = (pack: Uint8Array, store: ReturnType<typeof makeObjectStore>) => |
| 129 | Effect.gen(function* () { |
| 130 | const seen = new Map<string, number>(); |
| 131 | const summary = yield* ingestPack({ |
| 132 | source: bufferRandomAccess(pack), |
| 133 | store, |
| 134 | sink: (entry) => |
| 135 | Effect.sync(() => { |
| 136 | seen.set(entry.oid, entry.size); |
| 137 | }), |
| 138 | }); |
| 139 | return { count: summary.count, seen }; |
| 140 | }); |
| 141 | |
| 142 | const compactAll = ( |
| 143 | sql: ReturnType<typeof makeTestSqlClient>, |
| 144 | blobs: ReturnType<typeof makeMemoryBlobStore>, |
| 145 | maxObjects = 100, |
| 146 | ) => |
| 147 | Effect.gen(function* () { |
| 148 | const packIds: Array<string> = []; |
| 149 | for (;;) { |
| 150 | const out = yield* runCompactJob({ |
| 151 | repoId: REPO, |
| 152 | sql, |
| 153 | blobs, |
| 154 | maxObjects, |
| 155 | maxBytes: 64 * 1024 * 1024, |
| 156 | }); |
| 157 | if (out.packId !== undefined) packIds.push(out.packId); |
| 158 | if (!out.more) break; |
| 159 | } |
| 160 | return packIds; |
| 161 | }); |
| 162 | |
| 163 | describe("compaction is blob-only", { tags: ["unit", "local"] }, () => { |
| 164 | test("shouldCompact ignores commits/trees; runCompactJob leaves them as rows", async () => { |
| 165 | await run( |
| 166 | Effect.gen(function* () { |
| 167 | const { sql, blobs, fixtures } = yield* seedRepo({ blobs: 5, blobSize: 100, trees: 50 }); |
| 168 | expect(BLOB_TYPE).toBe(3); |
| 169 | // 55 rows total, but only 5 blobs: below a count threshold of 10. |
| 170 | expect(yield* shouldCompact(sql, { countThreshold: 10, bytesThreshold: 1 << 30 })).toBe( |
| 171 | false, |
| 172 | ); |
| 173 | expect(yield* shouldCompact(sql, { countThreshold: 5, bytesThreshold: 1 << 30 })).toBe( |
| 174 | true, |
| 175 | ); |
| 176 | const packs = yield* compactAll(sql, blobs); |
| 177 | expect(packs.length).toBe(1); |
| 178 | const rows = yield* sql.all<{ location: string; type: number; n: number }>( |
| 179 | `SELECT location, type, COUNT(*) AS n FROM objects GROUP BY location, type ORDER BY location, type`, |
| 180 | ); |
| 181 | expect(rows).toEqual([ |
| 182 | { location: "pack", type: 3, n: 5 }, |
| 183 | { location: "row", type: 2, n: 50 }, |
| 184 | ]); |
| 185 | expect(yield* runCompactJob({ repoId: REPO, sql, blobs })).toMatchObject({ |
| 186 | moved: 0, |
| 187 | more: false, |
| 188 | }); |
| 189 | expect(fixtures.length).toBe(55); |
| 190 | }), |
| 191 | ); |
| 192 | }); |
| 193 | }); |
| 194 | |
| 195 | describe("packEntries", { tags: ["unit", "local"] }, () => { |
| 196 | test( |
| 197 | "rows + multi-window pack (with straddling objects) emit a pack the parser accepts", |
| 198 | async () => { |
| 199 | await run( |
| 200 | Effect.gen(function* () { |
| 201 | // 300 × 30 KiB incompressible ≈ 9 MiB → 3 windows, with objects |
| 202 | // straddling both window edges. |
| 203 | const { sql, blobs, store, fixtures } = yield* seedRepo({ |
| 204 | blobs: 300, |
| 205 | blobSize: 30 * 1024, |
| 206 | trees: 40, |
| 207 | }); |
| 208 | const packs = yield* compactAll(sql, blobs, 1000); // one pack |
| 209 | expect(packs.length).toBe(1); |
| 210 | const packBytes = blobs.objects.get(packKey(REPO, packs[0]!))!; |
| 211 | expect(packBytes.length).toBeGreaterThan(2 * WINDOW_BYTES); // 3 windows |
| 212 | const coords = yield* sql.all<{ pack_id: string; n: number }>( |
| 213 | `SELECT pack_id, COUNT(*) AS n FROM objects WHERE location='pack' GROUP BY pack_id`, |
| 214 | ); |
| 215 | expect(coords.map((c) => c.n)).toEqual([300]); |
| 216 | |
| 217 | // Manifest in the order a closure would produce (trees first). |
| 218 | const entries = [ |
| 219 | ...manifestOf( |
| 220 | fixtures.filter((f) => f.type === 2), |
| 221 | "row", |
| 222 | ), |
| 223 | ...manifestOf( |
| 224 | fixtures.filter((f) => f.type === 3), |
| 225 | "pack", |
| 226 | ), |
| 227 | ]; |
| 228 | blobs.gets.length = 0; |
| 229 | const pack = yield* buildPack(store, entries); |
| 230 | expect(verifyPack(pack).error).toBeUndefined(); |
| 231 | const parsed = yield* parsePack(pack, store); |
| 232 | expect(parsed.count).toBe(340); |
| 233 | expect(parsed.seen.size).toBe(340); |
| 234 | for (const f of fixtures) expect(parsed.seen.get(f.oid)).toBe(f.content.length); |
| 235 | |
| 236 | // One ranged GET per window touched, not per object. |
| 237 | const windowGets = blobs.gets.filter((g) => g.length === WINDOW_BYTES); |
| 238 | const straddleGets = blobs.gets.filter((g) => g.length !== WINDOW_BYTES); |
| 239 | expect(windowGets.length).toBe(3); |
| 240 | expect(straddleGets.length).toBeLessThanOrEqual(2); // ≤ one per window edge |
| 241 | }), |
| 242 | ); |
| 243 | }, |
| 244 | { timeout: 60_000 }, |
| 245 | ); |
| 246 | |
| 247 | test("a manifest entry that compaction moved after the closure is still emitted (re-dispatch)", async () => { |
| 248 | await run( |
| 249 | Effect.gen(function* () { |
| 250 | const { sql, blobs, store, fixtures } = yield* seedRepo({ |
| 251 | blobs: 20, |
| 252 | blobSize: 2048, |
| 253 | trees: 3, |
| 254 | }); |
| 255 | // The closure saw everything as rows… |
| 256 | const entries = manifestOf(fixtures, "row"); |
| 257 | // …then compaction ran before emission. |
| 258 | yield* compactAll(sql, blobs); |
| 259 | const pack = yield* buildPack(store, entries); |
| 260 | expect(verifyPack(pack).error).toBeUndefined(); |
| 261 | const parsed = yield* parsePack(pack, store); |
| 262 | expect(parsed.seen.size).toBe(23); |
| 263 | }), |
| 264 | ); |
| 265 | }); |
| 266 | |
| 267 | test("a truly absent object fails the stream instead of emitting a short pack", async () => { |
| 268 | await run( |
| 269 | Effect.gen(function* () { |
| 270 | const { store, fixtures } = yield* seedRepo({ blobs: 2, blobSize: 100, trees: 0 }); |
| 271 | const ghost = { ...manifestOf(fixtures, "row")[0]!, oid: "0".repeat(40) as Oid }; |
| 272 | const result = yield* Effect.result(Stream.runCollect(store.packEntries([ghost]))); |
| 273 | expect(result._tag).toBe("Failure"); |
| 274 | }), |
| 275 | ); |
| 276 | }); |
| 277 | }); |
| 278 | |
| 279 | describe("readContentBatch", { tags: ["unit", "local"] }, () => { |
| 280 | test("returns inflated content for rows and packed objects alike", async () => { |
| 281 | await run( |
| 282 | Effect.gen(function* () { |
| 283 | const { sql, blobs, store, fixtures } = yield* seedRepo({ |
| 284 | blobs: 30, |
| 285 | blobSize: 512, |
| 286 | trees: 10, |
| 287 | }); |
| 288 | yield* compactAll(sql, blobs); |
| 289 | const out = yield* store.readContentBatch(fixtures.map((f) => f.oid)); |
| 290 | expect(out.size).toBe(40); |
| 291 | for (const f of fixtures) { |
| 292 | expect(Array.from(out.get(f.oid)!)).toEqual(Array.from(f.content)); |
| 293 | } |
| 294 | }), |
| 295 | ); |
| 296 | }); |
| 297 | }); |
| 298 | |
| 299 | describe("window cache", { tags: ["unit", "local"] }, () => { |
| 300 | test( |
| 301 | "is bounded by WINDOW_CACHE_BYTES: re-reading the first of nine windows refetches", |
| 302 | async () => { |
| 303 | await run( |
| 304 | Effect.gen(function* () { |
| 305 | // 36 × 1 MiB incompressible → one 36 MiB pack = 9 windows > 8 cached. |
| 306 | const { sql, blobs, store, fixtures } = yield* seedRepo({ |
| 307 | blobs: 36, |
| 308 | blobSize: 1024 * 1024, |
| 309 | trees: 0, |
| 310 | }); |
| 311 | const out = yield* runCompactJob({ |
| 312 | repoId: REPO, |
| 313 | sql, |
| 314 | blobs, |
| 315 | maxObjects: 100, |
| 316 | maxBytes: 64 * 1024 * 1024, |
| 317 | }); |
| 318 | expect(out.moved).toBe(36); |
| 319 | expect(WINDOW_CACHE_BYTES / WINDOW_BYTES).toBe(8); |
| 320 | const sorted = [...fixtures]; |
| 321 | const order = yield* sql.all<{ oid: string; pack_offset: number; zsize: number }>( |
| 322 | `SELECT oid, pack_offset, zsize FROM objects ORDER BY pack_offset`, |
| 323 | ); |
| 324 | const byOid = new Map(sorted.map((f) => [f.oid, f])); |
| 325 | const first = byOid.get(order[0]!.oid as Oid)!; |
| 326 | // An object lying entirely inside window 7 (objects that straddle a |
| 327 | // window edge take a ranged read that bypasses the cache by design). |
| 328 | const inside7 = order.find( |
| 329 | (r) => |
| 330 | Math.floor(r.pack_offset / WINDOW_BYTES) === 7 && |
| 331 | r.pack_offset + r.zsize <= 8 * WINDOW_BYTES, |
| 332 | )!; |
| 333 | const last = byOid.get(inside7.oid as Oid)!; |
| 334 | blobs.gets.length = 0; |
| 335 | yield* store.readContentBatch([first.oid]); |
| 336 | const afterFirst = blobs.gets.length; |
| 337 | expect(afterFirst).toBeGreaterThanOrEqual(1); |
| 338 | // Touch every window. |
| 339 | yield* store.readContentBatch(order.map((r) => r.oid as Oid)); |
| 340 | blobs.gets.length = 0; |
| 341 | // The first window has been evicted (9 > 8): reading it fetches again. |
| 342 | yield* store.readContentBatch([first.oid]); |
| 343 | expect(blobs.gets.length).toBeGreaterThanOrEqual(1); |
| 344 | // The last window is still hot: no fetch. |
| 345 | blobs.gets.length = 0; |
| 346 | yield* store.readContentBatch([last.oid]); |
| 347 | expect(blobs.gets.length).toBe(0); |
| 348 | }), |
| 349 | ); |
| 350 | }, |
| 351 | { timeout: 60_000 }, |
| 352 | ); |
| 353 | }); |
| 354 | |
| 355 | describe("insertStagedBatch", { tags: ["unit", "local"] }, () => { |
| 356 | test("stages a batch with multi-row inserts; staged rows are invisible to live reads", async () => { |
| 357 | await run( |
| 358 | Effect.gen(function* () { |
| 359 | const sql = makeTestSqlClient(); |
| 360 | const store = makeObjectStore({ sql, blobs: makeMemoryBlobStore(), repoId: REPO }); |
| 361 | const objects = []; |
| 362 | for (let i = 0; i < STAGE_INSERT_ROWS * 2 + 7; i++) { |
| 363 | const f = yield* makeFixture(3, bytes(64, i + 1)); |
| 364 | objects.push({ oid: f.oid, type: f.type, size: f.content.length, zdata: f.zdata }); |
| 365 | } |
| 366 | yield* store.insertStagedBatch("push-A", objects); |
| 367 | const staged = yield* sql.first<{ n: number }>( |
| 368 | `SELECT COUNT(*) AS n FROM objects WHERE staged_push = 'push-A'`, |
| 369 | ); |
| 370 | expect(staged?.n).toBe(objects.length); |
| 371 | expect(yield* store.has(objects[0]!.oid)).toBe(false); |
| 372 | }), |
| 373 | ); |
| 374 | }); |
| 375 | |
| 376 | test("a committed push's rows are live before the flip, cannot be adopted, and GC flips them (DESIGN §22.14)", async () => { |
| 377 | await run( |
| 378 | Effect.gen(function* () { |
| 379 | const sql = makeTestSqlClient(); |
| 380 | const store = makeObjectStore({ sql, blobs: makeMemoryBlobStore(), repoId: REPO }); |
| 381 | const objects = []; |
| 382 | for (let i = 0; i < 6; i++) { |
| 383 | const f = yield* makeFixture(3, bytes(64, i + 300)); |
| 384 | objects.push({ oid: f.oid, type: f.type, size: f.content.length, zdata: f.zdata }); |
| 385 | } |
| 386 | yield* sql.run( |
| 387 | `INSERT INTO pushes (push_id, started_at, state) VALUES ('push-live', ?, 'committed')`, |
| 388 | Date.now(), |
| 389 | ); |
| 390 | yield* store.insertStagedBatch("push-live", objects); |
| 391 | // Live for readers although `staged_push` is still set. |
| 392 | expect(yield* store.has(objects[0]!.oid)).toBe(true); |
| 393 | const missing = yield* store.missingObjects( |
| 394 | objects.map((o) => o.oid), |
| 395 | "push-other", |
| 396 | ); |
| 397 | expect(missing).toEqual([]); |
| 398 | // A later push re-staging the same oids must not steal them. |
| 399 | yield* store.insertStagedBatch("push-other", objects.slice(0, 3)); |
| 400 | const still = yield* sql.first<{ n: number }>( |
| 401 | `SELECT COUNT(*) AS n FROM objects WHERE staged_push = 'push-live'`, |
| 402 | ); |
| 403 | expect(still?.n).toBe(6); |
| 404 | // The deferred flip (here: what GC does) nulls the column. |
| 405 | yield* sql.run( |
| 406 | `UPDATE objects SET staged_push = NULL WHERE staged_push IN (SELECT push_id FROM pushes WHERE state = 'committed')`, |
| 407 | ); |
| 408 | const flipped = yield* sql.first<{ n: number }>( |
| 409 | `SELECT COUNT(*) AS n FROM objects WHERE staged_push IS NULL`, |
| 410 | ); |
| 411 | expect(flipped?.n).toBe(6); |
| 412 | expect(yield* store.has(objects[5]!.oid)).toBe(true); |
| 413 | }), |
| 414 | ); |
| 415 | }); |
| 416 | |
| 417 | test("rows left by a crashed push are adopted; live objects are left alone", async () => { |
| 418 | await run( |
| 419 | Effect.gen(function* () { |
| 420 | const sql = makeTestSqlClient(); |
| 421 | const store = makeObjectStore({ sql, blobs: makeMemoryBlobStore(), repoId: REPO }); |
| 422 | const objects = []; |
| 423 | for (let i = 0; i < 10; i++) { |
| 424 | const f = yield* makeFixture(3, bytes(64, i + 100)); |
| 425 | objects.push({ oid: f.oid, type: f.type, size: f.content.length, zdata: f.zdata }); |
| 426 | } |
| 427 | yield* store.insertStagedBatch("push-crashed", objects.slice(0, 4)); |
| 428 | yield* store.insertStagedBatch("push-B", objects); |
| 429 | const byPush = yield* sql.all<{ staged_push: string; n: number }>( |
| 430 | `SELECT staged_push, COUNT(*) AS n FROM objects GROUP BY staged_push`, |
| 431 | ); |
| 432 | expect(byPush).toEqual([{ staged_push: "push-B", n: 10 }]); |
| 433 | yield* sql.run(`UPDATE objects SET staged_push = NULL WHERE oid = ?`, objects[0]!.oid); |
| 434 | yield* store.insertStagedBatch("push-C", objects.slice(0, 2)); |
| 435 | const live = yield* sql.first<{ staged_push: string | null }>( |
| 436 | `SELECT staged_push FROM objects WHERE oid = ?`, |
| 437 | objects[0]!.oid, |
| 438 | ); |
| 439 | expect(live?.staged_push).toBeNull(); |
| 440 | const other = yield* sql.first<{ staged_push: string | null }>( |
| 441 | `SELECT staged_push FROM objects WHERE oid = ?`, |
| 442 | objects[1]!.oid, |
| 443 | ); |
| 444 | expect(other?.staged_push).toBe("push-C"); |
| 445 | }), |
| 446 | ); |
| 447 | }); |
| 448 | }); |
| 449 | |
| 450 | describe("promoted wire packs (DESIGN §22.5)", { tags: ["unit", "local"] }, () => { |
| 451 | /** Lays fixtures out the way a wire pack does: typeSize header + zdata per entry. */ |
| 452 | const layout = (fixtures: ReadonlyArray<Fixture>, base: number) => { |
| 453 | const pieces: Array<Uint8Array> = [new Uint8Array(base)]; |
| 454 | const offsets = new Map<Oid, number>(); |
| 455 | let at = base; |
| 456 | for (const f of fixtures) { |
| 457 | const head = encodeTypeSize(f.type as 3, f.content.length); |
| 458 | pieces.push(head, f.zdata); |
| 459 | offsets.set(f.oid, at + head.length); |
| 460 | at += head.length + f.zdata.length; |
| 461 | } |
| 462 | return { bytes: concat(pieces), offsets }; |
| 463 | }; |
| 464 | |
| 465 | test("rows staged as pack references read back through the wire key on every path", async () => { |
| 466 | await run( |
| 467 | Effect.gen(function* () { |
| 468 | const sql = makeTestSqlClient(); |
| 469 | const blobs = makeMemoryBlobStore(); |
| 470 | const store = makeObjectStore({ sql, blobs, repoId: REPO }); |
| 471 | const fixtures: Array<Fixture> = []; |
| 472 | for (let i = 0; i < 40; i++) fixtures.push(yield* makeFixture(3, bytes(3000, i + 1))); |
| 473 | const packId = wirePackId("01RECEIVE00000000000000000"); |
| 474 | const { bytes: wire, offsets } = layout(fixtures, 137); // 137 bytes of pkt-line commands first |
| 475 | yield* blobs.put(packKeyOf(REPO, packId), wire); |
| 476 | expect(packKeyOf(REPO, packId)).toContain("/incoming/"); |
| 477 | yield* store.insertStagedBatch( |
| 478 | "push-P", |
| 479 | fixtures.map((f) => ({ |
| 480 | oid: f.oid, |
| 481 | type: f.type, |
| 482 | size: f.content.length, |
| 483 | zdata: f.zdata, |
| 484 | pack: { packId, offset: offsets.get(f.oid)! }, |
| 485 | })), |
| 486 | ); |
| 487 | const staged = yield* sql.all<{ location: string; n: number; withBlob: number }>( |
| 488 | `SELECT location, COUNT(*) AS n, SUM(zdata IS NOT NULL) AS withBlob FROM objects GROUP BY location`, |
| 489 | ); |
| 490 | expect(staged).toEqual([{ location: "pack", n: 40, withBlob: 0 }]); |
| 491 | yield* sql.run(`UPDATE objects SET staged_push = NULL WHERE staged_push = 'push-P'`); |
| 492 | // Single reads, batched reads, and pack emission all resolve the wire key. |
| 493 | const one = yield* store.readContent(fixtures[7]!.oid); |
| 494 | expect(Array.from(one)).toEqual(Array.from(fixtures[7]!.content)); |
| 495 | const many = yield* store.readContentBatch(fixtures.map((f) => f.oid)); |
| 496 | expect(many.size).toBe(40); |
| 497 | const pack = yield* buildPack(store, manifestOf(fixtures, "pack")); |
| 498 | expect(verifyPack(pack).error).toBeUndefined(); |
| 499 | const parsed = yield* parsePack( |
| 500 | pack, |
| 501 | makeObjectStore({ sql: makeTestSqlClient(), blobs: makeMemoryBlobStore(), repoId: "X" }), |
| 502 | ); |
| 503 | expect(parsed.seen.size).toBe(40); |
| 504 | }), |
| 505 | ); |
| 506 | }); |
| 507 | |
| 508 | test("compaction leaves promoted rows alone; geometric merge REWRITES a wire pack into the merged pack", async () => { |
| 509 | await run( |
| 510 | Effect.gen(function* () { |
| 511 | const { sql, blobs, store, fixtures } = yield* seedRepo({ |
| 512 | blobs: 30, |
| 513 | blobSize: 2000, |
| 514 | trees: 3, |
| 515 | }); |
| 516 | const promotedFixtures: Array<Fixture> = []; |
| 517 | for (let i = 0; i < 12; i++) |
| 518 | promotedFixtures.push(yield* makeFixture(3, bytes(2000, i + 500))); |
| 519 | const packId = wirePackId("01RECEIVE00000000000000001"); |
| 520 | // The wire object carries junk around the referenced spans (a real |
| 521 | // one carries trees, commits and delta entries): only spans move. |
| 522 | const { bytes: wire, offsets } = layout(promotedFixtures, 333); |
| 523 | const withJunk = concat([wire, bytes(5000, 9)]); |
| 524 | yield* blobs.put(packKeyOf(REPO, packId), withJunk); |
| 525 | yield* store.insertStagedBatch( |
| 526 | "push-Q", |
| 527 | promotedFixtures.map((f) => ({ |
| 528 | oid: f.oid, |
| 529 | type: f.type, |
| 530 | size: f.content.length, |
| 531 | zdata: f.zdata, |
| 532 | pack: { packId, offset: offsets.get(f.oid)! }, |
| 533 | })), |
| 534 | ); |
| 535 | yield* sql.run(`UPDATE objects SET staged_push = NULL`); |
| 536 | const p1 = yield* runCompactJob({ |
| 537 | repoId: REPO, |
| 538 | sql, |
| 539 | blobs, |
| 540 | maxObjects: 15, |
| 541 | maxBytes: 1 << 30, |
| 542 | }); |
| 543 | const p2 = yield* runCompactJob({ |
| 544 | repoId: REPO, |
| 545 | sql, |
| 546 | blobs, |
| 547 | maxObjects: 15, |
| 548 | maxBytes: 1 << 30, |
| 549 | }); |
| 550 | expect(p1.moved + p2.moved).toBe(30); |
| 551 | const merge = yield* runGeometricMergeJob({ repoId: REPO, sql, blobs }); |
| 552 | expect(merge.packs).toBe(3); |
| 553 | expect(merge.pendingDelete).toContain(packKeyOf(REPO, packId)); |
| 554 | const packs = yield* sql.all<{ pack_id: string; n: number }>( |
| 555 | `SELECT pack_id, COUNT(*) AS n FROM objects WHERE location = 'pack' GROUP BY pack_id`, |
| 556 | ); |
| 557 | expect(packs).toEqual([{ pack_id: merge.packId, n: 42 }]); |
| 558 | // The merged pack is a clean packfile the parser accepts, and every |
| 559 | // object — promoted ones included — reads back through it. |
| 560 | const merged = blobs.objects.get(packKey(REPO, merge.packId!))!; |
| 561 | expect(verifyPack(merged)).toEqual({ objects: 42 }); |
| 562 | const all = yield* store.readContentBatch( |
| 563 | [...fixtures, ...promotedFixtures].map((f) => f.oid), |
| 564 | ); |
| 565 | expect(all.size).toBe(45); |
| 566 | for (const f of promotedFixtures) |
| 567 | expect(Array.from(all.get(f.oid)!)).toEqual(Array.from(f.content)); |
| 568 | }), |
| 569 | ); |
| 570 | }); |
| 571 | }); |
| 572 | |
| 573 | describe("prepared object view", { tags: ["unit", "local"] }, () => { |
| 574 | test("reads only its own staged objects and live objects, with a size cap", () => |
| 575 | run( |
| 576 | Effect.gen(function* () { |
| 577 | const sql = makeTestSqlClient(); |
| 578 | const store = makeObjectStore({ sql, blobs: makeMemoryBlobStore(), repoId: REPO }); |
| 579 | const own = yield* makeFixture(3, new TextEncoder().encode("own")); |
| 580 | const other = yield* makeFixture(3, new TextEncoder().encode("other")); |
| 581 | yield* store.insertStaged("mine", { ...own, size: own.content.length }); |
| 582 | yield* store.insertStaged("theirs", { ...other, size: other.content.length }); |
| 583 | expect(yield* store.getMeta(own.oid)).toBeUndefined(); |
| 584 | expect((yield* store.readPrepared("mine", own.oid, 3))?.content).toEqual(own.content); |
| 585 | expect(yield* store.readPrepared("mine", other.oid, 100)).toBeUndefined(); |
| 586 | const tooBig = yield* Effect.result(store.readPrepared("mine", own.oid, 2)); |
| 587 | expect(tooBig._tag).toBe("Failure"); |
| 588 | }), |
| 589 | )); |
| 590 | }); |