@devries-cameron20 / t3code Mirror · read-only / commits / 67699df3d1
Fork 0

The best way to control Claude Code, Codex and more from any computer or phone

Mirrored from https://github.com/Camo67/t3code.git

Synced Nov 21, 16:00 UTC

perf(azure-devops): PR Code tab loads a page's files in one az run instead of two per file (#35295)

Co-authored-by: Julius Marminge <julius0216@outlook.com>

Kieren-Foenander committed Nov 14, 2026 at 11:28 UTC 67699df3d12eff024173c6533aa2451460ab378c
5 files changed +623 -25
packages/source-control-azure-devops/src/server/AzureDevOpsPullRequestCli.test.ts
+126
@@ -1,3 +1,8 @@
1 +// @effect-diagnostics nodeBuiltinImport:off -- Builds the zip Azure returns and stands in for az writing it.
2 +import * as NodeFS from "node:fs";
3 +import * as NodeZlib from "node:zlib";
4 +
5 +import * as NodeServices from "@effect/platform-node/NodeServices";
6 import { afterEach, assert, expect, it, vi } from "@effect/vitest";
7 import * as Effect from "effect/Effect";
8 import * as Layer from "effect/Layer";
@@ -16,6 +21,7 @@ const layer = it.layer(
21 execute: mockedExecute,
22 }),
23 ),
24 + Layer.provide(NodeServices.layer),
25 ),
26 );
27
@@ -81,6 +87,52 @@ function pullRequests(count: number, firstNumber: number): string {
87 return JSON.stringify(pullRequestRows(count, firstNumber));
88 }
89
90 +/** Entries as Azure streams them: data descriptors, with sizes only in the central directory. */
91 +function streamedZip(
92 + entries: ReadonlyArray<{
93 + readonly name: string;
94 + readonly data: Uint8Array;
95 + readonly stored?: true;
96 + }>,
97 +): Buffer {
98 + const parts: Buffer[] = [];
99 + const directory: Buffer[] = [];
100 + let offset = 0;
101 + for (const entry of entries) {
102 + const name = Buffer.from(entry.name);
103 + const method = entry.stored ? 0 : 8;
104 + const body = entry.stored ? Buffer.from(entry.data) : NodeZlib.deflateRawSync(entry.data);
105 + const local = Buffer.alloc(30);
106 + local.writeUInt32LE(0x04034b50, 0);
107 + local.writeUInt16LE(0x0008, 6);
108 + local.writeUInt16LE(method, 8);
109 + local.writeUInt16LE(name.length, 26);
110 + const descriptor = Buffer.alloc(16);
111 + descriptor.writeUInt32LE(0x08074b50, 0);
112 + descriptor.writeUInt32LE(body.length, 8);
113 + descriptor.writeUInt32LE(entry.data.length, 12);
114 + const central = Buffer.alloc(46);
115 + central.writeUInt32LE(0x02014b50, 0);
116 + central.writeUInt16LE(0x0008, 8);
117 + central.writeUInt16LE(method, 10);
118 + central.writeUInt32LE(body.length, 20);
119 + central.writeUInt32LE(entry.data.length, 24);
120 + central.writeUInt16LE(name.length, 28);
121 + central.writeUInt32LE(offset, 42);
122 + parts.push(local, name, body, descriptor);
123 + directory.push(central, name);
124 + offset += local.length + name.length + body.length + descriptor.length;
125 + }
126 + const centralDirectory = Buffer.concat(directory);
127 + const end = Buffer.alloc(22);
128 + end.writeUInt32LE(0x06054b50, 0);
129 + end.writeUInt16LE(entries.length, 8);
130 + end.writeUInt16LE(entries.length, 10);
131 + end.writeUInt32LE(centralDirectory.length, 12);
132 + end.writeUInt32LE(offset, 16);
133 + return Buffer.concat([...parts, centralDirectory, end]);
134 +}
135 +
136 function argsOfCall(index: number): ReadonlyArray<string> {
137 const call = mockedExecute.mock.calls[index];
138 assert.isDefined(call);
@@ -850,6 +902,80 @@ layer("AzureDevOpsPullRequestCli.layer", (it) => {
902 }),
903 );
904
905 + it.effect(
906 + "reads a batch of files from one blobs zip, keeping only what Azure's read would say",
907 + () =>
908 + Effect.gen(function* () {
909 + const id = (digit: string) => digit.repeat(40);
910 + const utf8 = (text: string) => new TextEncoder().encode(text);
911 + let sent = "";
912 + mockedExecute.mockImplementationOnce((input) =>
913 + Effect.sync(() => {
914 + const flag = (name: string) => input.args[input.args.indexOf(name) + 1] ?? "";
915 + sent = NodeFS.readFileSync(flag("--in-file"), "utf8");
916 + NodeFS.writeFileSync(
917 + flag("--out-file"),
918 + streamedZip([
919 + { name: id("a"), data: utf8("plain text\n") },
920 + { name: id("b"), data: utf8("\uFEFFcafé ✓\n") },
921 + { name: id("c"), data: Uint8Array.of(0x89, 0x50, 0x00, 0x01) },
922 + // Windows-1252 "café" and UTF-16: Azure decodes these itself.
923 + { name: id("d"), data: Uint8Array.of(0x63, 0x61, 0x66, 0xe9) },
924 + { name: id("e"), data: Uint8Array.of(0xff, 0xfe, 0x68, 0x00) },
925 + { name: id("f"), data: utf8("stored\n"), stored: true },
926 + { name: id("9"), data: utf8("never asked for\n") },
927 + ]),
928 + );
929 + return output("");
930 + }),
931 + );
932 + const cli = yield* AzureDevOpsPullRequestCli.AzureDevOpsPullRequestCli;
933 +
934 + const blobs = yield* cli.readBlobs({
935 + cwd: "/w",
936 + location: { project: "platform", repository: "web" },
937 + objectIds: [id("A"), id("b"), id("C"), id("d"), id("e"), id("f"), id("a"), "../etc"],
938 + });
939 +
940 + expect(JSON.parse(sent)).toEqual([id("A"), id("b"), id("C"), id("d"), id("e"), id("f")]);
941 + expect(argsOfCall(0)).toEqual(
942 + expect.arrayContaining(["blobs", "POST", "application/zip", "project=platform"]),
943 + );
944 + expect(Object.fromEntries(blobs)).toEqual({
945 + [id("a")]: { contents: "plain text\n", isBinary: false },
946 + [id("b")]: { contents: "café ✓\n", isBinary: false },
947 + [id("c")]: { contents: "", isBinary: true },
948 + [id("f")]: { contents: "stored\n", isBinary: false },
949 + });
950 + }),
951 + );
952 +
953 + it.effect("leaves blobs past the batch's total size to per-file reads", () =>
954 + Effect.gen(function* () {
955 + const ids = ["1", "2", "3", "4", "5", "6"].map((digit) => digit.repeat(40));
956 + // Compresses to a few KiB each, so only the inflated total can bound memory.
957 + const blob = new Uint8Array(3 * 1024 * 1024).fill(0x61);
958 + mockedExecute.mockImplementationOnce((input) =>
959 + Effect.sync(() => {
960 + NodeFS.writeFileSync(
961 + input.args[input.args.indexOf("--out-file") + 1] ?? "",
962 + streamedZip(ids.map((name) => ({ name, data: blob }))),
963 + );
964 + return output("");
965 + }),
966 + );
967 + const cli = yield* AzureDevOpsPullRequestCli.AzureDevOpsPullRequestCli;
968 +
969 + const blobs = yield* cli.readBlobs({
970 + cwd: "/w",
971 + location: { project: "platform", repository: "web" },
972 + objectIds: ids,
973 + });
974 +
975 + expect([...blobs.keys()]).toEqual(ids.slice(0, 5));
976 + }),
977 + );
978 +
979 it.effect("escapes characters in a file path that would cut the query string", () =>
980 Effect.gen(function* () {
981 mockedExecute.mockReturnValueOnce(Effect.succeed(output(json({ content: "x" }))));
packages/source-control-azure-devops/src/server/AzureDevOpsPullRequestCli.ts
+95
@@ -1,5 +1,6 @@
1 import * as Context from "effect/Context";
2 import * as Effect from "effect/Effect";
3 +import * as FileSystem from "effect/FileSystem";
4 import * as Layer from "effect/Layer";
5 import * as Result from "effect/Result";
6 import * as Schema from "effect/Schema";
@@ -12,6 +13,7 @@ import type {
13 } from "@t3tools/contracts";
14
15 import * as AzureDevOpsCli from "./AzureDevOpsCli.ts";
16 +import { blobContent, readBlobZip } from "./azureDevOpsBlobZip.ts";
17 import {
18 decodeItemContentJson,
19 decodeIterationChangesJson,
@@ -108,6 +110,13 @@ const PULL_REQUEST_LIST_MAX_OUTPUT_BYTES = 16 * 1024 * 1024;
110 const CHANGE_ENTRIES_MAX_OUTPUT_BYTES = 8 * 1024 * 1024;
111 /** JSON escaping and base64 can exceed the decoded file's size. */
112 const ITEM_CONTENT_MAX_OUTPUT_BYTES = 4 * 1024 * 1024;
113 +/** A larger batch is left to per-file reads rather than held in memory at once. */
114 +const BLOB_ARCHIVE_MAX_BYTES = 64 * 1024 * 1024;
115 +/** Matches what a per-file read could carry. */
116 +const BLOB_MAX_BYTES = ITEM_CONTENT_MAX_OUTPUT_BYTES;
117 +/** Blobs past this, together, are left to per-file reads, which hold a few at a time. */
118 +const BLOB_BATCH_MAX_BYTES = 16 * 1024 * 1024;
119 +const BLOB_ID = /^[0-9a-f]{40}$/i;
120 /** Review history is unpaged and independent of diff size. */
121 const REVIEW_HISTORY_MAX_OUTPUT_BYTES = 8 * 1024 * 1024;
122
@@ -184,6 +193,19 @@ export class AzureDevOpsPullRequestCli extends Context.Service<
193 readonly commit: string;
194 }) => Effect.Effect<AzureDevOpsItemContent, AzureDevOpsPullRequestCliError>;
195
196 + /**
197 + * Many files in one `az` run, by blob id, keyed by lowercase id. A blob only Azure's item read
198 + * decodes faithfully, or one too large to carry, is left out.
199 + */
200 + readonly readBlobs: (input: {
201 + readonly cwd: string;
202 + readonly location: AzureDevOpsRepositoryLocation;
203 + readonly objectIds: ReadonlyArray<string>;
204 + }) => Effect.Effect<
205 + ReadonlyMap<string, AzureDevOpsItemContent>,
206 + AzureDevOpsPullRequestCliError
207 + >;
208 +
209 readonly runPullRequestAction: (input: {
210 readonly cwd: string;
211 readonly number: number;
@@ -276,6 +298,7 @@ function isReviewerName(value: string): boolean {
298 /** @public */
299 export const make = Effect.gen(function* () {
300 const azure = yield* AzureDevOpsCli.AzureDevOpsCli;
301 + const fileSystem = yield* FileSystem.FileSystem;
302
303 // Let `az` parse all three remote formats.
304 const detectArgs = ["--detect", "true"] as const;
@@ -596,6 +619,78 @@ export const make = Effect.gen(function* () {
619 },
620 ),
621
622 + readBlobs: Effect.fn("AzureDevOpsPullRequestCli.readBlobs")(function* (input) {
623 + const requested = new Map<string, string>();
624 + // Azure states ids in uppercase and names entries in lowercase.
625 + for (const id of input.objectIds) {
626 + const key = id.toLowerCase();
627 + if (BLOB_ID.test(id) && !requested.has(key)) requested.set(key, id);
628 + }
629 + if (requested.size === 0) return new Map<string, AzureDevOpsItemContent>();
630 + const readFailure = (cause: unknown) =>
631 + new AzureDevOpsPullRequestReadError({
632 + command: "az",
633 + cwd: input.cwd,
634 + operation: "readBlobs",
635 + cause,
636 + });
637 + return yield* Effect.scoped(
638 + Effect.gen(function* () {
639 + const directory = yield* fileSystem
640 + .makeTempDirectoryScoped({ prefix: "t3-azure-blobs-" })
641 + .pipe(Effect.mapError(readFailure));
642 + const request = `${directory}/ids.json`;
643 + // invoke refuses to overwrite its output, so the archive gets a name nothing holds yet.
644 + const archive = `${directory}/blobs.zip`;
645 + yield* fileSystem
646 + .writeFileString(
647 + request,
648 + `[${[...requested.values()].map((id) => `"${id}"`).join(",")}]`,
649 + )
650 + .pipe(Effect.mapError(readFailure));
651 + yield* azure.execute({
652 + cwd: input.cwd,
653 + args: [
654 + "devops",
655 + "invoke",
656 + ...detectArgs,
657 + "--area",
658 + "git",
659 + "--resource",
660 + "blobs",
661 + "--api-version",
662 + REST_API_VERSION,
663 + "--route-parameters",
664 + ...repositoryRoute(input.location),
665 + "--http-method",
666 + "POST",
667 + "--in-file",
668 + request,
669 + "--accept-media-type",
670 + "application/zip",
671 + "--out-file",
672 + archive,
673 + "--only-show-errors",
674 + ],
675 + });
676 + const size = yield* fileSystem.stat(archive).pipe(
677 + Effect.map((info) => Number(info.size)),
678 + Effect.mapError(readFailure),
679 + );
680 + const contents = new Map<string, AzureDevOpsItemContent>();
681 + if (size > BLOB_ARCHIVE_MAX_BYTES) return contents;
682 + const zip = yield* fileSystem.readFile(archive).pipe(Effect.mapError(readFailure));
683 + const entries = readBlobZip(zip, BLOB_MAX_BYTES, BLOB_BATCH_MAX_BYTES);
684 + if (entries === null) return yield* readFailure("Azure returned an unreadable archive.");
685 + for (const [id, bytes] of entries) {
686 + const content = requested.has(id) ? blobContent(bytes) : null;
687 + if (content !== null) contents.set(id, content);
688 + }
689 + return contents;
690 + }),
691 + );
692 + }),
693 +
694 readItemContent: Effect.fn("AzureDevOpsPullRequestCli.readItemContent")(function (input) {
695 return invoke({
696 cwd: input.cwd,
packages/source-control-azure-devops/src/server/AzureDevOpsPullRequestProvider.test.ts
+180
@@ -4,6 +4,7 @@ import * as Layer from "effect/Layer";
4
5 import * as AzureDevOpsPullRequestCli from "./AzureDevOpsPullRequestCli.ts";
6 import {
7 + BLOB_BATCH_FILES,
8 LOCATION_CACHE_CAPACITY,
9 make,
10 MAX_DIFF_SPAWNS,
@@ -46,6 +47,9 @@ function change(
47 return { path, oldPath: path, changeKind, objectId: "8f80", originalObjectId: "0ca4" };
48 }
49
50 +/** A batch that returns nothing, so every file takes the per-file read. */
51 +const NO_BLOBS = { readBlobs: () => Effect.succeed(new Map()) };
52 +
53 function side(prefix: string, lines: number, width: number): string {
54 const pad = "z".repeat(width);
55 return `${Array.from({ length: lines }, (_, line) => `${prefix} ${line} ${pad}`).join("\n")}\n`;
@@ -69,6 +73,7 @@ const readSlice = (input: {
73 const provider = yield* make.pipe(
74 Effect.provide(
75 Layer.mock(AzureDevOpsPullRequestCli.AzureDevOpsPullRequestCli)({
76 + ...NO_BLOBS,
77 getPullRequest: () => Effect.succeed(PULL_REQUEST),
78 listIterations: () => Effect.succeed([ITERATION]),
79 listIterationChanges: () =>
@@ -199,6 +204,7 @@ describe("getDiff reads", () => {
204 const provider = yield* make.pipe(
205 Effect.provide(
206 Layer.mock(AzureDevOpsPullRequestCli.AzureDevOpsPullRequestCli)({
207 + ...NO_BLOBS,
208 getPullRequest: () => Effect.succeed(PULL_REQUEST),
209 listIterations: () => Effect.succeed([ITERATION]),
210 listIterationChanges: () =>
@@ -226,6 +232,179 @@ describe("getDiff reads", () => {
232 }),
233 );
234
235 + it.effect("reads a slice's files in one blob batch, and per file only what it missed", () =>
236 + Effect.gen(function* () {
237 + const paths = ["a.ts", "b.ts", "c.ts", "gone.ts", "added.ts"];
238 + const changes = [
239 + ...["a.ts", "b.ts", "c.ts"].map((path) => ({
240 + ...change(path),
241 + objectId: `${path.toUpperCase()}-NEW`,
242 + originalObjectId: `${path.toUpperCase()}-OLD`,
243 + })),
244 + { ...change("gone.ts", "deleted"), objectId: null, originalObjectId: "GONE-OLD" },
245 + { ...change("added.ts", "new"), objectId: "ADDED-NEW", originalObjectId: null },
246 + ];
247 + const batches: ReadonlyArray<string>[] = [];
248 + const reads: string[] = [];
249 +
250 + const provider = yield* make.pipe(
251 + Effect.provide(
252 + Layer.mock(AzureDevOpsPullRequestCli.AzureDevOpsPullRequestCli)({
253 + getPullRequest: () => Effect.succeed(PULL_REQUEST),
254 + listIterations: () => Effect.succeed([ITERATION]),
255 + listIterationChanges: () => Effect.succeed({ changes, truncated: false }),
256 + readBlobs: (input) =>
257 + Effect.sync(() => {
258 + batches.push(input.objectIds);
259 + // Azure left one side of b.ts out, as it does for a file only it can decode.
260 + return new Map(
261 + input.objectIds
262 + .filter((id) => id !== "B.TS-OLD")
263 + .map((id) => [
264 + id.toLowerCase(),
265 + {
266 + contents: side(id.endsWith("NEW") ? "new" : "old", 2, 4),
267 + isBinary: false,
268 + },
269 + ]),
270 + );
271 + }),
272 + readItemContent: (item) =>
273 + Effect.sync(() => {
274 + reads.push(`${item.commit}:${item.path}`);
275 + return {
276 + contents: side(item.commit === ITERATION.headCommit ? "new" : "old", 2, 4),
277 + isBinary: false,
278 + };
279 + }),
280 + }),
281 + ),
282 + );
283 +
284 + const slice = yield* provider.getDiff({
285 + cwd: "/w",
286 + repository: "acme/web",
287 + host: "dev.azure.com",
288 + number: 7,
289 + });
290 +
291 + expect(patchedPaths(slice.patch)).toEqual(paths);
292 + expect(slice.patch.match(/^@@ /gmu)).toHaveLength(paths.length);
293 + expect(batches).toEqual([
294 + [
295 + "A.TS-OLD",
296 + "A.TS-NEW",
297 + "B.TS-OLD",
298 + "B.TS-NEW",
299 + "C.TS-OLD",
300 + "C.TS-NEW",
301 + "GONE-OLD",
302 + "ADDED-NEW",
303 + ],
304 + ]);
305 + expect(reads.toSorted()).toEqual(["base:b.ts", "head:b.ts"]);
306 + }),
307 + );
308 +
309 + it.effect("starts a new blob batch where the last one's files run out", () =>
310 + Effect.gen(function* () {
311 + const paths = Array.from({ length: BLOB_BATCH_FILES + 3 }, (_, at) => `f${at}.ts`);
312 + // Added lines that bring the slice near its edit budget by file 96, so the read batches
313 + // there narrow to 3 and the one from file 99 would reach past the blob batch.
314 + const added = (at: number) => (at < 60 ? 41 : at < 96 ? 40 : at < 99 ? 0 : 10);
315 + const batchSizes: number[] = [];
316 + let fileReads = 0;
317 +
318 + const provider = yield* make.pipe(
319 + Effect.provide(
320 + Layer.mock(AzureDevOpsPullRequestCli.AzureDevOpsPullRequestCli)({
321 + getPullRequest: () => Effect.succeed(PULL_REQUEST),
322 + listIterations: () => Effect.succeed([ITERATION]),
323 + listIterationChanges: () =>
324 + Effect.succeed({
325 + changes: paths.map((path) => ({
326 + ...change(path),
327 + objectId: `${path}-new`,
328 + originalObjectId: `${path}-old`,
329 + })),
330 + truncated: false,
331 + }),
332 + readBlobs: (input) =>
333 + Effect.sync(() => {
334 + batchSizes.push(input.objectIds.length);
335 + return new Map(
336 + input.objectIds.map((id) => {
337 + const at = Number(id.slice(1, id.indexOf(".")));
338 + const lines = id.endsWith("-new") ? added(at) : 0;
339 + return [
340 + id,
341 + { contents: lines === 0 ? "" : side("+", lines, 1), isBinary: false },
342 + ];
343 + }),
344 + );
345 + }),
346 + readItemContent: () =>
347 + Effect.sync(() => ((fileReads += 1), { contents: "", isBinary: false })),
348 + }),
349 + ),
350 + );
351 +
352 + const slice = yield* provider.getDiff({
353 + cwd: "/w",
354 + repository: "acme/web",
355 + host: "dev.azure.com",
356 + number: 7,
357 + });
358 +
359 + expect(patchedPaths(slice.patch)).toHaveLength(paths.length);
360 + expect(batchSizes).toEqual([2 * BLOB_BATCH_FILES, 6]);
361 + expect(fileReads).toBe(0);
362 + }),
363 + );
364 +
365 + it.effect("serves a later slice from the iteration and change list the first one read", () =>
366 + Effect.gen(function* () {
367 + const paths = ["a.ts", "b.ts", "c.ts", "d.ts", "e.ts", "f.ts"];
368 + let iterationReads = 0;
369 + let changeReads = 0;
370 +
371 + const provider = yield* make.pipe(
372 + Effect.provide(
373 + Layer.mock(AzureDevOpsPullRequestCli.AzureDevOpsPullRequestCli)({
374 + ...NO_BLOBS,
375 + getPullRequest: () => Effect.succeed(PULL_REQUEST),
376 + listIterations: () => Effect.sync(() => ((iterationReads += 1), [ITERATION])),
377 + listIterationChanges: () =>
378 + Effect.sync(() => {
379 + changeReads += 1;
380 + return { changes: paths.map((path) => change(path)), truncated: false };
381 + }),
382 + readItemContent: (item) =>
383 + Effect.succeed({
384 + contents: side(item.commit === ITERATION.headCommit ? "new" : "old", 100, 900),
385 + isBinary: false,
386 + }),
387 + }),
388 + ),
389 + );
390 + const readDiff = (cursor?: string) =>
391 + provider.getDiff({
392 + cwd: "/w",
393 + repository: "acme/web",
394 + host: "dev.azure.com",
395 + number: 7,
396 + ...(cursor === undefined ? {} : { cursor }),
397 + });
398 +
399 + const first = yield* readDiff();
400 + const second = yield* readDiff(first.nextCursor ?? undefined);
401 +
402 + expect(patchedPaths(second.patch)).toEqual(["c.ts", "d.ts"]);
403 + expect(iterationReads).toBe(1);
404 + expect(changeReads).toBe(1);
405 + }),
406 + );
407 +
408 it.effect("leaves a run of files it could not diff at all for the next slice", () =>
409 Effect.gen(function* () {
410 const paths = Array.from({ length: MAX_DIFF_SLICE_FILES + 20 }, (_, at) => `gen/a${at}.bin`);
@@ -321,6 +500,7 @@ describe("what one diff slice spends", () => {
500 const provider = yield* make.pipe(
501 Effect.provide(
502 Layer.mock(AzureDevOpsPullRequestCli.AzureDevOpsPullRequestCli)({
503 + ...NO_BLOBS,
504 getPullRequest: (input) =>
505 Effect.sync(() => {
506 readsOf.set(input.number, (readsOf.get(input.number) ?? 0) + 1);
packages/source-control-azure-devops/src/server/AzureDevOpsPullRequestProvider.ts
+126 -25
@@ -35,7 +35,7 @@ import type {
35 } from "./azureDevOpsPullRequestJson.ts";
36 import { definition } from "../shared/definition.ts";
37
38 -/** Each file pays for two Python startups. */
38 +/** A file the blob batch missed pays for two Python startups. */
39 const DIFF_FILE_CONCURRENCY = 4;
40
41 /** Readers share a spawn budget across requests. */
@@ -43,6 +43,42 @@ export const MAX_DIFF_SPAWNS = 2 * DIFF_FILE_CONCURRENCY;
43
44 export const LOCATION_CACHE_CAPACITY = 128;
45
46 +/** A version's change list can run to thousands of paths. */
47 +const CHANGES_CACHE_CAPACITY = 32;
48 +
49 +/**
50 + * Files whose contents one `az` run fetches. The run costs about the same for 10 blobs as for
51 + * 100, and a slice rarely spends this many before its budgets close it.
52 + */
53 +export const BLOB_BATCH_FILES = 100;
54 +
55 +/** Least recently used goes first. */
56 +function remember<V>(cache: Map<string, V>, capacity: number, key: string, value: V): V {
57 + cache.delete(key);
58 + if (cache.size >= capacity) {
59 + const oldest = cache.keys().next().value;
60 + if (oldest !== undefined) cache.delete(oldest);
61 + }
62 + cache.set(key, value);
63 + return value;
64 +}
65 +
66 +function recall<V>(cache: Map<string, V>, key: string): V | undefined {
67 + const held = cache.get(key);
68 + if (held !== undefined) remember(cache, Number.POSITIVE_INFINITY, key, held);
69 + return held;
70 +}
71 +
72 +/** The blobs a file's two sides need, as the change list names them. */
73 +function blobIdsOf(change: AzureDevOpsChangeEntry): ReadonlyArray<string> {
74 + const ids: string[] = [];
75 + if (change.changeKind !== "new" && change.originalObjectId !== null) {
76 + ids.push(change.originalObjectId);
77 + }
78 + if (change.changeKind !== "deleted" && change.objectId !== null) ids.push(change.objectId);
79 + return ids;
80 +}
81 +
82 const CAPABILITIES: PullRequestCapabilities = {
83 diff: true,
84 comment: false,
@@ -137,6 +173,8 @@ export const make = Effect.gen(function* () {
173 const diffSpawns = yield* Semaphore.make(MAX_DIFF_SPAWNS);
174 const readItemContent = (input: Parameters<typeof cli.readItemContent>[0]) =>
175 diffSpawns.withPermits(1)(cli.readItemContent(input));
176 + const readBlobs = (input: Parameters<typeof cli.readBlobs>[0]) =>
177 + diffSpawns.withPermits(1)(cli.readBlobs(input));
178
179 const fail =
180 (operation: string) => (error: AzureDevOpsPullRequestCli.AzureDevOpsPullRequestCliError) =>
@@ -162,43 +200,67 @@ export const make = Effect.gen(function* () {
200
201 // PR locations are immutable; caching avoids full PR reads for viewed-file checks.
202 const locations = new Map<string, AzureDevOpsRepositoryLocation>();
203 + // A pushed iteration never changes, so neither do its commits nor its change list.
204 + const knownIterations = new Map<string, ReadonlyArray<AzureDevOpsIteration>>();
205 + const iterationChanges = new Map<string, AzureDevOpsPullRequestCli.AzureDevOpsIterationChanges>();
206
207 const locationOf = (input: { readonly cwd: string; readonly number: number }) => {
208 const key = `${input.cwd} ${input.number}`;
168 - const held = locations.get(key);
169 - if (held !== undefined) {
170 - locations.delete(key);
171 - locations.set(key, held);
172 - return Effect.succeed(held);
173 - }
174 - return cli.getPullRequest({ cwd: input.cwd, number: input.number }).pipe(
175 - Effect.map((pullRequest) => {
176 - const location = pullRequest.location;
177 - if (location === null) return null;
178 - if (locations.size >= LOCATION_CACHE_CAPACITY) {
179 - const oldest = locations.keys().next().value;
180 - if (oldest !== undefined) locations.delete(oldest);
181 - }
182 - locations.set(key, location);
183 - return location;
184 - }),
185 - );
209 + const held = recall(locations, key);
210 + if (held !== undefined) return Effect.succeed(held);
211 + return cli
212 + .getPullRequest({ cwd: input.cwd, number: input.number })
213 + .pipe(
214 + Effect.map((pullRequest) =>
215 + pullRequest.location === null
216 + ? null
217 + : remember(locations, LOCATION_CACHE_CAPACITY, key, pullRequest.location),
218 + ),
219 + );
220 };
221
222 + /** A later slice names its iteration, which the list it was cut from already holds. */
223 const diffScope = Effect.fn("AzureDevOpsPullRequestProvider.diffScope")(function* (input: {
224 readonly cwd: string;
225 readonly number: number;
226 + readonly iterationId?: number;
227 }) {
228 const location = yield* locationOf(input);
229 if (location === null) return null;
230 + const key = `${input.cwd} ${input.number}`;
231 + const held = recall(knownIterations, key);
232 + if (
233 + input.iterationId !== undefined &&
234 + held?.some((iteration) => iteration.id === input.iterationId)
235 + ) {
236 + return { location, iterations: held };
237 + }
238 const iterations = yield* cli.listIterations({
239 cwd: input.cwd,
240 location,
241 number: input.number,
242 });
243 + remember(knownIterations, LOCATION_CACHE_CAPACITY, key, iterations);
244 return { location, iterations };
245 });
246
247 + const changesOf = (input: {
248 + readonly cwd: string;
249 + readonly location: AzureDevOpsRepositoryLocation;
250 + readonly number: number;
251 + readonly iterationId: number;
252 + }) => {
253 + const key = `${input.cwd} ${input.number} ${input.iterationId}`;
254 + const held = recall(iterationChanges, key);
255 + return held !== undefined
256 + ? Effect.succeed(held)
257 + : cli
258 + .listIterationChanges(input)
259 + .pipe(
260 + Effect.map((listed) => remember(iterationChanges, CHANGES_CACHE_CAPACITY, key, listed)),
261 + );
262 + };
263 +
264 const EMPTY_ITEM: AzureDevOpsItemContent = { contents: "", isBinary: false };
265
266 /** Azure fails reads of absent files. */
@@ -249,7 +311,7 @@ export const make = Effect.gen(function* () {
311 changes: [],
312 truncated: false,
313 } as AzureDevOpsPullRequestCli.AzureDevOpsIterationChanges)
252 - : cli.listIterationChanges({
314 + : changesOf({
315 cwd: input.cwd,
316 location: input.location,
317 number: input.number,
@@ -391,15 +453,19 @@ export const make = Effect.gen(function* () {
453 // the Code tab has nothing to scope itself to and always asks for the whole change.
454 getDiff: Effect.fn("AzureDevOpsPullRequestProvider.getDiff")(
455 function* (input) {
394 - const scope = yield* diffScope(input);
395 - if (scope === null) return EMPTY_DIFF_SLICE;
456 const cursor = parseAzureDevOpsDiffCursor(input.cursor);
457 + const scope = yield* diffScope({
458 + cwd: input.cwd,
459 + number: input.number,
460 + ...(cursor === null ? {} : { iterationId: cursor.iterationId }),
461 + });
462 + if (scope === null) return EMPTY_DIFF_SLICE;
463 const iteration =
464 cursor === null
465 ? scope.iterations.at(-1)
466 : scope.iterations.find((candidate) => candidate.id === cursor.iterationId);
467 if (iteration === undefined) return EMPTY_DIFF_SLICE;
402 - const listed = yield* cli.listIterationChanges({
468 + const listed = yield* changesOf({
469 cwd: input.cwd,
470 location: scope.location,
471 number: input.number,
@@ -428,12 +494,47 @@ export const make = Effect.gen(function* () {
494 ),
495 );
496 };
497 + let blobs: ReadonlyMap<string, AzureDevOpsItemContent> = new Map();
498 + let blobsUntil = index;
499 + const fromBlobs = (change: AzureDevOpsChangeEntry): AzureDevOpsFileTexts | null => {
500 + const side = (id: string | null) =>
501 + id === null ? undefined : blobs.get(id.toLowerCase());
502 + const oldItem = change.changeKind === "new" ? EMPTY_ITEM : side(change.originalObjectId);
503 + const newItem = change.changeKind === "deleted" ? EMPTY_ITEM : side(change.objectId);
504 + return oldItem === undefined || newItem === undefined
505 + ? null
506 + : {
507 + oldContents: oldItem.contents,
508 + newContents: newItem.contents,
509 + binary: oldItem.isBinary || newItem.isBinary,
510 + };
511 + };
512 while (!full && index < changes.length) {
432 - const batch = changes.slice(index, index + batchWidth());
513 + if (index >= blobsUntil) {
514 + blobsUntil = index + BLOB_BATCH_FILES;
515 + blobs = yield* readBlobs({
516 + cwd: input.cwd,
517 + location: scope.location,
518 + objectIds: changes.slice(index, blobsUntil).flatMap(blobIdsOf),
519 + }).pipe(
520 + // Per-file reads then answer, or fail, for each file.
521 + Effect.catchTags({
522 + AzureDevOpsPullRequestNotFoundError: () => Effect.succeed(new Map()),
523 + AzureDevOpsCommandFailedError: () => Effect.succeed(new Map()),
524 + AzureDevOpsPullRequestReadError: () => Effect.succeed(new Map()),
525 + }),
526 + );
527 + }
528 + const batch = changes.slice(index, Math.min(index + batchWidth(), blobsUntil));
529 const read = yield* Effect.forEach(
530 batch,
531 (change) =>
436 - readTexts({ cwd: input.cwd, location: scope.location, iteration, change }).pipe(
532 + Effect.suspend(() => {
533 + const held = fromBlobs(change);
534 + return held === null
535 + ? readTexts({ cwd: input.cwd, location: scope.location, iteration, change })
536 + : Effect.succeed(held);
537 + }).pipe(
538 // Auth, rate-limit and CLI failures must reach the caller.
539 Effect.catchTags({
540 AzureDevOpsPullRequestNotFoundError: () => Effect.succeed(null),
packages/source-control-azure-devops/src/server/azureDevOpsBlobZip.ts new
+96
@@ -0,0 +1,96 @@
1 +// @effect-diagnostics nodeBuiltinImport:off
2 +// Effect has no inflate, and Azure's blob batch only comes as a deflate zip.
3 +import * as NodeZlib from "node:zlib";
4 +
5 +import type { AzureDevOpsItemContent } from "./azureDevOpsPullRequestJson.ts";
6 +
7 +const END_OF_CENTRAL_DIRECTORY = 0x06054b50;
8 +const CENTRAL_DIRECTORY_ENTRY = 0x02014b50;
9 +const LOCAL_FILE_HEADER = 0x04034b50;
10 +const STORED = 0;
11 +const DEFLATED = 8;
12 +const ZIP64_MARKER = 0xffffffff;
13 +
14 +const entryName = new TextDecoder();
15 +
16 +/**
17 + * The entries of the zip Azure's Blobs API answers with, keyed by lowercase blob id. Azure streams
18 + * entries with data descriptors, so sizes come from the central directory. Null for an archive
19 + * this cannot read; an entry over `maxEntryBytes`, past `maxTotalBytes` in all, or in another method
20 + * is left out.
21 + */
22 +export function readBlobZip(
23 + zip: Uint8Array,
24 + maxEntryBytes: number,
25 + maxTotalBytes: number,
26 +): ReadonlyMap<string, Uint8Array> | null {
27 + const view = new DataView(zip.buffer, zip.byteOffset, zip.byteLength);
28 + // The end record is 22 bytes followed by a comment of at most 65,535.
29 + let end = -1;
30 + for (let at = zip.length - 22; at >= 0 && at >= zip.length - 22 - 0xffff; at -= 1) {
31 + if (view.getUint32(at, true) === END_OF_CENTRAL_DIRECTORY) {
32 + end = at;
33 + break;
34 + }
35 + }
36 + if (end === -1) return null;
37 + const count = view.getUint16(end + 10, true);
38 + let at = view.getUint32(end + 16, true);
39 + if (count === 0xffff || at === ZIP64_MARKER) return null;
40 +
41 + const entries = new Map<string, Uint8Array>();
42 + let total = 0;
43 + for (let entry = 0; entry < count; entry += 1) {
44 + if (at + 46 > zip.length || view.getUint32(at, true) !== CENTRAL_DIRECTORY_ENTRY) return null;
45 + const method = view.getUint16(at + 10, true);
46 + const compressedSize = view.getUint32(at + 20, true);
47 + const size = view.getUint32(at + 24, true);
48 + const nameLength = view.getUint16(at + 28, true);
49 + const local = view.getUint32(at + 42, true);
50 + const name = entryName.decode(zip.subarray(at + 46, at + 46 + nameLength)).toLowerCase();
51 + at += 46 + nameLength + view.getUint16(at + 30, true) + view.getUint16(at + 32, true);
52 + if (compressedSize === ZIP64_MARKER || size === ZIP64_MARKER || local === ZIP64_MARKER) {
53 + return null;
54 + }
55 + if (local + 30 > zip.length || view.getUint32(local, true) !== LOCAL_FILE_HEADER) return null;
56 + const start = local + 30 + view.getUint16(local + 26, true) + view.getUint16(local + 28, true);
57 + if (start + compressedSize > zip.length) return null;
58 + if (size > maxEntryBytes || total + size > maxTotalBytes) continue;
59 + const data = zip.subarray(start, start + compressedSize);
60 + if (method === STORED) {
61 + if (compressedSize !== size) return null;
62 + entries.set(name, data);
63 + total += size;
64 + } else if (method === DEFLATED) {
65 + try {
66 + const inflated = NodeZlib.inflateRawSync(data, { maxOutputLength: maxEntryBytes });
67 + if (inflated.length !== size) return null;
68 + entries.set(name, inflated);
69 + total += size;
70 + } catch {
71 + return null;
72 + }
73 + }
74 + }
75 + return entries;
76 +}
77 +
78 +/** Strips a UTF-8 BOM, as Azure's `content` does. */
79 +const strictUtf8 = new TextDecoder("utf-8", { fatal: true });
80 +
81 +/**
82 + * A blob's text as Azure's item read would give it, or null where Azure decodes the bytes itself
83 + * (UTF-16, or text that is not UTF-8 such as Windows-1252) and only that read is faithful.
84 + */
85 +export function blobContent(bytes: Uint8Array): AzureDevOpsItemContent | null {
86 + const utf16 =
87 + (bytes[0] === 0xff && bytes[1] === 0xfe) || (bytes[0] === 0xfe && bytes[1] === 0xff);
88 + if (utf16) return null;
89 + // Azure flags a binary on the same evidence.
90 + if (bytes.includes(0)) return { contents: "", isBinary: true };
91 + try {
92 + return { contents: strictUtf8.decode(bytes), isBinary: false };
93 + } catch {
94 + return null;
95 + }
96 +}