The best way to control Claude Code, Codex and more from any computer or phone
Mirrored from https://github.com/Camo67/t3code.git
Synced Nov 21, 16:00 UTC
perf(azure-devops): PR Code tab loads a page's files in one az run instead of two per file (#35295)
Co-authored-by: Julius Marminge <julius0216@outlook.com>
Kieren-Foenander committed
Nov 14, 2026 at 11:28 UTC
67699df3d12eff024173c6533aa2451460ab378c
5 files changed
+623
-25
packages/source-control-azure-devops/src/server/AzureDevOpsPullRequestCli.test.ts
+126
@@ -1,3 +1,8 @@
1
+// @effect-diagnostics nodeBuiltinImport:off -- Builds the zip Azure returns and stands in for az writing it.
2
+import * as NodeFS from "node:fs";
3
+import * as NodeZlib from "node:zlib";
4
+
5
+import * as NodeServices from "@effect/platform-node/NodeServices";
6
import { afterEach, assert, expect, it, vi } from "@effect/vitest";
7
import * as Effect from "effect/Effect";
8
import * as Layer from "effect/Layer";
@@ -16,6 +21,7 @@ const layer = it.layer(
21
execute: mockedExecute,
22
}),
23
),
24
+ Layer.provide(NodeServices.layer),
25
),
26
);
27
@@ -81,6 +87,52 @@ function pullRequests(count: number, firstNumber: number): string {
87
return JSON.stringify(pullRequestRows(count, firstNumber));
88
}
89
90
+/** Entries as Azure streams them: data descriptors, with sizes only in the central directory. */
91
+function streamedZip(
92
+ entries: ReadonlyArray<{
93
+ readonly name: string;
94
+ readonly data: Uint8Array;
95
+ readonly stored?: true;
96
+ }>,
97
+): Buffer {
98
+ const parts: Buffer[] = [];
99
+ const directory: Buffer[] = [];
100
+ let offset = 0;
101
+ for (const entry of entries) {
102
+ const name = Buffer.from(entry.name);
103
+ const method = entry.stored ? 0 : 8;
104
+ const body = entry.stored ? Buffer.from(entry.data) : NodeZlib.deflateRawSync(entry.data);
105
+ const local = Buffer.alloc(30);
106
+ local.writeUInt32LE(0x04034b50, 0);
107
+ local.writeUInt16LE(0x0008, 6);
108
+ local.writeUInt16LE(method, 8);
109
+ local.writeUInt16LE(name.length, 26);
110
+ const descriptor = Buffer.alloc(16);
111
+ descriptor.writeUInt32LE(0x08074b50, 0);
112
+ descriptor.writeUInt32LE(body.length, 8);
113
+ descriptor.writeUInt32LE(entry.data.length, 12);
114
+ const central = Buffer.alloc(46);
115
+ central.writeUInt32LE(0x02014b50, 0);
116
+ central.writeUInt16LE(0x0008, 8);
117
+ central.writeUInt16LE(method, 10);
118
+ central.writeUInt32LE(body.length, 20);
119
+ central.writeUInt32LE(entry.data.length, 24);
120
+ central.writeUInt16LE(name.length, 28);
121
+ central.writeUInt32LE(offset, 42);
122
+ parts.push(local, name, body, descriptor);
123
+ directory.push(central, name);
124
+ offset += local.length + name.length + body.length + descriptor.length;
125
+ }
126
+ const centralDirectory = Buffer.concat(directory);
127
+ const end = Buffer.alloc(22);
128
+ end.writeUInt32LE(0x06054b50, 0);
129
+ end.writeUInt16LE(entries.length, 8);
130
+ end.writeUInt16LE(entries.length, 10);
131
+ end.writeUInt32LE(centralDirectory.length, 12);
132
+ end.writeUInt32LE(offset, 16);
133
+ return Buffer.concat([...parts, centralDirectory, end]);
134
+}
135
+
136
function argsOfCall(index: number): ReadonlyArray<string> {
137
const call = mockedExecute.mock.calls[index];
138
assert.isDefined(call);
@@ -850,6 +902,80 @@ layer("AzureDevOpsPullRequestCli.layer", (it) => {
902
}),
903
);
904
905
+ it.effect(
906
+ "reads a batch of files from one blobs zip, keeping only what Azure's read would say",
907
+ () =>
908
+ Effect.gen(function* () {
909
+ const id = (digit: string) => digit.repeat(40);
910
+ const utf8 = (text: string) => new TextEncoder().encode(text);
911
+ let sent = "";
912
+ mockedExecute.mockImplementationOnce((input) =>
913
+ Effect.sync(() => {
914
+ const flag = (name: string) => input.args[input.args.indexOf(name) + 1] ?? "";
915
+ sent = NodeFS.readFileSync(flag("--in-file"), "utf8");
916
+ NodeFS.writeFileSync(
917
+ flag("--out-file"),
918
+ streamedZip([
919
+ { name: id("a"), data: utf8("plain text\n") },
920
+ { name: id("b"), data: utf8("\uFEFFcafé ✓\n") },
921
+ { name: id("c"), data: Uint8Array.of(0x89, 0x50, 0x00, 0x01) },
922
+ // Windows-1252 "café" and UTF-16: Azure decodes these itself.
923
+ { name: id("d"), data: Uint8Array.of(0x63, 0x61, 0x66, 0xe9) },
924
+ { name: id("e"), data: Uint8Array.of(0xff, 0xfe, 0x68, 0x00) },
925
+ { name: id("f"), data: utf8("stored\n"), stored: true },
926
+ { name: id("9"), data: utf8("never asked for\n") },
927
+ ]),
928
+ );
929
+ return output("");
930
+ }),
931
+ );
932
+ const cli = yield* AzureDevOpsPullRequestCli.AzureDevOpsPullRequestCli;
933
+
934
+ const blobs = yield* cli.readBlobs({
935
+ cwd: "/w",
936
+ location: { project: "platform", repository: "web" },
937
+ objectIds: [id("A"), id("b"), id("C"), id("d"), id("e"), id("f"), id("a"), "../etc"],
938
+ });
939
+
940
+ expect(JSON.parse(sent)).toEqual([id("A"), id("b"), id("C"), id("d"), id("e"), id("f")]);
941
+ expect(argsOfCall(0)).toEqual(
942
+ expect.arrayContaining(["blobs", "POST", "application/zip", "project=platform"]),
943
+ );
944
+ expect(Object.fromEntries(blobs)).toEqual({
945
+ [id("a")]: { contents: "plain text\n", isBinary: false },
946
+ [id("b")]: { contents: "café ✓\n", isBinary: false },
947
+ [id("c")]: { contents: "", isBinary: true },
948
+ [id("f")]: { contents: "stored\n", isBinary: false },
949
+ });
950
+ }),
951
+ );
952
+
953
+ it.effect("leaves blobs past the batch's total size to per-file reads", () =>
954
+ Effect.gen(function* () {
955
+ const ids = ["1", "2", "3", "4", "5", "6"].map((digit) => digit.repeat(40));
956
+ // Compresses to a few KiB each, so only the inflated total can bound memory.
957
+ const blob = new Uint8Array(3 * 1024 * 1024).fill(0x61);
958
+ mockedExecute.mockImplementationOnce((input) =>
959
+ Effect.sync(() => {
960
+ NodeFS.writeFileSync(
961
+ input.args[input.args.indexOf("--out-file") + 1] ?? "",
962
+ streamedZip(ids.map((name) => ({ name, data: blob }))),
963
+ );
964
+ return output("");
965
+ }),
966
+ );
967
+ const cli = yield* AzureDevOpsPullRequestCli.AzureDevOpsPullRequestCli;
968
+
969
+ const blobs = yield* cli.readBlobs({
970
+ cwd: "/w",
971
+ location: { project: "platform", repository: "web" },
972
+ objectIds: ids,
973
+ });
974
+
975
+ expect([...blobs.keys()]).toEqual(ids.slice(0, 5));
976
+ }),
977
+ );
978
+
979
it.effect("escapes characters in a file path that would cut the query string", () =>
980
Effect.gen(function* () {
981
mockedExecute.mockReturnValueOnce(Effect.succeed(output(json({ content: "x" }))));
packages/source-control-azure-devops/src/server/AzureDevOpsPullRequestCli.ts
+95
@@ -1,5 +1,6 @@
1
import * as Context from "effect/Context";
2
import * as Effect from "effect/Effect";
3
+import * as FileSystem from "effect/FileSystem";
4
import * as Layer from "effect/Layer";
5
import * as Result from "effect/Result";
6
import * as Schema from "effect/Schema";
@@ -12,6 +13,7 @@ import type {
13
} from "@t3tools/contracts";
14
15
import * as AzureDevOpsCli from "./AzureDevOpsCli.ts";
16
+import { blobContent, readBlobZip } from "./azureDevOpsBlobZip.ts";
17
import {
18
decodeItemContentJson,
19
decodeIterationChangesJson,
@@ -108,6 +110,13 @@ const PULL_REQUEST_LIST_MAX_OUTPUT_BYTES = 16 * 1024 * 1024;
110
const CHANGE_ENTRIES_MAX_OUTPUT_BYTES = 8 * 1024 * 1024;
111
/** JSON escaping and base64 can exceed the decoded file's size. */
112
const ITEM_CONTENT_MAX_OUTPUT_BYTES = 4 * 1024 * 1024;
113
+/** A larger batch is left to per-file reads rather than held in memory at once. */
114
+const BLOB_ARCHIVE_MAX_BYTES = 64 * 1024 * 1024;
115
+/** Matches what a per-file read could carry. */
116
+const BLOB_MAX_BYTES = ITEM_CONTENT_MAX_OUTPUT_BYTES;
117
+/** Blobs past this, together, are left to per-file reads, which hold a few at a time. */
118
+const BLOB_BATCH_MAX_BYTES = 16 * 1024 * 1024;
119
+const BLOB_ID = /^[0-9a-f]{40}$/i;
120
/** Review history is unpaged and independent of diff size. */
121
const REVIEW_HISTORY_MAX_OUTPUT_BYTES = 8 * 1024 * 1024;
122
@@ -184,6 +193,19 @@ export class AzureDevOpsPullRequestCli extends Context.Service<
193
readonly commit: string;
194
}) => Effect.Effect<AzureDevOpsItemContent, AzureDevOpsPullRequestCliError>;
195
196
+ /**
197
+ * Many files in one `az` run, by blob id, keyed by lowercase id. A blob only Azure's item read
198
+ * decodes faithfully, or one too large to carry, is left out.
199
+ */
200
+ readonly readBlobs: (input: {
201
+ readonly cwd: string;
202
+ readonly location: AzureDevOpsRepositoryLocation;
203
+ readonly objectIds: ReadonlyArray<string>;
204
+ }) => Effect.Effect<
205
+ ReadonlyMap<string, AzureDevOpsItemContent>,
206
+ AzureDevOpsPullRequestCliError
207
+ >;
208
+
209
readonly runPullRequestAction: (input: {
210
readonly cwd: string;
211
readonly number: number;
@@ -276,6 +298,7 @@ function isReviewerName(value: string): boolean {
298
/** @public */
299
export const make = Effect.gen(function* () {
300
const azure = yield* AzureDevOpsCli.AzureDevOpsCli;
301
+ const fileSystem = yield* FileSystem.FileSystem;
302
303
// Let `az` parse all three remote formats.
304
const detectArgs = ["--detect", "true"] as const;
@@ -596,6 +619,78 @@ export const make = Effect.gen(function* () {
619
},
620
),
621
622
+ readBlobs: Effect.fn("AzureDevOpsPullRequestCli.readBlobs")(function* (input) {
623
+ const requested = new Map<string, string>();
624
+ // Azure states ids in uppercase and names entries in lowercase.
625
+ for (const id of input.objectIds) {
626
+ const key = id.toLowerCase();
627
+ if (BLOB_ID.test(id) && !requested.has(key)) requested.set(key, id);
628
+ }
629
+ if (requested.size === 0) return new Map<string, AzureDevOpsItemContent>();
630
+ const readFailure = (cause: unknown) =>
631
+ new AzureDevOpsPullRequestReadError({
632
+ command: "az",
633
+ cwd: input.cwd,
634
+ operation: "readBlobs",
635
+ cause,
636
+ });
637
+ return yield* Effect.scoped(
638
+ Effect.gen(function* () {
639
+ const directory = yield* fileSystem
640
+ .makeTempDirectoryScoped({ prefix: "t3-azure-blobs-" })
641
+ .pipe(Effect.mapError(readFailure));
642
+ const request = `${directory}/ids.json`;
643
+ // invoke refuses to overwrite its output, so the archive gets a name nothing holds yet.
644
+ const archive = `${directory}/blobs.zip`;
645
+ yield* fileSystem
646
+ .writeFileString(
647
+ request,
648
+ `[${[...requested.values()].map((id) => `"${id}"`).join(",")}]`,
649
+ )
650
+ .pipe(Effect.mapError(readFailure));
651
+ yield* azure.execute({
652
+ cwd: input.cwd,
653
+ args: [
654
+ "devops",
655
+ "invoke",
656
+ ...detectArgs,
657
+ "--area",
658
+ "git",
659
+ "--resource",
660
+ "blobs",
661
+ "--api-version",
662
+ REST_API_VERSION,
663
+ "--route-parameters",
664
+ ...repositoryRoute(input.location),
665
+ "--http-method",
666
+ "POST",
667
+ "--in-file",
668
+ request,
669
+ "--accept-media-type",
670
+ "application/zip",
671
+ "--out-file",
672
+ archive,
673
+ "--only-show-errors",
674
+ ],
675
+ });
676
+ const size = yield* fileSystem.stat(archive).pipe(
677
+ Effect.map((info) => Number(info.size)),
678
+ Effect.mapError(readFailure),
679
+ );
680
+ const contents = new Map<string, AzureDevOpsItemContent>();
681
+ if (size > BLOB_ARCHIVE_MAX_BYTES) return contents;
682
+ const zip = yield* fileSystem.readFile(archive).pipe(Effect.mapError(readFailure));
683
+ const entries = readBlobZip(zip, BLOB_MAX_BYTES, BLOB_BATCH_MAX_BYTES);
684
+ if (entries === null) return yield* readFailure("Azure returned an unreadable archive.");
685
+ for (const [id, bytes] of entries) {
686
+ const content = requested.has(id) ? blobContent(bytes) : null;
687
+ if (content !== null) contents.set(id, content);
688
+ }
689
+ return contents;
690
+ }),
691
+ );
692
+ }),
693
+
694
readItemContent: Effect.fn("AzureDevOpsPullRequestCli.readItemContent")(function (input) {
695
return invoke({
696
cwd: input.cwd,
packages/source-control-azure-devops/src/server/AzureDevOpsPullRequestProvider.test.ts
+180
@@ -4,6 +4,7 @@ import * as Layer from "effect/Layer";
4
5
import * as AzureDevOpsPullRequestCli from "./AzureDevOpsPullRequestCli.ts";
6
import {
7
+ BLOB_BATCH_FILES,
8
LOCATION_CACHE_CAPACITY,
9
make,
10
MAX_DIFF_SPAWNS,
@@ -46,6 +47,9 @@ function change(
47
return { path, oldPath: path, changeKind, objectId: "8f80", originalObjectId: "0ca4" };
48
}
49
50
+/** A batch that returns nothing, so every file takes the per-file read. */
51
+const NO_BLOBS = { readBlobs: () => Effect.succeed(new Map()) };
52
+
53
function side(prefix: string, lines: number, width: number): string {
54
const pad = "z".repeat(width);
55
return `${Array.from({ length: lines }, (_, line) => `${prefix} ${line} ${pad}`).join("\n")}\n`;
@@ -69,6 +73,7 @@ const readSlice = (input: {
73
const provider = yield* make.pipe(
74
Effect.provide(
75
Layer.mock(AzureDevOpsPullRequestCli.AzureDevOpsPullRequestCli)({
76
+ ...NO_BLOBS,
77
getPullRequest: () => Effect.succeed(PULL_REQUEST),
78
listIterations: () => Effect.succeed([ITERATION]),
79
listIterationChanges: () =>
@@ -199,6 +204,7 @@ describe("getDiff reads", () => {
204
const provider = yield* make.pipe(
205
Effect.provide(
206
Layer.mock(AzureDevOpsPullRequestCli.AzureDevOpsPullRequestCli)({
207
+ ...NO_BLOBS,
208
getPullRequest: () => Effect.succeed(PULL_REQUEST),
209
listIterations: () => Effect.succeed([ITERATION]),
210
listIterationChanges: () =>
@@ -226,6 +232,179 @@ describe("getDiff reads", () => {
232
}),
233
);
234
235
+ it.effect("reads a slice's files in one blob batch, and per file only what it missed", () =>
236
+ Effect.gen(function* () {
237
+ const paths = ["a.ts", "b.ts", "c.ts", "gone.ts", "added.ts"];
238
+ const changes = [
239
+ ...["a.ts", "b.ts", "c.ts"].map((path) => ({
240
+ ...change(path),
241
+ objectId: `${path.toUpperCase()}-NEW`,
242
+ originalObjectId: `${path.toUpperCase()}-OLD`,
243
+ })),
244
+ { ...change("gone.ts", "deleted"), objectId: null, originalObjectId: "GONE-OLD" },
245
+ { ...change("added.ts", "new"), objectId: "ADDED-NEW", originalObjectId: null },
246
+ ];
247
+ const batches: ReadonlyArray<string>[] = [];
248
+ const reads: string[] = [];
249
+
250
+ const provider = yield* make.pipe(
251
+ Effect.provide(
252
+ Layer.mock(AzureDevOpsPullRequestCli.AzureDevOpsPullRequestCli)({
253
+ getPullRequest: () => Effect.succeed(PULL_REQUEST),
254
+ listIterations: () => Effect.succeed([ITERATION]),
255
+ listIterationChanges: () => Effect.succeed({ changes, truncated: false }),
256
+ readBlobs: (input) =>
257
+ Effect.sync(() => {
258
+ batches.push(input.objectIds);
259
+ // Azure left one side of b.ts out, as it does for a file only it can decode.
260
+ return new Map(
261
+ input.objectIds
262
+ .filter((id) => id !== "B.TS-OLD")
263
+ .map((id) => [
264
+ id.toLowerCase(),
265
+ {
266
+ contents: side(id.endsWith("NEW") ? "new" : "old", 2, 4),
267
+ isBinary: false,
268
+ },
269
+ ]),
270
+ );
271
+ }),
272
+ readItemContent: (item) =>
273
+ Effect.sync(() => {
274
+ reads.push(`${item.commit}:${item.path}`);
275
+ return {
276
+ contents: side(item.commit === ITERATION.headCommit ? "new" : "old", 2, 4),
277
+ isBinary: false,
278
+ };
279
+ }),
280
+ }),
281
+ ),
282
+ );
283
+
284
+ const slice = yield* provider.getDiff({
285
+ cwd: "/w",
286
+ repository: "acme/web",
287
+ host: "dev.azure.com",
288
+ number: 7,
289
+ });
290
+
291
+ expect(patchedPaths(slice.patch)).toEqual(paths);
292
+ expect(slice.patch.match(/^@@ /gmu)).toHaveLength(paths.length);
293
+ expect(batches).toEqual([
294
+ [
295
+ "A.TS-OLD",
296
+ "A.TS-NEW",
297
+ "B.TS-OLD",
298
+ "B.TS-NEW",
299
+ "C.TS-OLD",
300
+ "C.TS-NEW",
301
+ "GONE-OLD",
302
+ "ADDED-NEW",
303
+ ],
304
+ ]);
305
+ expect(reads.toSorted()).toEqual(["base:b.ts", "head:b.ts"]);
306
+ }),
307
+ );
308
+
309
+ it.effect("starts a new blob batch where the last one's files run out", () =>
310
+ Effect.gen(function* () {
311
+ const paths = Array.from({ length: BLOB_BATCH_FILES + 3 }, (_, at) => `f${at}.ts`);
312
+ // Added lines that bring the slice near its edit budget by file 96, so the read batches
313
+ // there narrow to 3 and the one from file 99 would reach past the blob batch.
314
+ const added = (at: number) => (at < 60 ? 41 : at < 96 ? 40 : at < 99 ? 0 : 10);
315
+ const batchSizes: number[] = [];
316
+ let fileReads = 0;
317
+
318
+ const provider = yield* make.pipe(
319
+ Effect.provide(
320
+ Layer.mock(AzureDevOpsPullRequestCli.AzureDevOpsPullRequestCli)({
321
+ getPullRequest: () => Effect.succeed(PULL_REQUEST),
322
+ listIterations: () => Effect.succeed([ITERATION]),
323
+ listIterationChanges: () =>
324
+ Effect.succeed({
325
+ changes: paths.map((path) => ({
326
+ ...change(path),
327
+ objectId: `${path}-new`,
328
+ originalObjectId: `${path}-old`,
329
+ })),
330
+ truncated: false,
331
+ }),
332
+ readBlobs: (input) =>
333
+ Effect.sync(() => {
334
+ batchSizes.push(input.objectIds.length);
335
+ return new Map(
336
+ input.objectIds.map((id) => {
337
+ const at = Number(id.slice(1, id.indexOf(".")));
338
+ const lines = id.endsWith("-new") ? added(at) : 0;
339
+ return [
340
+ id,
341
+ { contents: lines === 0 ? "" : side("+", lines, 1), isBinary: false },
342
+ ];
343
+ }),
344
+ );
345
+ }),
346
+ readItemContent: () =>
347
+ Effect.sync(() => ((fileReads += 1), { contents: "", isBinary: false })),
348
+ }),
349
+ ),
350
+ );
351
+
352
+ const slice = yield* provider.getDiff({
353
+ cwd: "/w",
354
+ repository: "acme/web",
355
+ host: "dev.azure.com",
356
+ number: 7,
357
+ });
358
+
359
+ expect(patchedPaths(slice.patch)).toHaveLength(paths.length);
360
+ expect(batchSizes).toEqual([2 * BLOB_BATCH_FILES, 6]);
361
+ expect(fileReads).toBe(0);
362
+ }),
363
+ );
364
+
365
+ it.effect("serves a later slice from the iteration and change list the first one read", () =>
366
+ Effect.gen(function* () {
367
+ const paths = ["a.ts", "b.ts", "c.ts", "d.ts", "e.ts", "f.ts"];
368
+ let iterationReads = 0;
369
+ let changeReads = 0;
370
+
371
+ const provider = yield* make.pipe(
372
+ Effect.provide(
373
+ Layer.mock(AzureDevOpsPullRequestCli.AzureDevOpsPullRequestCli)({
374
+ ...NO_BLOBS,
375
+ getPullRequest: () => Effect.succeed(PULL_REQUEST),
376
+ listIterations: () => Effect.sync(() => ((iterationReads += 1), [ITERATION])),
377
+ listIterationChanges: () =>
378
+ Effect.sync(() => {
379
+ changeReads += 1;
380
+ return { changes: paths.map((path) => change(path)), truncated: false };
381
+ }),
382
+ readItemContent: (item) =>
383
+ Effect.succeed({
384
+ contents: side(item.commit === ITERATION.headCommit ? "new" : "old", 100, 900),
385
+ isBinary: false,
386
+ }),
387
+ }),
388
+ ),
389
+ );
390
+ const readDiff = (cursor?: string) =>
391
+ provider.getDiff({
392
+ cwd: "/w",
393
+ repository: "acme/web",
394
+ host: "dev.azure.com",
395
+ number: 7,
396
+ ...(cursor === undefined ? {} : { cursor }),
397
+ });
398
+
399
+ const first = yield* readDiff();
400
+ const second = yield* readDiff(first.nextCursor ?? undefined);
401
+
402
+ expect(patchedPaths(second.patch)).toEqual(["c.ts", "d.ts"]);
403
+ expect(iterationReads).toBe(1);
404
+ expect(changeReads).toBe(1);
405
+ }),
406
+ );
407
+
408
it.effect("leaves a run of files it could not diff at all for the next slice", () =>
409
Effect.gen(function* () {
410
const paths = Array.from({ length: MAX_DIFF_SLICE_FILES + 20 }, (_, at) => `gen/a${at}.bin`);
@@ -321,6 +500,7 @@ describe("what one diff slice spends", () => {
500
const provider = yield* make.pipe(
501
Effect.provide(
502
Layer.mock(AzureDevOpsPullRequestCli.AzureDevOpsPullRequestCli)({
503
+ ...NO_BLOBS,
504
getPullRequest: (input) =>
505
Effect.sync(() => {
506
readsOf.set(input.number, (readsOf.get(input.number) ?? 0) + 1);
packages/source-control-azure-devops/src/server/AzureDevOpsPullRequestProvider.ts
+126
-25
@@ -35,7 +35,7 @@ import type {
35
} from "./azureDevOpsPullRequestJson.ts";
36
import { definition } from "../shared/definition.ts";
37
38
-/** Each file pays for two Python startups. */
38
+/** A file the blob batch missed pays for two Python startups. */
39
const DIFF_FILE_CONCURRENCY = 4;
40
41
/** Readers share a spawn budget across requests. */
@@ -43,6 +43,42 @@ export const MAX_DIFF_SPAWNS = 2 * DIFF_FILE_CONCURRENCY;
43
44
export const LOCATION_CACHE_CAPACITY = 128;
45
46
+/** A version's change list can run to thousands of paths. */
47
+const CHANGES_CACHE_CAPACITY = 32;
48
+
49
+/**
50
+ * Files whose contents one `az` run fetches. The run costs about the same for 10 blobs as for
51
+ * 100, and a slice rarely spends this many before its budgets close it.
52
+ */
53
+export const BLOB_BATCH_FILES = 100;
54
+
55
+/** Least recently used goes first. */
56
+function remember<V>(cache: Map<string, V>, capacity: number, key: string, value: V): V {
57
+ cache.delete(key);
58
+ if (cache.size >= capacity) {
59
+ const oldest = cache.keys().next().value;
60
+ if (oldest !== undefined) cache.delete(oldest);
61
+ }
62
+ cache.set(key, value);
63
+ return value;
64
+}
65
+
66
+function recall<V>(cache: Map<string, V>, key: string): V | undefined {
67
+ const held = cache.get(key);
68
+ if (held !== undefined) remember(cache, Number.POSITIVE_INFINITY, key, held);
69
+ return held;
70
+}
71
+
72
+/** The blobs a file's two sides need, as the change list names them. */
73
+function blobIdsOf(change: AzureDevOpsChangeEntry): ReadonlyArray<string> {
74
+ const ids: string[] = [];
75
+ if (change.changeKind !== "new" && change.originalObjectId !== null) {
76
+ ids.push(change.originalObjectId);
77
+ }
78
+ if (change.changeKind !== "deleted" && change.objectId !== null) ids.push(change.objectId);
79
+ return ids;
80
+}
81
+
82
const CAPABILITIES: PullRequestCapabilities = {
83
diff: true,
84
comment: false,
@@ -137,6 +173,8 @@ export const make = Effect.gen(function* () {
173
const diffSpawns = yield* Semaphore.make(MAX_DIFF_SPAWNS);
174
const readItemContent = (input: Parameters<typeof cli.readItemContent>[0]) =>
175
diffSpawns.withPermits(1)(cli.readItemContent(input));
176
+ const readBlobs = (input: Parameters<typeof cli.readBlobs>[0]) =>
177
+ diffSpawns.withPermits(1)(cli.readBlobs(input));
178
179
const fail =
180
(operation: string) => (error: AzureDevOpsPullRequestCli.AzureDevOpsPullRequestCliError) =>
@@ -162,43 +200,67 @@ export const make = Effect.gen(function* () {
200
201
// PR locations are immutable; caching avoids full PR reads for viewed-file checks.
202
const locations = new Map<string, AzureDevOpsRepositoryLocation>();
203
+ // A pushed iteration never changes, so neither do its commits nor its change list.
204
+ const knownIterations = new Map<string, ReadonlyArray<AzureDevOpsIteration>>();
205
+ const iterationChanges = new Map<string, AzureDevOpsPullRequestCli.AzureDevOpsIterationChanges>();
206
207
const locationOf = (input: { readonly cwd: string; readonly number: number }) => {
208
const key = `${input.cwd} ${input.number}`;
168
- const held = locations.get(key);
169
- if (held !== undefined) {
170
- locations.delete(key);
171
- locations.set(key, held);
172
- return Effect.succeed(held);
173
- }
174
- return cli.getPullRequest({ cwd: input.cwd, number: input.number }).pipe(
175
- Effect.map((pullRequest) => {
176
- const location = pullRequest.location;
177
- if (location === null) return null;
178
- if (locations.size >= LOCATION_CACHE_CAPACITY) {
179
- const oldest = locations.keys().next().value;
180
- if (oldest !== undefined) locations.delete(oldest);
181
- }
182
- locations.set(key, location);
183
- return location;
184
- }),
185
- );
209
+ const held = recall(locations, key);
210
+ if (held !== undefined) return Effect.succeed(held);
211
+ return cli
212
+ .getPullRequest({ cwd: input.cwd, number: input.number })
213
+ .pipe(
214
+ Effect.map((pullRequest) =>
215
+ pullRequest.location === null
216
+ ? null
217
+ : remember(locations, LOCATION_CACHE_CAPACITY, key, pullRequest.location),
218
+ ),
219
+ );
220
};
221
222
+ /** A later slice names its iteration, which the list it was cut from already holds. */
223
const diffScope = Effect.fn("AzureDevOpsPullRequestProvider.diffScope")(function* (input: {
224
readonly cwd: string;
225
readonly number: number;
226
+ readonly iterationId?: number;
227
}) {
228
const location = yield* locationOf(input);
229
if (location === null) return null;
230
+ const key = `${input.cwd} ${input.number}`;
231
+ const held = recall(knownIterations, key);
232
+ if (
233
+ input.iterationId !== undefined &&
234
+ held?.some((iteration) => iteration.id === input.iterationId)
235
+ ) {
236
+ return { location, iterations: held };
237
+ }
238
const iterations = yield* cli.listIterations({
239
cwd: input.cwd,
240
location,
241
number: input.number,
242
});
243
+ remember(knownIterations, LOCATION_CACHE_CAPACITY, key, iterations);
244
return { location, iterations };
245
});
246
247
+ const changesOf = (input: {
248
+ readonly cwd: string;
249
+ readonly location: AzureDevOpsRepositoryLocation;
250
+ readonly number: number;
251
+ readonly iterationId: number;
252
+ }) => {
253
+ const key = `${input.cwd} ${input.number} ${input.iterationId}`;
254
+ const held = recall(iterationChanges, key);
255
+ return held !== undefined
256
+ ? Effect.succeed(held)
257
+ : cli
258
+ .listIterationChanges(input)
259
+ .pipe(
260
+ Effect.map((listed) => remember(iterationChanges, CHANGES_CACHE_CAPACITY, key, listed)),
261
+ );
262
+ };
263
+
264
const EMPTY_ITEM: AzureDevOpsItemContent = { contents: "", isBinary: false };
265
266
/** Azure fails reads of absent files. */
@@ -249,7 +311,7 @@ export const make = Effect.gen(function* () {
311
changes: [],
312
truncated: false,
313
} as AzureDevOpsPullRequestCli.AzureDevOpsIterationChanges)
252
- : cli.listIterationChanges({
314
+ : changesOf({
315
cwd: input.cwd,
316
location: input.location,
317
number: input.number,
@@ -391,15 +453,19 @@ export const make = Effect.gen(function* () {
453
// the Code tab has nothing to scope itself to and always asks for the whole change.
454
getDiff: Effect.fn("AzureDevOpsPullRequestProvider.getDiff")(
455
function* (input) {
394
- const scope = yield* diffScope(input);
395
- if (scope === null) return EMPTY_DIFF_SLICE;
456
const cursor = parseAzureDevOpsDiffCursor(input.cursor);
457
+ const scope = yield* diffScope({
458
+ cwd: input.cwd,
459
+ number: input.number,
460
+ ...(cursor === null ? {} : { iterationId: cursor.iterationId }),
461
+ });
462
+ if (scope === null) return EMPTY_DIFF_SLICE;
463
const iteration =
464
cursor === null
465
? scope.iterations.at(-1)
466
: scope.iterations.find((candidate) => candidate.id === cursor.iterationId);
467
if (iteration === undefined) return EMPTY_DIFF_SLICE;
402
- const listed = yield* cli.listIterationChanges({
468
+ const listed = yield* changesOf({
469
cwd: input.cwd,
470
location: scope.location,
471
number: input.number,
@@ -428,12 +494,47 @@ export const make = Effect.gen(function* () {
494
),
495
);
496
};
497
+ let blobs: ReadonlyMap<string, AzureDevOpsItemContent> = new Map();
498
+ let blobsUntil = index;
499
+ const fromBlobs = (change: AzureDevOpsChangeEntry): AzureDevOpsFileTexts | null => {
500
+ const side = (id: string | null) =>
501
+ id === null ? undefined : blobs.get(id.toLowerCase());
502
+ const oldItem = change.changeKind === "new" ? EMPTY_ITEM : side(change.originalObjectId);
503
+ const newItem = change.changeKind === "deleted" ? EMPTY_ITEM : side(change.objectId);
504
+ return oldItem === undefined || newItem === undefined
505
+ ? null
506
+ : {
507
+ oldContents: oldItem.contents,
508
+ newContents: newItem.contents,
509
+ binary: oldItem.isBinary || newItem.isBinary,
510
+ };
511
+ };
512
while (!full && index < changes.length) {
432
- const batch = changes.slice(index, index + batchWidth());
513
+ if (index >= blobsUntil) {
514
+ blobsUntil = index + BLOB_BATCH_FILES;
515
+ blobs = yield* readBlobs({
516
+ cwd: input.cwd,
517
+ location: scope.location,
518
+ objectIds: changes.slice(index, blobsUntil).flatMap(blobIdsOf),
519
+ }).pipe(
520
+ // Per-file reads then answer, or fail, for each file.
521
+ Effect.catchTags({
522
+ AzureDevOpsPullRequestNotFoundError: () => Effect.succeed(new Map()),
523
+ AzureDevOpsCommandFailedError: () => Effect.succeed(new Map()),
524
+ AzureDevOpsPullRequestReadError: () => Effect.succeed(new Map()),
525
+ }),
526
+ );
527
+ }
528
+ const batch = changes.slice(index, Math.min(index + batchWidth(), blobsUntil));
529
const read = yield* Effect.forEach(
530
batch,
531
(change) =>
436
- readTexts({ cwd: input.cwd, location: scope.location, iteration, change }).pipe(
532
+ Effect.suspend(() => {
533
+ const held = fromBlobs(change);
534
+ return held === null
535
+ ? readTexts({ cwd: input.cwd, location: scope.location, iteration, change })
536
+ : Effect.succeed(held);
537
+ }).pipe(
538
// Auth, rate-limit and CLI failures must reach the caller.
539
Effect.catchTags({
540
AzureDevOpsPullRequestNotFoundError: () => Effect.succeed(null),
packages/source-control-azure-devops/src/server/azureDevOpsBlobZip.ts
new
+96
@@ -0,0 +1,96 @@
1
+// @effect-diagnostics nodeBuiltinImport:off
2
+// Effect has no inflate, and Azure's blob batch only comes as a deflate zip.
3
+import * as NodeZlib from "node:zlib";
4
+
5
+import type { AzureDevOpsItemContent } from "./azureDevOpsPullRequestJson.ts";
6
+
7
+const END_OF_CENTRAL_DIRECTORY = 0x06054b50;
8
+const CENTRAL_DIRECTORY_ENTRY = 0x02014b50;
9
+const LOCAL_FILE_HEADER = 0x04034b50;
10
+const STORED = 0;
11
+const DEFLATED = 8;
12
+const ZIP64_MARKER = 0xffffffff;
13
+
14
+const entryName = new TextDecoder();
15
+
16
+/**
17
+ * The entries of the zip Azure's Blobs API answers with, keyed by lowercase blob id. Azure streams
18
+ * entries with data descriptors, so sizes come from the central directory. Null for an archive
19
+ * this cannot read; an entry over `maxEntryBytes`, past `maxTotalBytes` in all, or in another method
20
+ * is left out.
21
+ */
22
+export function readBlobZip(
23
+ zip: Uint8Array,
24
+ maxEntryBytes: number,
25
+ maxTotalBytes: number,
26
+): ReadonlyMap<string, Uint8Array> | null {
27
+ const view = new DataView(zip.buffer, zip.byteOffset, zip.byteLength);
28
+ // The end record is 22 bytes followed by a comment of at most 65,535.
29
+ let end = -1;
30
+ for (let at = zip.length - 22; at >= 0 && at >= zip.length - 22 - 0xffff; at -= 1) {
31
+ if (view.getUint32(at, true) === END_OF_CENTRAL_DIRECTORY) {
32
+ end = at;
33
+ break;
34
+ }
35
+ }
36
+ if (end === -1) return null;
37
+ const count = view.getUint16(end + 10, true);
38
+ let at = view.getUint32(end + 16, true);
39
+ if (count === 0xffff || at === ZIP64_MARKER) return null;
40
+
41
+ const entries = new Map<string, Uint8Array>();
42
+ let total = 0;
43
+ for (let entry = 0; entry < count; entry += 1) {
44
+ if (at + 46 > zip.length || view.getUint32(at, true) !== CENTRAL_DIRECTORY_ENTRY) return null;
45
+ const method = view.getUint16(at + 10, true);
46
+ const compressedSize = view.getUint32(at + 20, true);
47
+ const size = view.getUint32(at + 24, true);
48
+ const nameLength = view.getUint16(at + 28, true);
49
+ const local = view.getUint32(at + 42, true);
50
+ const name = entryName.decode(zip.subarray(at + 46, at + 46 + nameLength)).toLowerCase();
51
+ at += 46 + nameLength + view.getUint16(at + 30, true) + view.getUint16(at + 32, true);
52
+ if (compressedSize === ZIP64_MARKER || size === ZIP64_MARKER || local === ZIP64_MARKER) {
53
+ return null;
54
+ }
55
+ if (local + 30 > zip.length || view.getUint32(local, true) !== LOCAL_FILE_HEADER) return null;
56
+ const start = local + 30 + view.getUint16(local + 26, true) + view.getUint16(local + 28, true);
57
+ if (start + compressedSize > zip.length) return null;
58
+ if (size > maxEntryBytes || total + size > maxTotalBytes) continue;
59
+ const data = zip.subarray(start, start + compressedSize);
60
+ if (method === STORED) {
61
+ if (compressedSize !== size) return null;
62
+ entries.set(name, data);
63
+ total += size;
64
+ } else if (method === DEFLATED) {
65
+ try {
66
+ const inflated = NodeZlib.inflateRawSync(data, { maxOutputLength: maxEntryBytes });
67
+ if (inflated.length !== size) return null;
68
+ entries.set(name, inflated);
69
+ total += size;
70
+ } catch {
71
+ return null;
72
+ }
73
+ }
74
+ }
75
+ return entries;
76
+}
77
+
78
+/** Strips a UTF-8 BOM, as Azure's `content` does. */
79
+const strictUtf8 = new TextDecoder("utf-8", { fatal: true });
80
+
81
+/**
82
+ * A blob's text as Azure's item read would give it, or null where Azure decodes the bytes itself
83
+ * (UTF-16, or text that is not UTF-8 such as Windows-1252) and only that read is faithful.
84
+ */
85
+export function blobContent(bytes: Uint8Array): AzureDevOpsItemContent | null {
86
+ const utf16 =
87
+ (bytes[0] === 0xff && bytes[1] === 0xfe) || (bytes[0] === 0xfe && bytes[1] === 0xff);
88
+ if (utf16) return null;
89
+ // Azure flags a binary on the same evidence.
90
+ if (bytes.includes(0)) return { contents: "", isBinary: true };
91
+ try {
92
+ return { contents: strictUtf8.decode(bytes), isBinary: false };
93
+ } catch {
94
+ return null;
95
+ }
96
+}