@devries-cameron20 / t3code Mirror · read-only
Fork 0

The best way to control Claude Code, Codex and more from any computer or phone

Mirrored from https://github.com/Camo67/t3code.git

Synced Nov 21, 04:45 UTC

main
ts 1,438 lines 72.1 KB
Raw
1 // @effect-diagnostics nodeBuiltinImport:off
2 import * as NodeChildProcess from "node:child_process";
3 import * as NodeFS from "node:fs";
4 import * as NodePath from "node:path";
5 import * as NodeServices from "@effect/platform-node/NodeServices";
6 import { describe, expect, it } from "@effect/vitest";
7 import * as Effect from "effect/Effect";
8 import * as Deferred from "effect/Deferred";
9 import * as Fiber from "effect/Fiber";
10 import * as Queue from "effect/Queue";
11 import * as FileSystem from "effect/FileSystem";
12 import * as Layer from "effect/Layer";
13 import * as TestClock from "effect/testing/TestClock";
14 import { vi } from "vite-plus/test";
15 import * as HostProcess from "@t3tools/shared/HostProcess";
16 import * as ServerConfig from "../config.ts";
17 import * as DocumentText from "./DocumentText.ts";
18 import { extractDocumentText } from "./documentTextWorker.ts";
19 import * as Cleanup from "../orchestration-v2/ResourceCleanupService.ts";
20 import * as DeviceService from "../device/DeviceService.ts";
21 import * as TerminalManager from "../terminal/Manager.ts";
22 import * as PreviewManager from "../preview/Manager.ts";
23 import * as ScheduledTaskService from "../scheduledTasks/ScheduledTaskService.ts";
24 import * as NodeZlib from "node:zlib";
25 import { appendUserInputAttachmentPaths } from "./userInputAttachments.ts";
26
27 vi.mock("node:child_process", { spy: true });
28
29 const layer = Layer.merge(
30 ServerConfig.layerTest(process.cwd(), { prefix: "t3-answer-paths-" }),
31 DocumentText.layer,
32 ).pipe(Layer.provideMerge(NodeServices.layer));
33 const attachment = {
34 type: "file" as const,
35 id: "thread-1-00000000-0000-4000-8000-0000000000aa-txt",
36 name: 'spec "final".txt',
37 mimeType: "text/plain",
38 sizeBytes: 4,
39 };
40 describe("question answer paths", () => {
41 it.effect.each([false, true])(
42 "keeps unrelated files available during extraction (failure: %s)",
43 (failure) =>
44 Effect.gen(function* () {
45 const config = yield* ServerConfig.ServerConfig;
46 const fs = yield* FileSystem.FileSystem;
47 const path = NodePath.join(config.attachmentsDir, "blocked.rtf");
48 const cached = NodePath.join(config.attachmentsDir, "cached.rtf");
49 NodeFS.writeFileSync(path, failure ? "invalid document" : String.raw`{\rtf1 Available}`);
50 NodeFS.writeFileSync(cached + ".txt", "Cached text");
51 const started = yield* Deferred.make<void>();
52 const release = yield* Deferred.make<void>();
53 let reads = 0;
54 yield* Effect.gen(function* () {
55 const documents = yield* DocumentText.DocumentText;
56 const extraction = yield* documents.ensure(path).pipe(Effect.forkChild);
57 yield* Deferred.await(started);
58 const duplicate = yield* documents.ensure(path).pipe(Effect.forkChild);
59 expect(yield* documents.ensure(path + ".png")).toBeUndefined();
60 expect(yield* documents.ensure(path + ".pdf")).toBeUndefined();
61 expect(yield* documents.ensure(cached)).toBe(cached + ".txt");
62 expect(NodeFS.readFileSync(cached + ".txt", "utf8")).toBe("Cached text");
63 yield* Deferred.succeed(release, undefined);
64 const expected = failure ? undefined : path + ".txt";
65 expect(yield* Fiber.join(extraction)).toBe(expected);
66 expect(yield* Fiber.join(duplicate)).toBe(expected);
67 expect(yield* documents.ensure(path)).toBe(expected);
68 expect(reads).toBe(1);
69 if (failure) {
70 NodeFS.writeFileSync(path + ".txt", "Recovered text");
71 expect(yield* documents.ensure(path)).toBe(path + ".txt");
72 expect(NodeFS.readFileSync(path + ".txt", "utf8")).toBe("Recovered text");
73 } else expect(NodeFS.readFileSync(path + ".txt", "utf8")).toBe("Available");
74 }).pipe(
75 Effect.provide(Layer.fresh(DocumentText.layer)),
76 Effect.provideService(FileSystem.FileSystem, {
77 ...fs,
78 stat: (file) =>
79 file === path
80 ? Effect.gen(function* () {
81 reads++;
82 yield* Deferred.succeed(started, undefined);
83 yield* Deferred.await(release);
84 return yield* fs.stat(file);
85 })
86 : fs.stat(file),
87 }),
88 );
89 }).pipe(Effect.provide(layer)),
90 );
91
92 it.effect.each(["interrupted", "failed"])(
93 "extracts two documents at once and frees a slot when one is %s",
94 (outcome) =>
95 Effect.gen(function* () {
96 const config = yield* ServerConfig.ServerConfig;
97 const fs = yield* FileSystem.FileSystem;
98 const [first, second, third] = ["first", "second", "third"].map((name) =>
99 NodePath.join(config.attachmentsDir, `${name}.rtf`),
100 );
101 NodeFS.writeFileSync(first!, String.raw`{\rtf1 First}`);
102 NodeFS.writeFileSync(second!, "invalid document");
103 NodeFS.writeFileSync(third!, String.raw`{\rtf1 Third}`);
104 const started = yield* Queue.unbounded<string>();
105 const releases = new Map<string, Deferred.Deferred<void>>();
106 for (const path of [first!, second!, third!])
107 releases.set(path, yield* Deferred.make<void>());
108 yield* Effect.gen(function* () {
109 const documents = yield* DocumentText.DocumentText;
110 const leaving = yield* documents.ensure(first!).pipe(Effect.forkChild);
111 const staying = yield* documents.ensure(first!).pipe(Effect.forkChild);
112 const freed = yield* documents.ensure(second!).pipe(Effect.forkChild);
113 expect(new Set([yield* Queue.take(started), yield* Queue.take(started)])).toEqual(
114 new Set([first, second]),
115 );
116 const waiting = yield* documents.ensure(third!).pipe(Effect.forkChild);
117 yield* Fiber.interrupt(leaving);
118 if (outcome === "interrupted") yield* Fiber.interrupt(freed);
119 else {
120 yield* Deferred.succeed(releases.get(second!)!, undefined);
121 expect(yield* Fiber.join(freed)).toBeUndefined();
122 }
123 expect(yield* Queue.take(started)).toBe(third);
124 yield* Deferred.succeed(releases.get(first!)!, undefined);
125 yield* Deferred.succeed(releases.get(third!)!, undefined);
126 expect(yield* Fiber.join(staying)).toBe(first + ".txt");
127 expect(yield* Fiber.join(waiting)).toBe(third + ".txt");
128 expect(yield* Queue.size(started)).toBe(0);
129 expect(NodeFS.existsSync(second + ".txt")).toBe(false);
130 expect(
131 NodeFS.readdirSync(config.attachmentsDir).filter((file) => file.endsWith(".part")),
132 ).toEqual([]);
133 }).pipe(
134 Effect.provide(Layer.fresh(DocumentText.layer)),
135 Effect.provideService(FileSystem.FileSystem, {
136 ...fs,
137 // An async probe can let `staying` join only after `leaving` cancelled the lookup.
138 exists: (file) => Effect.sync(() => NodeFS.existsSync(file)),
139 stat: (file) =>
140 Effect.gen(function* () {
141 yield* Queue.offer(started, file);
142 yield* Deferred.await(releases.get(file)!);
143 return yield* fs.stat(file);
144 }),
145 }),
146 );
147 }).pipe(Effect.provide(layer)),
148 );
149
150 it.effect("retries a document while its cancelled extraction is still publishing", () =>
151 Effect.gen(function* () {
152 const config = yield* ServerConfig.ServerConfig;
153 const fs = yield* FileSystem.FileSystem;
154 const path = NodePath.join(config.attachmentsDir, "retried.rtf");
155 NodeFS.writeFileSync(path, String.raw`{\rtf1 Retried}`);
156 const writing = yield* Deferred.make<void>();
157 const writeReleased = yield* Deferred.make<void>();
158 const renaming = yield* Deferred.make<void>();
159 const renameReleased = yield* Deferred.make<void>();
160 let writes = 0;
161 let renames = 0;
162 yield* Effect.gen(function* () {
163 const documents = yield* DocumentText.DocumentText;
164 const cancelled = yield* documents.ensure(path).pipe(Effect.forkChild);
165 yield* Deferred.await(writing);
166 const interrupting = yield* Fiber.interrupt(cancelled).pipe(Effect.forkChild);
167 const retry = yield* documents.ensure(path).pipe(Effect.forkChild);
168 yield* Deferred.await(renaming);
169 yield* Deferred.succeed(writeReleased, undefined);
170 yield* Fiber.join(interrupting);
171 yield* Deferred.succeed(renameReleased, undefined);
172 expect(yield* Fiber.join(retry)).toBe(path + ".txt");
173 expect(NodeFS.readFileSync(path + ".txt", "utf8")).toBe("Retried");
174 expect(
175 NodeFS.readdirSync(config.attachmentsDir).filter((file) => file.endsWith(".part")),
176 ).toEqual([]);
177 }).pipe(
178 Effect.provide(Layer.fresh(DocumentText.layer)),
179 Effect.provideService(FileSystem.FileSystem, {
180 ...fs,
181 writeFileString: (file, data) =>
182 Effect.gen(function* () {
183 if (++writes === 1) {
184 yield* Deferred.succeed(writing, undefined);
185 yield* Deferred.await(writeReleased);
186 }
187 return yield* fs.writeFileString(file, data);
188 }),
189 rename: (from, to) =>
190 Effect.gen(function* () {
191 if (++renames === 1) {
192 yield* Deferred.succeed(renaming, undefined);
193 yield* Deferred.await(renameReleased);
194 }
195 return yield* fs.rename(from, to);
196 }),
197 }),
198 );
199 }).pipe(Effect.provide(layer)),
200 );
201
202 it.effect.each(["spawn", "read"])("retries after a transient %s failure expires", (failure) =>
203 Effect.acquireUseRelease(
204 Effect.promise(async () => {
205 const { execFile } = await vi.importActual<typeof NodeChildProcess>("node:child_process");
206 const worker = vi.spyOn(NodeChildProcess, "execFile");
207 worker.mockImplementationOnce(((
208 file: string,
209 args: readonly string[],
210 options: NodeChildProcess.ExecFileOptionsWithStringEncoding,
211 callback: (
212 error: NodeChildProcess.ExecFileException | null,
213 stdout: string,
214 stderr: string,
215 ) => void,
216 ) => {
217 if (failure === "spawn") {
218 throw Object.assign(new Error("spawn EAGAIN"), { code: "EAGAIN" });
219 }
220 const preload = `import fs from 'node:fs/promises';
221 import { syncBuiltinESMExports } from 'node:module';
222 fs.readFile = async () => { throw Object.assign(new Error('read EAGAIN'), { code: 'EAGAIN' }); };
223 syncBuiltinESMExports();`;
224 return execFile(
225 file,
226 ["--import", `data:text/javascript,${encodeURIComponent(preload)}`, ...args],
227 options,
228 callback,
229 );
230 }) as typeof NodeChildProcess.execFile);
231 return worker;
232 }),
233 (worker) =>
234 Effect.gen(function* () {
235 const config = yield* ServerConfig.ServerConfig;
236 const path = NodePath.join(config.attachmentsDir, "retry.rtf");
237 NodeFS.writeFileSync(path, String.raw`{\rtf1 Available}`);
238 const documents = yield* DocumentText.DocumentText;
239 expect(yield* documents.ensure(path)).toBeUndefined();
240 expect(yield* documents.ensure(path)).toBeUndefined();
241 expect(worker).toHaveBeenCalledTimes(1);
242 expect(NodeFS.existsSync(path + ".txt")).toBe(false);
243 yield* TestClock.adjust("1 second");
244 expect(yield* documents.ensure(path)).toBe(path + ".txt");
245 expect(NodeFS.readFileSync(path + ".txt", "utf8")).toBe("Available");
246 expect(worker).toHaveBeenCalledTimes(2);
247 expect(yield* documents.ensure(path)).toBe(path + ".txt");
248 expect(worker).toHaveBeenCalledTimes(2);
249 }).pipe(Effect.provide(layer)),
250 (worker) => Effect.sync(() => worker.mockRestore()),
251 ),
252 );
253
254 it.effect.each([
255 ["ipynb", "{"],
256 ["rtf", "invalid document"],
257 ["rtf", String.raw`{\rtf1}`],
258 ["rtf", String.raw`{\rtf1\ansi\ansicpg99999 Unsupported}`],
259 ] as const)("caches unchanged document rejections: %s %s", ([extension, source]) =>
260 Effect.acquireUseRelease(
261 Effect.sync(() => vi.spyOn(NodeChildProcess, "execFile")),
262 (worker) =>
263 Effect.gen(function* () {
264 const config = yield* ServerConfig.ServerConfig;
265 const path = NodePath.join(config.attachmentsDir, `invalid.${extension}`);
266 NodeFS.writeFileSync(path, source);
267 const documents = yield* DocumentText.DocumentText;
268 for (const seconds of [0, 1, 60]) {
269 yield* TestClock.adjust(`${seconds} seconds`);
270 expect(yield* documents.ensure(path)).toBeUndefined();
271 expect(worker).toHaveBeenCalledTimes(1);
272 expect(NodeFS.existsSync(path + ".txt")).toBe(false);
273 }
274 }).pipe(Effect.provide(layer)),
275 (worker) => Effect.sync(() => worker.mockRestore()),
276 ),
277 );
278
279 it.effect.each([
280 [
281 "page-specific headers and footers",
282 String.raw`{\rtf1\ansi\facingp\sectd\titlepg
283 {\headerf First-page header}{\headerl Left-page header}{\headerr Right-page header}
284 {\footerf First-page footer}{\footerl Left-page footer}{\footerr Right-page footer}
285 \pard Approved delivery date: 24 November.\par
286 }`,
287 "Approved delivery date: 24 November.",
288 ],
289 [
290 "text symbols",
291 String.raw`{\rtf1\ansi Pages 1\endash 5\emdash\lquote left\rquote\ldblquote right\rdblquote\bullet\b End\b0\unknown{\*\secret\bullet hidden}}`,
292 "Pages 1–5—‘left’“right”•End",
293 ],
294 [
295 "space and hyphen symbols",
296 String.raw`{\rtf1\ansi A\enspace B\emspace C\qmspace D\~E\_F\-G}`,
297 "A\u2002B\u2003C\u2005D\u00a0E\u2011FG",
298 ],
299 [
300 "joiner and direction marks",
301 String.raw`{\rtf1\ansi A\zwj B\zwnj C\ltrmark D\rtlmark E}`,
302 "A\u200dB\u200cC\u200eD\u200fE",
303 ],
304 [
305 "text symbol fallback",
306 String.raw`{\rtf1\ansi\uc10\u945\endash\emdash\enspace\emspace\qmspace\lquote\rquote\ldblquote\rdblquote\bullet Z}`,
307 "αZ",
308 ],
309 ["hyphen symbol fallback", String.raw`{\rtf1\ansi\uc3\u945\~\_\-Z}`, "αZ"],
310 ["control word", String.raw`{\rtf1\ansi A\u9\tab X}`, "A\tX"],
311 ["line break paragraphs", "{\\rtf1\\ansi First\\\nSecond\\\r\nThird}", "First\nSecond\nThird"],
312 [
313 "tracked deletion",
314 String.raw`{\rtf1\ansi The fee is {\deleted\revauthdel1\revdttmdel1 $500}{\revised\revauth1 $700}.}`,
315 "The fee is $700.",
316 ],
317 [
318 "tracked deletion end",
319 String.raw`{\rtf1\ansi{\header Header\plain Plain}A\deleted B\deleted0 C\deleted D\plain E}`,
320 "ACE",
321 ],
322 [
323 "tracked deletion fallback",
324 String.raw`{\rtf1\ansi A{\deleted\uc1\u945\deleted0 B}\deleted\u946?\deleted0 C}`,
325 "AC",
326 ],
327 [
328 "tracked deletion table",
329 String.raw`{\rtf1\ansi\trowd\cellx1000\cellx2000\intbl A\deleted\cell\deleted0 B\cell\deleted\row\deleted0 C\cell}`,
330 "A\tB\t\nC",
331 ],
332 [
333 "document breaks",
334 String.raw`{\rtf1\ansi\sectd\cols2\pard North quota: 120\column South quota: 240\page East quota: 360\sect\sectd\pard West quota: 480\par}`,
335 "North quota: 120\nSouth quota: 240\nEast quota: 360\nWest quota: 480",
336 ],
337 ["group close", String.raw`{\rtf1\ansi{\u945}Z}`, "αZ"],
338 ["group open", String.raw`{\rtf1\ansi\u945{Z}}`, "αZ"],
339 ["hex byte", String.raw`{\rtf1\ansi\u945\'3fZ}`, "αZ"],
340 ["escaped symbol", String.raw`{\rtf1\ansi\u945\{Z}`, "αZ"],
341 ["Macintosh text", String.raw`{\rtf1\mac Caf\'8e Cr\'8fme}`, "Café Crème"],
342 [
343 "explicit Macintosh code page",
344 String.raw`{\rtf1\mac\ansicpg10000 Caf\'8e Cr\'8fme}`,
345 "Café Crème",
346 ],
347 ["destination symbol", String.raw`{\rtf1\ansi\u945\*Z}`, "αZ"],
348 ["multiple units", String.raw`{\rtf1\ansi\uc4\u945?\tab\'3f\~Z}`, "αZ"],
349 ["binary unit", String.raw`{\rtf1\ansi\u945\bin3 abcZ}`, "αZ"],
350 ["large fallback", String.raw`{\rtf1\ansi\uc20\u945????????????????????Z}`, "αZ"],
351 ["no fallback", String.raw`{\rtf1\ansi\uc0\u945Z}`, "αZ"],
352 ["scoped count", String.raw`{\rtf1\ansi{\uc2\u945??}\u946?Z}`, "αβZ"],
353 ] as const)("extracts RTF text: %s", ([name, source, expected]) =>
354 Effect.gen(function* () {
355 const config = yield* ServerConfig.ServerConfig;
356 const path = NodePath.join(config.attachmentsDir, `${name}.rtf`);
357 NodeFS.writeFileSync(path, source);
358 expect(yield* Effect.promise(() => extractDocumentText(path, ".rtf"))).toBe(expected);
359 expect(yield* (yield* DocumentText.DocumentText).ensure(path)).toBe(path + ".txt");
360 expect(NodeFS.readFileSync(path + ".txt", "utf8")).toBe(expected);
361 }).pipe(Effect.provide(layer)),
362 );
363
364 it.effect.each([false, true])("limits the child heap for standalone=%s", (standalone) =>
365 Effect.acquireUseRelease(
366 Effect.promise(async () => {
367 const { execFile } = await vi.importActual<typeof NodeChildProcess>("node:child_process");
368 const probe = vi.spyOn(NodeChildProcess, "execFile");
369 probe.mockImplementation(((
370 file: string,
371 _args: readonly string[],
372 options: NodeChildProcess.ExecFileOptionsWithStringEncoding,
373 callback: (
374 error: NodeChildProcess.ExecFileException | null,
375 stdout: string,
376 stderr: string,
377 ) => void,
378 ) =>
379 execFile(
380 file,
381 [
382 "-e",
383 'process.stdout.write(String(require("node:v8").getHeapStatistics().heap_size_limit))',
384 ],
385 options,
386 callback,
387 )) as typeof NodeChildProcess.execFile);
388 return probe;
389 }),
390 () =>
391 Effect.gen(function* () {
392 const config = yield* ServerConfig.ServerConfig;
393 const path = NodePath.join(config.attachmentsDir, "heap.rtf");
394 NodeFS.writeFileSync(path, String.raw`{\rtf1 hello}`);
395 const extracted = yield* Effect.gen(function* () {
396 return yield* (yield* DocumentText.DocumentText).ensure(path);
397 }).pipe(
398 Effect.provide(Layer.fresh(DocumentText.layer)),
399 Effect.provideService(HostProcess.IsExecutable, standalone),
400 Effect.provideService(HostProcess.Environment, {
401 ...process.env,
402 NODE_OPTIONS: "--max-old-space-size=1024",
403 }),
404 );
405 expect(extracted).toBe(path + ".txt");
406 const heapLimit = Number(NodeFS.readFileSync(path + ".txt", "utf8"));
407 expect(heapLimit).toBeGreaterThan(192 * 1024 * 1024);
408 // Newer V8 versions reserve up to 192 MiB for the young generation as well.
409 expect(heapLimit).toBeLessThanOrEqual(384 * 1024 * 1024);
410 }).pipe(Effect.provide(layer)),
411 (probe) => Effect.sync(() => probe.mockRestore()),
412 ),
413 );
414
415 it.effect("extracts notebook cells and outputs once and keeps both readable paths", () =>
416 Effect.gen(function* () {
417 const config = yield* ServerConfig.ServerConfig;
418 const file = {
419 ...attachment,
420 id: attachment.id.replace(/txt$/, "ipynb"),
421 name: "survey.ipynb",
422 mimeType: "application/x-ipynb+json",
423 };
424 const path = NodePath.join(config.attachmentsDir, `${file.id}.ipynb`);
425 const rawSource = String.raw`\begin{tabular}{lr} North & 120 \\ \end{tabular}`;
426 const notebook = JSON.stringify({
427 cells: [
428 { cell_type: "markdown", source: ["# Survey\n", "Vessel: Larkspur"] },
429 {
430 cell_type: "code",
431 source: ["print(43 * 7)"],
432 outputs: [
433 { output_type: "stream", name: "stdout", text: "\u001b[32m301\u001b[0m\n" },
434 {
435 output_type: "display_data",
436 data: { "text/plain": ["\u001b[", "33m42\u001b[0m\n", "Next line"] },
437 metadata: {},
438 },
439 {
440 output_type: "error",
441 ename: "ZeroDivisionError",
442 evalue: "division by zero",
443 traceback: [
444 "\u001b[32mCell In[1], line 1\u001b[0m",
445 "\u001b[31m----> 1 1 / 0\u001b[0m",
446 "ZeroDivisionError: division by zero",
447 ],
448 },
449 ],
450 },
451 { cell_type: "raw", source: [rawSource] },
452 ],
453 });
454 NodeFS.writeFileSync(path, notebook);
455 const answers = yield* appendUserInputAttachmentPaths({
456 answers: { q: "Read this" },
457 attachmentsDir: config.attachmentsDir,
458 attachmentsByQuestionId: { q: [file] },
459 });
460 expect(answers.q).toContain(`text extracted to: ${JSON.stringify(path + ".txt")}`);
461 expect(NodeFS.readFileSync(`${path}.txt`, "utf8")).toContain("Vessel: Larkspur");
462 const extracted = NodeFS.readFileSync(`${path}.txt`, "utf8");
463 expect(extracted).toContain("Output:\n301\n");
464 expect(extracted).toContain("42\nNext line");
465 expect(extracted).toContain(`## Cell 3 (raw)\n${rawSource}`);
466 expect(extracted).toContain(
467 "ZeroDivisionError: division by zero\nCell In[1], line 1\n----> 1 1 / 0\nZeroDivisionError: division by zero",
468 );
469 expect(extracted).not.toContain("\u001b");
470 expect(NodeFS.readFileSync(path, "utf8")).toBe(notebook);
471 const before = NodeFS.statSync(path + ".txt").mtimeMs;
472 // A changed binary would fail parsing if the cached extraction were not reused.
473 NodeFS.writeFileSync(path, "invalid notebook");
474 const documents = yield* DocumentText.DocumentText;
475 const results = yield* Effect.all([documents.ensure(path), documents.ensure(path)], {
476 concurrency: "unbounded",
477 });
478 expect(results).toEqual([path + ".txt", path + ".txt"]);
479 expect(NodeFS.statSync(path + ".txt").mtimeMs).toBe(before);
480 yield* Effect.gen(function* () {
481 const cleanup = yield* Cleanup.ResourceCleanupService;
482 yield* cleanup.cleanupAttachments([file.id]);
483 expect(NodeFS.existsSync(path)).toBe(false);
484 expect(NodeFS.existsSync(path + ".txt")).toBe(false);
485 NodeFS.writeFileSync(path + ".txt", "orphaned extraction");
486 yield* cleanup.cleanupAttachments([file.id]);
487 expect(NodeFS.existsSync(path + ".txt")).toBe(false);
488 }).pipe(
489 Effect.provide(
490 Cleanup.layer.pipe(
491 Layer.provide(
492 Layer.mergeAll(
493 Layer.mock(TerminalManager.TerminalManager)({}),
494 Layer.mock(PreviewManager.PreviewManager)({}),
495 Layer.mock(DeviceService.DeviceService)({}),
496 Layer.mock(ScheduledTaskService.ScheduledTaskService)({}),
497 ),
498 ),
499 ),
500 ),
501 );
502 }).pipe(Effect.provide(layer)),
503 );
504
505 it.effect(
506 "preserves corrupt documents, PDFs and unknown binaries without an extracted path",
507 () =>
508 Effect.gen(function* () {
509 const config = yield* ServerConfig.ServerConfig;
510 for (const extension of ["docx", "ipynb", "zip", "pdf"]) {
511 const file = {
512 ...attachment,
513 id: attachment.id.replace(/txt$/, extension),
514 name: `broken.${extension}`,
515 };
516 const path = NodePath.join(config.attachmentsDir, `${file.id}.${extension}`);
517 NodeFS.writeFileSync(path, "invalid document");
518 const answers = yield* appendUserInputAttachmentPaths({
519 answers: {},
520 attachmentsDir: config.attachmentsDir,
521 attachmentsByQuestionId: { q: [file] },
522 });
523 expect(answers.q).toContain(JSON.stringify(path));
524 expect(answers.q).not.toContain("text extracted");
525 expect(NodeFS.existsSync(`${path}.txt`)).toBe(false);
526 expect(NodeFS.readFileSync(path, "utf8")).toBe("invalid document");
527 }
528 }).pipe(Effect.provide(layer)),
529 );
530
531 it.effect("preserves office structure and EPUB reading order with bounded archives", () =>
532 Effect.gen(function* () {
533 const config = yield* ServerConfig.ServerConfig;
534 const cases = [
535 // Word stores object layout and field data outside the visible document text.
536 {
537 ext: "docx",
538 bounded: true,
539 parts: {
540 "word/document.xml":
541 '<w:document xmlns:w="urn:w" xmlns:wp="urn:wp" xmlns:wp14="urn:wp14" xmlns:mc="urn:mc"><w:body><w:p><w:r><w:t>Total cost is </w:t><w:drawing><wp:anchor><wp:positionH><wp:posOffset>1905000</wp:posOffset></wp:positionH><wp:positionV><wp:posOffset>-635</wp:posOffset></wp:positionV><wp14:sizeRelH><wp14:pctWidth>0</wp14:pctWidth></wp14:sizeRelH><wp14:sizeRelV><wp14:pctHeight>0</wp14:pctHeight></wp14:sizeRelV></wp:anchor></w:drawing><w:t>42 EUR.</w:t></w:r></w:p></w:body></w:document>',
542 },
543 expected: ["Total cost is 42 EUR."],
544 absent: ["1905000", "-635", "0042"],
545 },
546 {
547 ext: "docx",
548 bounded: true,
549 parts: {
550 "word/document.xml":
551 '<w:document xmlns:w="urn:w" xmlns:wp="urn:wp" xmlns:wp14="urn:wp14" xmlns:mc="urn:mc"><w:body><w:p><mc:AlternateContent><mc:Choice Requires="cx"><w:drawing><wp:anchor><wp:positionH><wp:posOffset>1905000</wp:posOffset></wp:positionH><wp:positionV><wp:posOffset>-635</wp:posOffset></wp:positionV><wp14:sizeRelH><wp14:pctWidth>0</wp14:pctWidth></wp14:sizeRelH><wp14:sizeRelV><wp14:pctHeight>0</wp14:pctHeight></wp14:sizeRelV></wp:anchor></w:drawing><cx:chart/></mc:Choice><mc:Fallback><w:pict><v:shape><v:textbox><w:txbxContent><w:p><w:r><w:t>Chart caption</w:t></w:r></w:p></w:txbxContent></v:textbox></v:shape></w:pict></mc:Fallback></mc:AlternateContent></w:p></w:body></w:document>',
552 },
553 expected: ["Chart caption"],
554 absent: ["1905000", "-635"],
555 },
556 {
557 ext: "docx",
558 bounded: true,
559 parts: {
560 "word/document.xml":
561 '<w:document xmlns:w="urn:w" xmlns:wp="urn:wp" xmlns:wp14="urn:wp14" xmlns:mc="urn:mc"><w:body><w:p><w:r><w:t>As shown </w:t><w:fldChar><w:fldData>RU5ETk9URQ==</w:fldData></w:fldChar><w:t>(Smith 2020) earlier.</w:t></w:r></w:p></w:body></w:document>',
562 },
563 expected: ["As shown (Smith 2020) earlier."],
564 absent: ["RU5ETk9URQ=="],
565 },
566 {
567 ext: "docx",
568 bounded: true,
569 parts: {
570 "word/document.xml": "<document><p>Readable despite embedded media</p></document>",
571 "word/media/black.bmp": "\0".repeat(64 * 1024),
572 "word/embeddings/unused.bin": "\0".repeat(33 * 1024 * 1024),
573 },
574 expected: ["Readable despite embedded media"],
575 absent: ["[Extracted text truncated"],
576 },
577 ...[false, true].map((deep) => ({
578 ext: "docx",
579 bounded: true,
580 parts: {
581 "word/document.xml": `<document>${deep ? "<div>".repeat(129) + "Lost" + "</div>".repeat(129) : "<p/>".repeat(200_001)}</document>`,
582 },
583 expected: ["[Extracted text truncated at document extraction limits.]"],
584 absent: ["Lost"],
585 })),
586 ...["ods", "odt", "odp", "docx"].flatMap((ext) =>
587 [false, true].map((rows) => ({
588 ext,
589 bounded: true,
590 parts: {
591 [ext === "docx" ? "word/document.xml" : "content.xml"]:
592 `<document><page><table><table-row${rows ? ' number-rows-repeated="10001"' : ""}><table-cell${rows ? "" : ' number-columns-repeated="513"'}><p>Repeated value</p></table-cell></table-row></table></page></document>`,
593 },
594 expected: [
595 "Repeated value",
596 "[Extracted text truncated at document extraction limits.]",
597 ],
598 occurrences: { text: "Repeated value", count: rows ? 10_000 : 512 },
599 })),
600 ),
601 ...["ods", "odt", "odp", "docx"].map((ext) => ({
602 ext,
603 bounded: true,
604 parts: {
605 [ext === "docx" ? "word/document.xml" : "content.xml"]:
606 `<document><page><table><table-row><table-cell><p>Kept</p></table-cell><table-cell number-columns-repeated="16383"> \n <p> \t </p> <s c="1048575"/> </table-cell></table-row><table-row number-rows-repeated="1048575"><table-cell> \n <p> \t </p> </table-cell></table-row></table></page></document>`,
607 },
608 expected: ["Kept"],
609 absent: ["[Extracted text truncated", "\t"],
610 })),
611 ...["ods", "odt", "odp"].map((ext) => ({
612 ext,
613 bounded: true,
614 parts: {
615 "content.xml": `<document><page><table><table-row>
616 <table-cell office:value-type="string" office:string-value="Harbor"/>
617 <table-cell office:value-type="boolean" office:boolean-value="false"/>
618 <table-cell office:value-type="date" office:date-value="2026-10-20"/>
619 <table-cell office:value-type="time" office:time-value="PT2H30M"/>
620 <table-cell office:value-type="currency" office:value="125.50" office:currency="USD"/>
621 <table-cell office:value-type="percentage" office:value="0.25"/>
622 <table-cell office:value-type="float" office:value="0"/>
623 <table-cell office:value-type="string" office:string-value="Fallback"><p> \n </p></table-cell>
624 <table-cell office:value-type="string" office:string-value="Unshown"><p>Displayed</p></table-cell>
625 <table-cell office:value-type="void" office:value="Unshown"/>
626 <table-cell office:value-type="currency" office:value=" " office:currency="Unshown"/>
627 <table-cell office:value-type="percentage" office:value=" "/>
628 </table-row></table></page></document>`,
629 },
630 expected: [
631 "Harbor\tfalse\t2026-10-20\tPT2H30M\t125.50 USD\t0.25 [ODF value-type=percentage]\t0\tFallback\tDisplayed",
632 ],
633 absent: ["Unshown", "[Extracted text truncated"],
634 })),
635 ...[false, true].map((rows) => ({
636 ext: "xlsx",
637 bounded: true,
638 parts: {
639 "xl/workbook.xml": '<workbook><sheet name="Limited" r:id="a"/></workbook>',
640 "xl/_rels/workbook.xml.rels":
641 '<Relationships><Relationship Id="a" Target="worksheets/a.xml"/></Relationships>',
642 "xl/worksheets/a.xml": `<worksheet>${rows ? "<row><c><v>7</v></c></row>".repeat(10_001) : '<row><c r="A1"><v>7</v></c><c r="SS1"><v>Beyond</v></c></row>'}</worksheet>`,
643 },
644 expected: ["7", "[Extracted text truncated at document extraction limits.]"],
645 absent: ["Beyond"],
646 })),
647 {
648 ext: "xlsx",
649 bounded: true,
650 parts: {
651 "xl/workbook.xml": '<workbook><sheet name="Padding" r:id="a"/></workbook>',
652 "xl/_rels/workbook.xml.rels":
653 '<Relationships><Relationship Id="a" Target="worksheets/a.xml"/></Relationships>',
654 "xl/sharedStrings.xml": "<sst><si><t> \n </t></si><si><t>Present</t></si></sst>",
655 "xl/worksheets/a.xml": `<worksheet>${'<row><c t="s"><v>0</v></c><c r="XFD1" t="inlineStr"><is><t> \t </t></is></c></row>'.repeat(10_001)}<row><c t="s"><v>0</v></c><c t="s"><v>1</v></c><c r="XFD2"><v> \n </v></c></row></worksheet>`,
656 },
657 expected: ["Present"],
658 absent: ["[Extracted text truncated"],
659 },
660 {
661 ext: "epub",
662 parts: {
663 "META-INF/container.xml": '<container><rootfile full-path="book.opf"/></container>',
664 "book.opf":
665 '<package><item id="bad" href="ch%ZZ.xhtml"/><item id="good" href="good.svg"/><itemref idref="bad"/><itemref idref="good"/></package>',
666 "good.svg":
667 '<svg xmlns="http://www.w3.org/2000/svg"><text>Readable SVG chapter</text></svg>',
668 },
669 expected: [
670 "Readable SVG chapter",
671 "[Skipped document part: chapter 1 (missing or invalid XML).]",
672 ],
673 absent: ["[Extracted text truncated"],
674 },
675 ...[false, true].flatMap((bom) =>
676 [false, true].map((bigEndian) => {
677 const encode = (source: string) => {
678 const data = Buffer.from(`${bom ? "\ufeff" : ""}${source}`, "utf16le");
679 return bigEndian ? data.swap16() : data;
680 };
681 return {
682 ext: "epub",
683 parts: {
684 "META-INF/container.xml": encode(
685 '<?xml version="1.0" encoding="UTF-16"?><container><rootfile full-path="book.opf"/></container>',
686 ),
687 "book.opf": encode(
688 '<?xml version="1.0" encoding="UTF-16"?><package><item id="a" href="a.xhtml"/><itemref idref="a"/></package>',
689 ),
690 "a.xhtml": encode(
691 '<?xml version="1.0" encoding="UTF-16"?><html><p>UTF-16 chapter: café 日本語</p></html>',
692 ),
693 },
694 expected: ["UTF-16 chapter: café 日本語"],
695 absent: ["[Skipped document part", "[Extracted text truncated"],
696 };
697 }),
698 ),
699 {
700 ext: "epub",
701 bounded: true,
702 parts: {
703 "META-INF/container.xml": '<container><rootfile full-path="book.opf"/></container>',
704 "book.opf":
705 '<package><item id="bad" href="bad.xhtml"/><item id="good" href="good.xhtml"/><itemref idref="bad"/><itemref idref="good"/></package>',
706 "bad.xhtml": "<html><p>Discard this broken chapter</html>",
707 "good.xhtml":
708 '<!DOCTYPE html PUBLIC "-//W3C//DTD XHTML 1.1//EN" "http://www.w3.org/TR/xhtml11/DTD/xhtml11.dtd"><html><p>North&nbsp;Harbor &NotEqualTilde; &CounterClockwiseContourIntegral; &amp; &lt;tag&gt; &quot;quoted&quot;</p><p><![CDATA[Literal &nbsp;]]></p><table><tr><td>&nbsp; </td></tr><tr><td>Kept&nbsp;cell</td></tr></table></html>',
709 },
710 expected: [
711 "## Chapter 2",
712 'North\u00a0Harbor ≂̸ ∳ & <tag> "quoted"',
713 "Literal &nbsp;",
714 "Kept cell",
715 "[Skipped document part: bad.xhtml (missing or invalid XML).]",
716 ],
717 absent: ["Discard this broken chapter", "[Extracted text truncated"],
718 },
719 ...["pptx", "xlsx"].map((ext) => ({
720 ext,
721 bounded: true,
722 parts:
723 ext === "pptx"
724 ? {
725 "ppt/presentation.xml":
726 '<presentation><sldIdLst><sldId r:id="bad"/><sldId r:id="good"/></sldIdLst></presentation>',
727 "ppt/_rels/presentation.xml.rels":
728 '<Relationships><Relationship Id="bad" Target="bad.xml"/><Relationship Id="good" Target="good.xml"/></Relationships>',
729 "ppt/bad.xml": "<slide><p>Discard</slide>",
730 "ppt/good.xml": "<slide><p>Healthy part</p></slide>",
731 }
732 : {
733 "xl/workbook.xml":
734 '<workbook><sheet name="Bad" r:id="bad"/><sheet name="Good" r:id="good"/></workbook>',
735 "xl/_rels/workbook.xml.rels":
736 '<Relationships><Relationship Id="bad" Target="bad.xml"/><Relationship Id="good" Target="good.xml"/></Relationships>',
737 "xl/bad.xml": "<worksheet><row>Discard</worksheet>",
738 "xl/good.xml":
739 '<worksheet><row><c t="inlineStr"><is><t>Healthy part</t></is></c></row></worksheet>',
740 },
741 expected: [
742 "Healthy part",
743 `[Skipped document part: ${ext === "pptx" ? "ppt" : "xl"}/bad.xml (missing or invalid XML).]`,
744 ],
745 absent: ["Discard", "[Extracted text truncated"],
746 })),
747 {
748 ext: "odt",
749 bounded: true,
750 parts: { "content.xml": '<document><p>Start<s c="1000001"/>Unneeded</p></document>' },
751 expected: ["Start", "[Extracted text truncated at document extraction limits.]"],
752 absent: ["Unneeded"],
753 },
754 {
755 ext: "docx",
756 bounded: true,
757 parts: {
758 "word/document.xml":
759 '<w:document xmlns:w="urn:w" xmlns:mc="urn:mc"><w:body><w:p><w:r><w:t>Body</w:t></w:r></w:p><w:p><w:r><mc:AlternateContent><mc:Choice Requires="wps"><w:drawing><wps:wsp><wps:txbx><w:txbxContent><w:p><w:r><w:t>Callout box</w:t></w:r></w:p></w:txbxContent></wps:txbx></wps:wsp></w:drawing></mc:Choice><mc:Fallback><w:pict><v:shape><v:textbox><w:txbxContent><w:p><w:r><w:t>Callout box</w:t></w:r></w:p></w:txbxContent></v:textbox></v:shape></w:pict></mc:Fallback></mc:AlternateContent></w:r></w:p></w:body></w:document>',
760 },
761 expected: ["Body\nCallout box"],
762 occurrences: { text: "Callout box", count: 1 },
763 },
764 {
765 ext: "docx",
766 bounded: true,
767 parts: {
768 "word/document.xml":
769 '<w:document xmlns:w="urn:w" xmlns:mc="urn:mc"><w:body><w:p><w:r><mc:AlternateContent><mc:Choice Requires="cx"><w:drawing><cx:chart/></w:drawing></mc:Choice><mc:Fallback><w:pict><v:shape><v:textbox><w:txbxContent><w:p><w:r><w:t>Chart caption</w:t></w:r></w:p></w:txbxContent></v:textbox></v:shape></w:pict></mc:Fallback></mc:AlternateContent></w:r></w:p></w:body></w:document>',
770 },
771 expected: ["Chart caption"],
772 },
773 {
774 ext: "docx",
775 bounded: true,
776 parts: {
777 "word/document.xml":
778 '<w:document xmlns:w="urn:w"><w:body><w:p><w:pPr><w:pStyle w:val="Heading2"/></w:pPr><w:r><w:t>Report &amp; totals</w:t></w:r></w:p><w:tbl><w:tr><w:tc><w:p><w:r><w:t>North</w:t></w:r></w:p></w:tc><w:tc><w:p><w:r><w:t>12</w:t></w:r></w:p><w:p><w:r><w:t>34</w:t></w:r></w:p></w:tc></w:tr><w:tr><w:tc><w:p><w:r><w:t>Before</w:t></w:r></w:p><w:tbl><w:tr><w:tc><w:p><w:r><w:t>Inner</w:t></w:r></w:p></w:tc></w:tr></w:tbl><w:p><w:r><w:t>After</w:t></w:r></w:p></w:tc><w:tc><w:p><w:r><w:t>Neighbor</w:t></w:r></w:p></w:tc><w:tc><w:p><w:r><w:noBreakHyphen/></w:r></w:p></w:tc></w:tr><w:tr><w:tc><w:p><w:r><w:noBreakHyphen/></w:r></w:p></w:tc></w:tr></w:tbl></w:body></w:document>',
779 },
780 expected: [
781 "## Report & totals",
782 "North\t12 34",
783 "Before Inner After\tNeighbor\t\u2011\n\u2011",
784 ],
785 },
786 {
787 ext: "docx",
788 bounded: true,
789 parts: {
790 "word/document.xml": [
791 '<w:document xmlns:w="urn:w" xmlns:mc="urn:mc"><w:body>',
792 '<w:p><w:r><w:t>First</w:t><w:cr/><w:t>Second</w:t><w:br/><w:t>Third</w:t><w:noBreakHyphen/><w:t>party</w:t><mc:AlternateContent><mc:Choice Requires="w"><w:noBreakHyphen/></mc:Choice><mc:Fallback><w:t>‑</w:t></mc:Fallback></mc:AlternateContent><w:t>End</w:t></w:r></w:p>',
793 '<w:p><w:r><w:t>Invoice 104</w:t><w:ptab w:alignment="right" w:relativeTo="margin" w:leader="none"/><w:t>$700.00</w:t></w:r></w:p>',
794 '<w:p><w:r><w:t>Invoice 105</w:t><mc:AlternateContent><mc:Choice Requires="w"><w:ptab w:alignment="right" w:relativeTo="margin" w:leader="none"/></mc:Choice><mc:Fallback><w:tab/></mc:Fallback></mc:AlternateContent><w:t>$800.00</w:t></w:r></w:p>',
795 '<w:p><w:r><w:t>Invoice 106</w:t><mc:AlternateContent><mc:Choice Requires="w"><w:ptab w:alignment="right" w:relativeTo="margin" w:leader="none"/></mc:Choice><mc:Fallback><w:t xml:space="preserve"> total </w:t></mc:Fallback></mc:AlternateContent><w:t>$900.00</w:t></w:r></w:p>',
796 "</w:body></w:document>",
797 ].join(""),
798 },
799 expected: [
800 "First\nSecond\nThird\u2011party\u2011End\nInvoice 104\t$700.00\nInvoice 105\t$800.00\nInvoice 106 total $900.00",
801 ],
802 },
803 {
804 ext: "docx",
805 bounded: true,
806 parts: {
807 "word/document.xml":
808 '<w:document xmlns:w="urn:w"><w:body><w:p><w:r><w:t>Intro</w:t></w:r><w:r><w:fldChar w:fldCharType="begin"/></w:r><w:r><w:instrText> PAGEREF _Toc1 \\h </w:instrText></w:r><w:r><w:fldChar w:fldCharType="separate"/></w:r><w:r><w:t> 3</w:t></w:r><w:r><w:fldChar w:fldCharType="end"/></w:r></w:p><w:p><w:pPr><w:tabs><w:tab w:val="right" w:pos="9350"/></w:tabs></w:pPr><w:r><w:t xml:space="preserve">Fee </w:t></w:r><w:del><w:r><w:delText>$500</w:delText><w:noBreakHyphen/></w:r></w:del><w:ins><w:r><w:t>$700</w:t></w:r></w:ins><w:moveFrom><w:r><w:t>Moved</w:t></w:r></w:moveFrom><w:r><w:tab/><w:t>due</w:t></w:r></w:p></w:body></w:document>',
809 },
810 expected: ["Intro 3\nFee $700\tdue"],
811 absent: ["PAGEREF", "$500", "Moved", "\u2011"],
812 },
813 {
814 ext: "odt",
815 bounded: true,
816 parts: {
817 "content.xml":
818 "<document><text><tracked-changes><changed-region><deletion><change-info><creator>Reviewer</creator></change-info><p>$500</p></deletion></changed-region></tracked-changes><p>Fee <change/>$700</p></text></document>",
819 },
820 expected: ["Fee $700"],
821 absent: ["Reviewer", "$500"],
822 },
823 {
824 ext: "odt",
825 bounded: true,
826 parts: {
827 "content.xml":
828 '<document><text><p>Fee <office:annotation><dc:creator>Reviewer</dc:creator><dc:date>2026-01-01T00:00:00</dc:date><initials>JR</initials><p>Check</p></office:annotation>$700</p><table><table-row><table-cell office:value-type="float" office:value="150"><office:annotation><p>was 120</p></office:annotation></table-cell></table-row></table><p>Formula <math:math><math:semantics><math:mrow>x+y</math:mrow><math:annotation encoding="application/x-tex">x+y</math:annotation></math:semantics></math:math></p></text></document>',
829 },
830 expected: ["Fee $700\n150\nFormula x+yx+y"],
831 absent: ["Reviewer", "2026-01-01", "JR", "Check", "was 120"],
832 },
833 {
834 ext: "docx",
835 bounded: true,
836 parts: {
837 "word/document.xml": `<w:document><w:body><w:tbl><w:tr><w:tc><w:p><w:r><w:t>${"A".repeat(1_000_000)}</w:t></w:r></w:p></w:tc><w:tc><w:p><w:r><w:t>Later cell</w:t></w:r></w:p></w:tc></w:tr></w:tbl></w:body></w:document>`,
838 },
839 expected: ["A".repeat(32), "[Extracted text truncated at document extraction limits.]"],
840 },
841 {
842 ext: "xlsx",
843 bounded: true,
844 parts: {
845 "xl/workbook.xml": '<workbook><sheet name="Formulas" r:id="a"/></workbook>',
846 "xl/_rels/workbook.xml.rels":
847 '<Relationships><Relationship Id="a" Target="worksheets/a.xml"/></Relationships>',
848 "xl/styles.xml": '<styleSheet><cellXfs><xf numFmtId="9"/></cellXfs></styleSheet>',
849 "xl/worksheets/a.xml":
850 "<worksheet><row><c><f>SUM(11,22)</f><v/></c></row><row><c><f>SUM(33,44)</f><v> \n </v></c></row><row><c><f>SUM(55,66)</f></c></row><row><c><f>SUM(11,22)</f><v>33</v></c><c><f>SUM(0,0)</f><v>0</v></c></row></worksheet>",
851 },
852 expected: [
853 "SUM(11,22)\nSUM(33,44)\nSUM(55,66)",
854 "33 [Excel numFmtId=9; date system 1900]\t0 [Excel numFmtId=9; date system 1900]",
855 ],
856 absent: ["SUM(0,0)", "[Extracted text truncated"],
857 },
858 {
859 ext: "xlsx",
860 bounded: true,
861 parts: {
862 "xl/workbook.xml": '<workbook><sheet name="Flags" r:id="a"/></workbook>',
863 "xl/_rels/workbook.xml.rels":
864 '<Relationships><Relationship Id="a" Target="worksheets/a.xml"/></Relationships>',
865 "xl/worksheets/a.xml":
866 '<worksheet><row><c r="A1" t="b"><v>1</v></c><c r="B1" t="b"><v>0</v></c><c r="C1" t="n"><v>1</v></c><c r="D1" t="b"><v>2</v></c></row></worksheet>',
867 },
868 expected: ["true\tfalse\t1\t2"],
869 },
870 {
871 ext: "xlsx",
872 parts: {
873 "xl/workbook.xml":
874 '<workbook><sheets><sheet name="Inventory" r:id="a"/><sheet name="Budget" r:id="b"/></sheets></workbook>',
875 "xl/_rels/workbook.xml.rels":
876 '<Relationships><Relationship Id="a" Target="worksheets/second.xml"/><Relationship Id="b" Target="worksheets/first.xml"/></Relationships>',
877 "xl/sharedStrings.xml": "<sst><si><t>Lantern</t></si></sst>",
878 "xl/styles.xml":
879 '<styleSheet><numFmts><numFmt numFmtId="164" formatCode="yyyy-mm-dd"/></numFmts><cellXfs><xf numFmtId="0"/><xf numFmtId="14"/><xf numFmtId="9"/><xf numFmtId="164"/></cellXfs></styleSheet>',
880 "xl/worksheets/second.xml":
881 '<worksheet><sheetData><row><c r="A1" t="s"><v>0</v></c><c r="C1"><v>37</v></c></row></sheetData></worksheet>',
882 "xl/worksheets/first.xml":
883 '<worksheet><sheetData><row><c t="inlineStr"><is><t>Ferry</t></is></c><c><v>628</v></c><c s="1"><v>45292</v></c><c s="2"><v>0.25</v></c><c s="3"><v>45293</v></c></row></sheetData></worksheet>',
884 },
885 expected: [
886 "## Sheet: Inventory",
887 "Lantern\t\t37",
888 "## Sheet: Budget",
889 "Ferry\t628",
890 "45292 [Excel numFmtId=14; date system 1900]",
891 "0.25 [Excel numFmtId=9; date system 1900]",
892 "45293 [Excel numFmtId=164; format=yyyy-mm-dd; date system 1900]",
893 ],
894 },
895 {
896 ext: "xlsx",
897 bounded: true,
898 parts: {
899 "xl/workbook.xml": '<workbook><sheet name="Phonetics" r:id="a"/></workbook>',
900 "xl/_rels/workbook.xml.rels":
901 '<Relationships><Relationship Id="a" Target="worksheets/a.xml"/></Relationships>',
902 "xl/sharedStrings.xml":
903 '<sst><si><t>東京</t><rPh sb="0" eb="2"><t>とうきょう</t></rPh></si></sst>',
904 "xl/worksheets/a.xml":
905 '<worksheet><row><c t="s"><v>0</v></c><c t="inlineStr"><is><t>東京</t><rPh sb="0" eb="2"><t>とうきょう</t></rPh></is></c></row></worksheet>',
906 },
907 expected: ["## Sheet: Phonetics", "東京\t東京"],
908 absent: ["とうきょう"],
909 },
910 {
911 ext: "xlsx",
912 bounded: true,
913 parts: {
914 "xl/workbook.xml": '<workbook><sheet name="Escapes" r:id="a"/></workbook>',
915 "xl/_rels/workbook.xml.rels":
916 '<Relationships><Relationship Id="a" Target="worksheets/a.xml"/></Relationships>',
917 "xl/sharedStrings.xml":
918 "<sst><si><t>first_x000D_\nsecond</t></si><si><t>_x005F_x0041_</t></si><si><t>_x005F_x005F_x0041_</t></si><si><r><t>Bold_x000D_</t></r><r><t>_x005F_x000D_</t></r></si><si><t>&amp;_xnothex_ cap_X0041_ _x000d_ tail</t></si></sst>",
919 "xl/worksheets/a.xml":
920 '<worksheet><row><c t="s"><v>0</v></c><c t="s"><v>1</v></c><c t="s"><v>2</v></c><c t="s"><v>3</v></c><c t="s"><v>4</v></c></row><row><c t="inlineStr"><is><t>_x005F_x005F_</t></is></c><c t="inlineStr"><is><t>first_x000D_second</t></is></c><c t="str"><f>A1</f><v>Total_x000D_due</v></c></row></worksheet>',
921 },
922 expected: [
923 "## Sheet: Escapes\nfirst second\t_x0041_\t_x005F_x0041_\tBold _x000D_\t&_xnothex_ cap_X0041_ tail\n_x005F_\tfirst second\tTotal due",
924 ],
925 },
926 {
927 ext: "xlsx",
928 bounded: true,
929 parts: {
930 "xl/workbook.xml":
931 '<workbook><workbookPr date1904="true"/><sheets><sheet name="Dates" r:id="a"/></sheets></workbook>',
932 "xl/_rels/workbook.xml.rels":
933 '<Relationships><Relationship Id="a" Target="worksheets/dates.xml"/></Relationships>',
934 "xl/styles.xml": '<styleSheet><cellXfs><xf numFmtId="14"/></cellXfs></styleSheet>',
935 "xl/worksheets/dates.xml":
936 "<worksheet><sheetData><row><c><v>1</v></c></row></sheetData></worksheet>",
937 },
938 expected: ["## Sheet: Dates", "1 [Excel numFmtId=14; date system 1904]"],
939 },
940 {
941 ext: "xlsx",
942 bounded: true,
943 parts: {
944 "xl/workbook.xml":
945 '<workbook><sheets><sheet name="Repeated" r:id="a"/><sheet name="Unneeded" r:id="missing"/></sheets></workbook>',
946 "xl/_rels/workbook.xml.rels":
947 '<Relationships><Relationship Id="a" Target="worksheets/repeated.xml"/></Relationships>',
948 "xl/sharedStrings.xml": `<sst><si><t>${"A".repeat(1_000_000 - "## Sheet: Repeated\n".length)}</t></si></sst>`,
949 "xl/worksheets/repeated.xml": `<worksheet><sheetData>${'<row><c t="s"><v>0</v></c><c t="s"><v>0</v></c></row>'.repeat(1_500)}</sheetData></worksheet>`,
950 },
951 expected: [
952 "## Sheet: Repeated",
953 "A".repeat(32),
954 "[Extracted text truncated at document extraction limits.]",
955 ],
956 },
957 {
958 ext: "pptx",
959 bounded: true,
960 parts: {
961 "ppt/presentation.xml": `<p:presentation xmlns:p="http://schemas.openxmlformats.org/presentationml/2006/main"
962 xmlns:r="http://schemas.openxmlformats.org/officeDocument/2006/relationships"
963 xmlns:p14="http://schemas.microsoft.com/office/powerpoint/2010/main">
964 <p:sldIdLst><p:sldId id="256" r:id="b"/><p:sldId id="257" r:id="a"/></p:sldIdLst>
965 <p:extLst><p:ext uri="{521415D9-36F7-43E2-AB2F-B90AF26B5E84}"><p14:sectionLst>
966 <p14:section name="Schedule" id="{01F07B81-39E6-4BBB-9B89-66EA253FBD29}">
967 <p14:sldIdLst><p14:sldId id="256"/><p14:sldId id="257"/></p14:sldIdLst>
968 </p14:section>
969 </p14:sectionLst></p:ext></p:extLst>
970 </p:presentation>`,
971 "ppt/_rels/presentation.xml.rels":
972 '<Relationships><Relationship Id="a" Target="slides/slide1.xml"/><Relationship Id="b" Target="slides/slide2.xml"/></Relationships>',
973 "ppt/slides/slide1.xml": "<p:sld><a:p><a:r><a:t>Departure</a:t></a:r></a:p></p:sld>",
974 "ppt/slides/slide2.xml": "<p:sld><a:p><a:r><a:t>Juniper Pier</a:t></a:r></a:p></p:sld>",
975 "ppt/slides/_rels/slide2.xml.rels":
976 '<Relationships><Relationship Type="http://schemas.openxmlformats.org/officeDocument/2006/relationships/slideLayout" Target="../notesSlides/ignored.xml"/><Relationship Type="http://schemas.openxmlformats.org/officeDocument/2006/relationships/notesSlide" TargetMode="External" Target="https://example.com/notes.xml"/><Relationship Type="http://schemas.openxmlformats.org/officeDocument/2006/relationships/notesSlide" Target="../notesSlides/notes7.xml"/></Relationships>',
977 "ppt/slides/_rels/slide1.xml.rels":
978 '<Relationships><Relationship Type="http://purl.oclc.org/ooxml/officeDocument/relationships/notesSlide" Target="/ppt/notesSlides/notes3.xml"/></Relationships>',
979 "ppt/notesSlides/notes7.xml":
980 "<p:notes><a:p><a:r><a:t>evacuation code ORCHID-81</a:t></a:r></a:p></p:notes>",
981 "ppt/notesSlides/notes3.xml":
982 "<p:notes><a:p><a:r><a:t>Board at gate 4</a:t></a:r></a:p></p:notes>",
983 "ppt/notesSlides/ignored.xml": "<p:notes><a:p>Unrelated notes</a:p></p:notes>",
984 },
985 expected: [
986 "## Slide 1",
987 "Juniper Pier\nNotes:\nevacuation code ORCHID-81",
988 "## Slide 2",
989 "Departure\nNotes:\nBoard at gate 4",
990 ],
991 occurrences: { text: "## Slide ", count: 2 },
992 absent: ["Unrelated notes", "[Skipped document part", "[Extracted text truncated"],
993 },
994 ...[false, true].map((missing) => ({
995 ext: "pptx",
996 bounded: true,
997 parts: {
998 "ppt/presentation.xml":
999 '<presentation><sldIdLst><sldId r:id="a"/></sldIdLst></presentation>',
1000 "ppt/_rels/presentation.xml.rels":
1001 '<Relationships><Relationship Id="a" Target="slides/a.xml"/></Relationships>',
1002 "ppt/slides/a.xml": "<slide><p>Slide text</p></slide>",
1003 "ppt/slides/_rels/a.xml.rels": `<Relationships><Relationship Type="http://schemas.openxmlformats.org/officeDocument/2006/relationships/notesSlide" Target="../notesSlides/${missing ? "missing" : "bad"}.xml"/></Relationships>`,
1004 "ppt/notesSlides/bad.xml": "<notes><p>Discard broken notes</notes>",
1005 },
1006 expected: [
1007 "Slide text",
1008 `[Skipped document part: ppt/notesSlides/${missing ? "missing" : "bad"}.xml (missing or invalid XML).]`,
1009 ],
1010 absent: ["Discard broken notes", "[Extracted text truncated"],
1011 })),
1012 {
1013 ext: "pptx",
1014 bounded: true,
1015 parts: {
1016 "ppt/presentation.xml":
1017 '<presentation><sldIdLst><sldId r:id="a"/><sldId r:id="b"/></sldIdLst></presentation>',
1018 "ppt/_rels/presentation.xml.rels":
1019 '<Relationships><Relationship Id="a" Target="slides/a.xml"/><Relationship Id="b" Target="slides/b.xml"/></Relationships>',
1020 "ppt/slides/a.xml": `<slide><p>${"A".repeat(600_000)}</p></slide>`,
1021 "ppt/slides/b.xml": "<slide><p>Unneeded slide</p></slide>",
1022 "ppt/slides/_rels/a.xml.rels":
1023 '<Relationships><Relationship Type="http://schemas.openxmlformats.org/officeDocument/2006/relationships/notesSlide" Target="../notesSlides/a.xml"/></Relationships>',
1024 "ppt/notesSlides/a.xml": `<notes><p>evacuation code ORCHID-81 ${"B".repeat(600_000)} Unneeded notes</p></notes>`,
1025 },
1026 expected: [
1027 "## Slide 1",
1028 "A".repeat(32),
1029 "Notes:\nevacuation code ORCHID-81",
1030 "B".repeat(32),
1031 "[Extracted text truncated at document extraction limits.]",
1032 ],
1033 absent: ["Unneeded notes", "Unneeded slide", "## Slide 2"],
1034 },
1035 {
1036 ext: "odt",
1037 parts: {
1038 "content.xml":
1039 '<office:document><text:h text:outline-level="2">\n Orchard\n</text:h><text:p>19<text:s text:c="3"/>violet<text:s/>crates <text:span>\n\t shipped</text:span> today<text:tab/>Dock<text:line-break/>Gate\u00a0B </text:p><text:p>Second</text:p><text:p>A<text:span><text:script>console.log(1);\n</text:script></text:span> B</text:p></office:document>',
1040 },
1041 expected: [
1042 "## Orchard\n19 violet crates shipped today\tDock\nGate\u00a0B\nSecond\nA B",
1043 ],
1044 },
1045 {
1046 ext: "ods",
1047 parts: {
1048 "content.xml":
1049 '<office:document><table:table table:name="Cargo"><table:table-row><table:table-cell><text:p>Violet</text:p></table:table-cell><table:table-cell office:value="19"/><table:table-cell office:value="27"><text:p/></table:table-cell></table:table-row></table:table></office:document>',
1050 },
1051 expected: ["## Sheet: Cargo", "Violet\t19\t27"],
1052 },
1053 {
1054 ext: "ods",
1055 bounded: true,
1056 parts: {
1057 "content.xml":
1058 '<office:document><table:table table:name="Padded rows"><table:table-row><table:table-cell><text:p>Left</text:p></table:table-cell><table:table-cell table:number-columns-repeated="2"/><table:table-cell><text:p>Right</text:p></table:table-cell></table:table-row><table:table-row table:number-rows-repeated="1048575"><table:table-cell/></table:table-row></table:table></office:document>',
1059 },
1060 expected: ["## Sheet: Padded rows", "Left\t\t\tRight"],
1061 absent: ["[Extracted text truncated"],
1062 },
1063 {
1064 ext: "ods",
1065 bounded: true,
1066 parts: {
1067 "content.xml":
1068 '<office:document><table:table table:name="Padded"><table:table-row><table:table-cell><text:p>Populated</text:p></table:table-cell><table:table-cell table:number-columns-repeated="16383"/></table:table-row><table:table-row table:number-rows-repeated="1048575"><table:table-cell table:number-columns-repeated="16384"><text:p/></table:table-cell></table:table-row></table:table></office:document>',
1069 },
1070 expected: ["## Sheet: Padded", "Populated"],
1071 absent: ["[Extracted text truncated", "\t"],
1072 },
1073 {
1074 ext: "ods",
1075 bounded: true,
1076 parts: {
1077 "content.xml": `<office:document><table:table table:name="Repeated"><table:table-row><table:table-cell table:number-columns-repeated="512"><text:p>Lead ${"a".repeat(600_000)} Tail</text:p></table:table-cell></table:table-row></table:table></office:document>`,
1078 },
1079 expected: [
1080 "## Sheet: Repeated",
1081 "Lead ",
1082 " Tail\tLead ",
1083 "[Extracted text truncated at document extraction limits.]",
1084 ],
1085 },
1086 {
1087 ext: "odp",
1088 parts: {
1089 "content.xml":
1090 "<office:document><draw:page><draw:frame><draw:text-box><text:p>\n First\n\t slide </text:p></draw:text-box></draw:frame></draw:page><draw:page><text:p>Second</text:p></draw:page></office:document>",
1091 },
1092 expected: ["## Slide 1\nFirst slide\n\n## Slide 2\nSecond"],
1093 },
1094 {
1095 ext: "ods",
1096 bounded: true,
1097 parts: {
1098 "content.xml": `<office:document>${Array.from(
1099 { length: 256 },
1100 (_, index) =>
1101 `<table:table table:name="${index === 0 ? "Shared" : "Unneeded"}"><table:table-row><table:table-cell table:number-columns-repeated="512"><text:p>${"B".repeat(4096)}</text:p></table:table-cell></table:table-row></table:table>`,
1102 ).join("")}</office:document>`,
1103 },
1104 expected: [
1105 "## Sheet: Shared",
1106 "B".repeat(32),
1107 "[Extracted text truncated at document extraction limits.]",
1108 ],
1109 absent: ["## Sheet: Unneeded"],
1110 },
1111 {
1112 ext: "odt",
1113 bounded: true,
1114 parts: {
1115 "content.xml": `<office:document><text:p>Before whitespace</text:p><table:table><table:table-row table:number-rows-repeated="10000"><table:table-cell table:number-columns-repeated="512"><text:p>${" ".repeat(10_000)}</text:p></table:table-cell></table:table-row></table:table><text:p>Unneeded</text:p></office:document>`,
1116 },
1117 expected: ["Before whitespace", "Unneeded"],
1118 absent: ["[Extracted text truncated"],
1119 },
1120 ...["odt", "odp", "ods"].flatMap((ext) =>
1121 [false, true].map((rows) => ({
1122 ext,
1123 bounded: true,
1124 parts: {
1125 "content.xml": `<office:document><draw:page>${Array.from(
1126 { length: 256 },
1127 () =>
1128 `<table:table><table:table-row${rows ? ' table:number-rows-repeated="10000"' : ""}><table:table-cell table:number-columns-repeated="512"><text:p>${"Budget ".repeat(rows ? 160 : 585)}</text:p></table:table-cell></table:table-row></table:table>`,
1129 ).join(
1130 "",
1131 )}</draw:page><draw:page><text:p>Unneeded</text:p></draw:page></office:document>`,
1132 },
1133 expected: ["Budget ", "[Extracted text truncated at document extraction limits.]"],
1134 absent: ["Unneeded"],
1135 })),
1136 ),
1137 {
1138 ext: "docx",
1139 bounded: true,
1140 parts: {
1141 "word/document.xml": `<w:document><w:p><w:t>${"A".repeat(1_000_100)}</w:t></w:p><w:p><w:t>Unneeded</w:t></w:p></w:document>`,
1142 },
1143 expected: ["A".repeat(32), "[Extracted text truncated at document extraction limits.]"],
1144 absent: ["Unneeded"],
1145 },
1146 {
1147 ext: "pptx",
1148 bounded: true,
1149 parts: {
1150 "ppt/presentation.xml":
1151 '<p:presentation><p:sldIdLst><p:sldId r:id="a"/><p:sldId r:id="missing"/></p:sldIdLst></p:presentation>',
1152 "ppt/_rels/presentation.xml.rels":
1153 '<Relationships><Relationship Id="a" Target="slides/slide.xml"/></Relationships>',
1154 "ppt/slides/slide.xml": `<p:sld><a:p><a:t>${"A".repeat(1_000_100)}</a:t></a:p></p:sld>`,
1155 },
1156 expected: [
1157 "## Slide 1",
1158 "A".repeat(32),
1159 "[Extracted text truncated at document extraction limits.]",
1160 ],
1161 absent: ["## Slide 2"],
1162 },
1163 {
1164 ext: "epub",
1165 bounded: true,
1166 parts: {
1167 "META-INF/container.xml": '<container><rootfile full-path="book.opf"/></container>',
1168 "book.opf":
1169 '<package><item id="a" href="a.xhtml"/><itemref idref="a"/><itemref idref="missing"/></package>',
1170 "a.xhtml": `<html><body><p>${"A".repeat(1_000_100)}</p></body></html>`,
1171 },
1172 expected: [
1173 "## Chapter 1",
1174 "A".repeat(32),
1175 "[Extracted text truncated at document extraction limits.]",
1176 ],
1177 absent: ["## Chapter 2"],
1178 },
1179 {
1180 ext: "epub",
1181 parts: {
1182 "META-INF/container.xml":
1183 '<container><rootfile full-path="OEBPS/book.opf"/></container>',
1184 "OEBPS/book.opf":
1185 '<package><manifest><item id="a" href="a.xhtml"/><item id="b" href="b.xhtml"/></manifest><spine><itemref idref="b"/><itemref idref="a"/></spine></package>',
1186 "OEBPS/a.xhtml":
1187 '<!DOCTYPE html PUBLIC "-//W3C//DTD XHTML 1.1//EN" "http://www.w3.org/TR/xhtml11/DTD/xhtml11.dtd"><html><body><p>Last chapter</p><p>Price <s>10</s> 8 <del>old</del></p></body></html>',
1188 "OEBPS/b.xhtml":
1189 '<!DOCTYPE html><html><head><title>My Novel</title><meta name="author" content="Hidden Author"/></head><body><h1>First chapter</h1><p>Larkspur</p><div>Direct div<section>Direct section<li>Direct item <em>emphasis</em> trailing</li></section></div><div>Account: A<br/>Balance: 731<line-break/>Currency: USD</div><table><thead><tr><th>Heading</th><th>Total</th></tr></thead><tbody><tr><td>Important value</td><td>731</td></tr><tr><td>Before<table><tr><td>Inner</td></tr></table>After</td><td>Neighbor</td></tr></tbody></table></body></html>',
1190 },
1191 expected: [
1192 "## Chapter 1",
1193 "\n# First chapter",
1194 "Larkspur",
1195 "Direct div",
1196 "Direct section",
1197 "Direct item emphasis trailing",
1198 "Account: A\nBalance: 731\nCurrency: USD",
1199 "Heading\tTotal",
1200 "Important value\t731",
1201 "Before Inner After\tNeighbor",
1202 "## Chapter 2",
1203 "Last chapter",
1204 "Price 10 8 old",
1205 ],
1206 absent: ["My Novel", "Hidden Author"],
1207 },
1208 ];
1209 for (const [caseIndex, item] of cases.entries()) {
1210 const local: Buffer[] = [];
1211 const central: Buffer[] = [];
1212 let offset = 0;
1213 for (const [name, content] of Object.entries(item.parts)) {
1214 const data = Buffer.from(content!);
1215 const compressed = /\.(bmp|bin)$/.test(name) ? NodeZlib.deflateRawSync(data) : data;
1216 const filename = Buffer.from(name);
1217 const header = Buffer.alloc(30);
1218 header.writeUInt32LE(0x04034b50);
1219 header.writeUInt16LE(20, 4);
1220 header.writeUInt16LE(compressed === data ? 0 : 8, 8);
1221 header.writeUInt32LE(NodeZlib.crc32(data), 14);
1222 header.writeUInt32LE(compressed.length, 18);
1223 header.writeUInt32LE(data.length, 22);
1224 header.writeUInt16LE(filename.length, 26);
1225 const directory = Buffer.alloc(46);
1226 directory.writeUInt32LE(0x02014b50);
1227 directory.writeUInt16LE(20, 4);
1228 directory.writeUInt16LE(20, 6);
1229 directory.writeUInt16LE(compressed === data ? 0 : 8, 10);
1230 directory.writeUInt32LE(NodeZlib.crc32(data), 16);
1231 directory.writeUInt32LE(compressed.length, 20);
1232 directory.writeUInt32LE(data.length, 24);
1233 directory.writeUInt16LE(filename.length, 28);
1234 directory.writeUInt32LE(offset, 42);
1235 local.push(header, filename, compressed);
1236 central.push(directory, filename);
1237 offset += header.length + filename.length + compressed.length;
1238 }
1239 const end = Buffer.alloc(22);
1240 end.writeUInt32LE(0x06054b50);
1241 end.writeUInt16LE(central.length / 2, 8);
1242 end.writeUInt16LE(central.length / 2, 10);
1243 end.writeUInt32LE(Buffer.concat(central).length, 12);
1244 end.writeUInt32LE(offset, 16);
1245 const path = NodePath.join(config.attachmentsDir, `structure-${caseIndex}.${item.ext}`);
1246 NodeFS.writeFileSync(path, Buffer.concat([...local, ...central, end]));
1247 const extracted = yield* Effect.promise(() => extractDocumentText(path, `.${item.ext}`));
1248 let previous = -1;
1249 for (const expected of item.expected) {
1250 const index = extracted.indexOf(expected);
1251 expect(index, `${item.ext}: ${expected} in ${extracted.slice(0, 500)}`).toBeGreaterThan(
1252 previous,
1253 );
1254 previous = index;
1255 }
1256 if ("occurrences" in item)
1257 expect(extracted.split(item.occurrences.text).length - 1).toBe(item.occurrences.count);
1258 if ("absent" in item)
1259 for (const absent of item.absent) expect(extracted).not.toContain(absent);
1260 if (("bounded" in item && item.bounded) || item.ext === "epub") {
1261 expect(extracted.length).toBeLessThan(1_000_100);
1262 expect(extracted.match(/\[Extracted text truncated/g)?.length ?? 0).toBeLessThanOrEqual(
1263 1,
1264 );
1265 const documents = yield* DocumentText.DocumentText;
1266 expect(yield* documents.ensure(path)).toBe(path + ".txt");
1267 expect(NodeFS.readFileSync(path + ".txt", "utf8")).toBe(extracted);
1268 }
1269 if (item.ext === "docx") {
1270 end.writeUInt16LE(2049, 8);
1271 end.writeUInt16LE(2049, 10);
1272 NodeFS.writeFileSync(path, Buffer.concat([...local, ...central, end]));
1273 yield* Effect.promise(async () => {
1274 expect(await extractDocumentText(path, ".docx")).toContain("[Extracted text truncated");
1275 });
1276 end.writeUInt16LE(central.length / 2, 8);
1277 end.writeUInt16LE(central.length / 2, 10);
1278 central[0]!.writeUInt16LE(8, 10);
1279 central[0]!.writeUInt32LE(33 * 1024 * 1024, 24);
1280 NodeFS.writeFileSync(path, Buffer.concat([...local, ...central, end]));
1281 yield* Effect.promise(async () => {
1282 expect(await extractDocumentText(path, ".docx")).toContain("[Extracted text truncated");
1283 });
1284 }
1285 }
1286 const path = NodePath.join(config.attachmentsDir, "unicode.rtf");
1287 NodeFS.writeFileSync(
1288 path,
1289 String.raw`{\rtf1\ansi{\fonttbl hidden}Hello\tab world\par Unicode \u945?{\*\secret hidden}}`,
1290 );
1291 const extracted = yield* Effect.promise(() => extractDocumentText(path, ".rtf"));
1292 expect(extracted).toContain("Hello\tworld\nUnicode α");
1293 expect(extracted).not.toContain("hidden");
1294
1295 for (const [name, content, expected] of [
1296 [
1297 "cyrillic-escaped",
1298 String.raw`{\rtf1\ansi\ansicpg1251 \'cf\'f0\'e8\'e2\'e5\'f2 {\ansicpg1252 \'e9} \'cc\'ee\'f1\'ea\'e2\'e0}`,
1299 "Привет é Москва",
1300 ],
1301 [
1302 "cyrillic-raw",
1303 Buffer.concat([
1304 Buffer.from(String.raw`{\rtf1\ansi\ansicpg1251 `),
1305 Buffer.from([0xcf, 0xf0, 0xe8, 0xe2, 0xe5, 0xf2]),
1306 Buffer.from("}"),
1307 ]),
1308 "Привет",
1309 ],
1310 ] as const) {
1311 const encodedPath = NodePath.join(config.attachmentsDir, `${name}.rtf`);
1312 NodeFS.writeFileSync(encodedPath, content);
1313 expect(yield* Effect.promise(() => extractDocumentText(encodedPath, ".rtf"))).toBe(
1314 expected,
1315 );
1316 const documents = yield* DocumentText.DocumentText;
1317 expect(yield* documents.ensure(encodedPath)).toBe(encodedPath + ".txt");
1318 expect(NodeFS.readFileSync(encodedPath + ".txt", "utf8")).toBe(expected);
1319 }
1320 const notebookPath = NodePath.join(config.attachmentsDir, "limited.ipynb");
1321 NodeFS.writeFileSync(
1322 notebookPath,
1323 JSON.stringify({ cells: Array.from({ length: 10_001 }, () => ({ source: "Kept" })) }),
1324 );
1325 const notebookLimited = yield* Effect.promise(() =>
1326 extractDocumentText(notebookPath, ".ipynb"),
1327 );
1328 expect(notebookLimited).toContain("## Cell 10000");
1329 expect(notebookLimited).not.toContain("## Cell 10001");
1330 expect(notebookLimited.match(/\[Extracted text truncated/g)).toHaveLength(1);
1331 expect(yield* (yield* DocumentText.DocumentText).ensure(notebookPath)).toBe(
1332 notebookPath + ".txt",
1333 );
1334 expect(NodeFS.readFileSync(notebookPath + ".txt", "utf8")).toBe(notebookLimited);
1335 const errorPath = NodePath.join(config.attachmentsDir, "limited-error.ipynb");
1336 NodeFS.writeFileSync(
1337 errorPath,
1338 JSON.stringify({
1339 cells: [
1340 {
1341 cell_type: "code",
1342 source: "1 / 0",
1343 outputs: [
1344 {
1345 output_type: "error",
1346 ename: "ZeroDivisionError",
1347 evalue: "division by zero",
1348 traceback: ["\u001b[31m" + "trace".repeat(250_000) + "\u001b[0m", "After limit"],
1349 },
1350 ],
1351 },
1352 ],
1353 }),
1354 );
1355 const errorLimited = yield* Effect.promise(() => extractDocumentText(errorPath, ".ipynb"));
1356 expect(errorLimited).toContain("ZeroDivisionError: division by zero\ntrace");
1357 expect(errorLimited.length).toBeLessThan(1_000_100);
1358 expect(errorLimited.match(/\[Extracted text truncated/g)).toHaveLength(1);
1359 expect(errorLimited).not.toContain("After limit");
1360 expect(errorLimited).not.toContain("\u001b");
1361 const deepPath = NodePath.join(config.attachmentsDir, "deep.rtf");
1362 NodeFS.writeFileSync(
1363 deepPath,
1364 String.raw`{\rtf1 Kept` + "{".repeat(128) + "Lost" + "}".repeat(129),
1365 );
1366 const deepLimited = yield* Effect.promise(() => extractDocumentText(deepPath, ".rtf"));
1367 expect(deepLimited).toBe("Kept\n\n[Extracted text truncated at document extraction limits.]");
1368 const unsupportedPath = NodePath.join(config.attachmentsDir, "unsupported-codepage.rtf");
1369 NodeFS.writeFileSync(unsupportedPath, String.raw`{\rtf1\ansi\ansicpg932 \'82\'a0}`);
1370 yield* Effect.promise(async () => {
1371 await expect(extractDocumentText(unsupportedPath, ".rtf")).rejects.toThrow(
1372 "Unsupported RTF code page",
1373 );
1374 });
1375 expect(yield* (yield* DocumentText.DocumentText).ensure(unsupportedPath)).toBeUndefined();
1376 expect(NodeFS.existsSync(unsupportedPath + ".txt")).toBe(false);
1377 NodeFS.writeFileSync(
1378 path,
1379 String.raw`{\rtf1 ` + "x".repeat(999_999) + String.raw`\u-10179?\u-8576? After}`,
1380 );
1381 const unicodeTruncated = yield* Effect.promise(() => extractDocumentText(path, ".rtf"));
1382 expect(unicodeTruncated).toContain("[Extracted text truncated");
1383 expect(unicodeTruncated.isWellFormed()).toBe(true);
1384 expect(unicodeTruncated.startsWith("x".repeat(999_999))).toBe(true);
1385 NodeFS.writeFileSync(path, String.raw`{\rtf1 First` + " ".repeat(1_000_100) + "Last}");
1386 const whitespaceTruncated = yield* Effect.promise(() => extractDocumentText(path, ".rtf"));
1387 expect(whitespaceTruncated).toBe(
1388 "First\n\n[Extracted text truncated at document extraction limits.]",
1389 );
1390 const documents = yield* DocumentText.DocumentText;
1391 expect(yield* documents.ensure(path)).toBe(path + ".txt");
1392 expect(NodeFS.readFileSync(path + ".txt", "utf8")).toBe(whitespaceTruncated);
1393 NodeFS.truncateSync(path, 17 * 1024 * 1024);
1394 yield* Effect.promise(async () => {
1395 expect(await extractDocumentText(path, ".rtf")).toContain("[Extracted text truncated");
1396 });
1397 }).pipe(Effect.provide(layer)),
1398 );
1399 it.effect("keeps selected values intact and appends an actual readable server path", () =>
1400 Effect.gen(function* () {
1401 const config = yield* ServerConfig.ServerConfig;
1402 const path = NodePath.join(config.attachmentsDir, `${attachment.id}.txt`);
1403 NodeFS.writeFileSync(path, "spec");
1404 const original = { q: ["First", "Second"], other: "No file" };
1405 const answers = yield* appendUserInputAttachmentPaths({
1406 answers: original,
1407 attachmentsDir: config.attachmentsDir,
1408 attachmentsByQuestionId: { q: [attachment] },
1409 });
1410 expect(answers.q).toEqual([
1411 "First",
1412 "Second",
1413 `Attached file "spec \\"final\\".txt": ${JSON.stringify(path)}`,
1414 ]);
1415 expect(answers.other).toBe("No file");
1416 expect(original.q).toEqual(["First", "Second"]);
1417 expect(NodeFS.readFileSync(path, "utf8")).toBe("spec");
1418 const specialKey = yield* appendUserInputAttachmentPaths({
1419 answers: {},
1420 attachmentsDir: config.attachmentsDir,
1421 attachmentsByQuestionId: { ["__proto__"]: [attachment] },
1422 });
1423 expect(Object.keys(specialKey)).toEqual(["__proto__"]);
1424 expect(specialKey["__proto__"]).toContain(JSON.stringify(path));
1425 }).pipe(Effect.provide(layer)),
1426 );
1427 it.effect("does not send an unavailable attachment path", () =>
1428 Effect.gen(function* () {
1429 const config = yield* ServerConfig.ServerConfig;
1430 const result = yield* appendUserInputAttachmentPaths({
1431 answers: { q: "" },
1432 attachmentsDir: config.attachmentsDir,
1433 attachmentsByQuestionId: { q: [attachment] },
1434 }).pipe(Effect.result);
1435 expect(result._tag).toBe("Failure");
1436 }).pipe(Effect.provide(layer)),
1437 );
1438 });