The best way to control Claude Code, Codex and more from any computer or phone
| 1 | // @effect-diagnostics nodeBuiltinImport:off |
| 2 | import * as NodeChildProcess from "node:child_process"; |
| 3 | import * as NodeFS from "node:fs"; |
| 4 | import * as NodePath from "node:path"; |
| 5 | import * as NodeServices from "@effect/platform-node/NodeServices"; |
| 6 | import { describe, expect, it } from "@effect/vitest"; |
| 7 | import * as Effect from "effect/Effect"; |
| 8 | import * as Deferred from "effect/Deferred"; |
| 9 | import * as Fiber from "effect/Fiber"; |
| 10 | import * as Queue from "effect/Queue"; |
| 11 | import * as FileSystem from "effect/FileSystem"; |
| 12 | import * as Layer from "effect/Layer"; |
| 13 | import * as TestClock from "effect/testing/TestClock"; |
| 14 | import { vi } from "vite-plus/test"; |
| 15 | import * as HostProcess from "@t3tools/shared/HostProcess"; |
| 16 | import * as ServerConfig from "../config.ts"; |
| 17 | import * as DocumentText from "./DocumentText.ts"; |
| 18 | import { extractDocumentText } from "./documentTextWorker.ts"; |
| 19 | import * as Cleanup from "../orchestration-v2/ResourceCleanupService.ts"; |
| 20 | import * as DeviceService from "../device/DeviceService.ts"; |
| 21 | import * as TerminalManager from "../terminal/Manager.ts"; |
| 22 | import * as PreviewManager from "../preview/Manager.ts"; |
| 23 | import * as ScheduledTaskService from "../scheduledTasks/ScheduledTaskService.ts"; |
| 24 | import * as NodeZlib from "node:zlib"; |
| 25 | import { appendUserInputAttachmentPaths } from "./userInputAttachments.ts"; |
| 26 | |
| 27 | vi.mock("node:child_process", { spy: true }); |
| 28 | |
| 29 | const layer = Layer.merge( |
| 30 | ServerConfig.layerTest(process.cwd(), { prefix: "t3-answer-paths-" }), |
| 31 | DocumentText.layer, |
| 32 | ).pipe(Layer.provideMerge(NodeServices.layer)); |
| 33 | const attachment = { |
| 34 | type: "file" as const, |
| 35 | id: "thread-1-00000000-0000-4000-8000-0000000000aa-txt", |
| 36 | name: 'spec "final".txt', |
| 37 | mimeType: "text/plain", |
| 38 | sizeBytes: 4, |
| 39 | }; |
| 40 | describe("question answer paths", () => { |
| 41 | it.effect.each([false, true])( |
| 42 | "keeps unrelated files available during extraction (failure: %s)", |
| 43 | (failure) => |
| 44 | Effect.gen(function* () { |
| 45 | const config = yield* ServerConfig.ServerConfig; |
| 46 | const fs = yield* FileSystem.FileSystem; |
| 47 | const path = NodePath.join(config.attachmentsDir, "blocked.rtf"); |
| 48 | const cached = NodePath.join(config.attachmentsDir, "cached.rtf"); |
| 49 | NodeFS.writeFileSync(path, failure ? "invalid document" : String.raw`{\rtf1 Available}`); |
| 50 | NodeFS.writeFileSync(cached + ".txt", "Cached text"); |
| 51 | const started = yield* Deferred.make<void>(); |
| 52 | const release = yield* Deferred.make<void>(); |
| 53 | let reads = 0; |
| 54 | yield* Effect.gen(function* () { |
| 55 | const documents = yield* DocumentText.DocumentText; |
| 56 | const extraction = yield* documents.ensure(path).pipe(Effect.forkChild); |
| 57 | yield* Deferred.await(started); |
| 58 | const duplicate = yield* documents.ensure(path).pipe(Effect.forkChild); |
| 59 | expect(yield* documents.ensure(path + ".png")).toBeUndefined(); |
| 60 | expect(yield* documents.ensure(path + ".pdf")).toBeUndefined(); |
| 61 | expect(yield* documents.ensure(cached)).toBe(cached + ".txt"); |
| 62 | expect(NodeFS.readFileSync(cached + ".txt", "utf8")).toBe("Cached text"); |
| 63 | yield* Deferred.succeed(release, undefined); |
| 64 | const expected = failure ? undefined : path + ".txt"; |
| 65 | expect(yield* Fiber.join(extraction)).toBe(expected); |
| 66 | expect(yield* Fiber.join(duplicate)).toBe(expected); |
| 67 | expect(yield* documents.ensure(path)).toBe(expected); |
| 68 | expect(reads).toBe(1); |
| 69 | if (failure) { |
| 70 | NodeFS.writeFileSync(path + ".txt", "Recovered text"); |
| 71 | expect(yield* documents.ensure(path)).toBe(path + ".txt"); |
| 72 | expect(NodeFS.readFileSync(path + ".txt", "utf8")).toBe("Recovered text"); |
| 73 | } else expect(NodeFS.readFileSync(path + ".txt", "utf8")).toBe("Available"); |
| 74 | }).pipe( |
| 75 | Effect.provide(Layer.fresh(DocumentText.layer)), |
| 76 | Effect.provideService(FileSystem.FileSystem, { |
| 77 | ...fs, |
| 78 | stat: (file) => |
| 79 | file === path |
| 80 | ? Effect.gen(function* () { |
| 81 | reads++; |
| 82 | yield* Deferred.succeed(started, undefined); |
| 83 | yield* Deferred.await(release); |
| 84 | return yield* fs.stat(file); |
| 85 | }) |
| 86 | : fs.stat(file), |
| 87 | }), |
| 88 | ); |
| 89 | }).pipe(Effect.provide(layer)), |
| 90 | ); |
| 91 | |
| 92 | it.effect.each(["interrupted", "failed"])( |
| 93 | "extracts two documents at once and frees a slot when one is %s", |
| 94 | (outcome) => |
| 95 | Effect.gen(function* () { |
| 96 | const config = yield* ServerConfig.ServerConfig; |
| 97 | const fs = yield* FileSystem.FileSystem; |
| 98 | const [first, second, third] = ["first", "second", "third"].map((name) => |
| 99 | NodePath.join(config.attachmentsDir, `${name}.rtf`), |
| 100 | ); |
| 101 | NodeFS.writeFileSync(first!, String.raw`{\rtf1 First}`); |
| 102 | NodeFS.writeFileSync(second!, "invalid document"); |
| 103 | NodeFS.writeFileSync(third!, String.raw`{\rtf1 Third}`); |
| 104 | const started = yield* Queue.unbounded<string>(); |
| 105 | const releases = new Map<string, Deferred.Deferred<void>>(); |
| 106 | for (const path of [first!, second!, third!]) |
| 107 | releases.set(path, yield* Deferred.make<void>()); |
| 108 | yield* Effect.gen(function* () { |
| 109 | const documents = yield* DocumentText.DocumentText; |
| 110 | const leaving = yield* documents.ensure(first!).pipe(Effect.forkChild); |
| 111 | const staying = yield* documents.ensure(first!).pipe(Effect.forkChild); |
| 112 | const freed = yield* documents.ensure(second!).pipe(Effect.forkChild); |
| 113 | expect(new Set([yield* Queue.take(started), yield* Queue.take(started)])).toEqual( |
| 114 | new Set([first, second]), |
| 115 | ); |
| 116 | const waiting = yield* documents.ensure(third!).pipe(Effect.forkChild); |
| 117 | yield* Fiber.interrupt(leaving); |
| 118 | if (outcome === "interrupted") yield* Fiber.interrupt(freed); |
| 119 | else { |
| 120 | yield* Deferred.succeed(releases.get(second!)!, undefined); |
| 121 | expect(yield* Fiber.join(freed)).toBeUndefined(); |
| 122 | } |
| 123 | expect(yield* Queue.take(started)).toBe(third); |
| 124 | yield* Deferred.succeed(releases.get(first!)!, undefined); |
| 125 | yield* Deferred.succeed(releases.get(third!)!, undefined); |
| 126 | expect(yield* Fiber.join(staying)).toBe(first + ".txt"); |
| 127 | expect(yield* Fiber.join(waiting)).toBe(third + ".txt"); |
| 128 | expect(yield* Queue.size(started)).toBe(0); |
| 129 | expect(NodeFS.existsSync(second + ".txt")).toBe(false); |
| 130 | expect( |
| 131 | NodeFS.readdirSync(config.attachmentsDir).filter((file) => file.endsWith(".part")), |
| 132 | ).toEqual([]); |
| 133 | }).pipe( |
| 134 | Effect.provide(Layer.fresh(DocumentText.layer)), |
| 135 | Effect.provideService(FileSystem.FileSystem, { |
| 136 | ...fs, |
| 137 | // An async probe can let `staying` join only after `leaving` cancelled the lookup. |
| 138 | exists: (file) => Effect.sync(() => NodeFS.existsSync(file)), |
| 139 | stat: (file) => |
| 140 | Effect.gen(function* () { |
| 141 | yield* Queue.offer(started, file); |
| 142 | yield* Deferred.await(releases.get(file)!); |
| 143 | return yield* fs.stat(file); |
| 144 | }), |
| 145 | }), |
| 146 | ); |
| 147 | }).pipe(Effect.provide(layer)), |
| 148 | ); |
| 149 | |
| 150 | it.effect("retries a document while its cancelled extraction is still publishing", () => |
| 151 | Effect.gen(function* () { |
| 152 | const config = yield* ServerConfig.ServerConfig; |
| 153 | const fs = yield* FileSystem.FileSystem; |
| 154 | const path = NodePath.join(config.attachmentsDir, "retried.rtf"); |
| 155 | NodeFS.writeFileSync(path, String.raw`{\rtf1 Retried}`); |
| 156 | const writing = yield* Deferred.make<void>(); |
| 157 | const writeReleased = yield* Deferred.make<void>(); |
| 158 | const renaming = yield* Deferred.make<void>(); |
| 159 | const renameReleased = yield* Deferred.make<void>(); |
| 160 | let writes = 0; |
| 161 | let renames = 0; |
| 162 | yield* Effect.gen(function* () { |
| 163 | const documents = yield* DocumentText.DocumentText; |
| 164 | const cancelled = yield* documents.ensure(path).pipe(Effect.forkChild); |
| 165 | yield* Deferred.await(writing); |
| 166 | const interrupting = yield* Fiber.interrupt(cancelled).pipe(Effect.forkChild); |
| 167 | const retry = yield* documents.ensure(path).pipe(Effect.forkChild); |
| 168 | yield* Deferred.await(renaming); |
| 169 | yield* Deferred.succeed(writeReleased, undefined); |
| 170 | yield* Fiber.join(interrupting); |
| 171 | yield* Deferred.succeed(renameReleased, undefined); |
| 172 | expect(yield* Fiber.join(retry)).toBe(path + ".txt"); |
| 173 | expect(NodeFS.readFileSync(path + ".txt", "utf8")).toBe("Retried"); |
| 174 | expect( |
| 175 | NodeFS.readdirSync(config.attachmentsDir).filter((file) => file.endsWith(".part")), |
| 176 | ).toEqual([]); |
| 177 | }).pipe( |
| 178 | Effect.provide(Layer.fresh(DocumentText.layer)), |
| 179 | Effect.provideService(FileSystem.FileSystem, { |
| 180 | ...fs, |
| 181 | writeFileString: (file, data) => |
| 182 | Effect.gen(function* () { |
| 183 | if (++writes === 1) { |
| 184 | yield* Deferred.succeed(writing, undefined); |
| 185 | yield* Deferred.await(writeReleased); |
| 186 | } |
| 187 | return yield* fs.writeFileString(file, data); |
| 188 | }), |
| 189 | rename: (from, to) => |
| 190 | Effect.gen(function* () { |
| 191 | if (++renames === 1) { |
| 192 | yield* Deferred.succeed(renaming, undefined); |
| 193 | yield* Deferred.await(renameReleased); |
| 194 | } |
| 195 | return yield* fs.rename(from, to); |
| 196 | }), |
| 197 | }), |
| 198 | ); |
| 199 | }).pipe(Effect.provide(layer)), |
| 200 | ); |
| 201 | |
| 202 | it.effect.each(["spawn", "read"])("retries after a transient %s failure expires", (failure) => |
| 203 | Effect.acquireUseRelease( |
| 204 | Effect.promise(async () => { |
| 205 | const { execFile } = await vi.importActual<typeof NodeChildProcess>("node:child_process"); |
| 206 | const worker = vi.spyOn(NodeChildProcess, "execFile"); |
| 207 | worker.mockImplementationOnce((( |
| 208 | file: string, |
| 209 | args: readonly string[], |
| 210 | options: NodeChildProcess.ExecFileOptionsWithStringEncoding, |
| 211 | callback: ( |
| 212 | error: NodeChildProcess.ExecFileException | null, |
| 213 | stdout: string, |
| 214 | stderr: string, |
| 215 | ) => void, |
| 216 | ) => { |
| 217 | if (failure === "spawn") { |
| 218 | throw Object.assign(new Error("spawn EAGAIN"), { code: "EAGAIN" }); |
| 219 | } |
| 220 | const preload = `import fs from 'node:fs/promises'; |
| 221 | import { syncBuiltinESMExports } from 'node:module'; |
| 222 | fs.readFile = async () => { throw Object.assign(new Error('read EAGAIN'), { code: 'EAGAIN' }); }; |
| 223 | syncBuiltinESMExports();`; |
| 224 | return execFile( |
| 225 | file, |
| 226 | ["--import", `data:text/javascript,${encodeURIComponent(preload)}`, ...args], |
| 227 | options, |
| 228 | callback, |
| 229 | ); |
| 230 | }) as typeof NodeChildProcess.execFile); |
| 231 | return worker; |
| 232 | }), |
| 233 | (worker) => |
| 234 | Effect.gen(function* () { |
| 235 | const config = yield* ServerConfig.ServerConfig; |
| 236 | const path = NodePath.join(config.attachmentsDir, "retry.rtf"); |
| 237 | NodeFS.writeFileSync(path, String.raw`{\rtf1 Available}`); |
| 238 | const documents = yield* DocumentText.DocumentText; |
| 239 | expect(yield* documents.ensure(path)).toBeUndefined(); |
| 240 | expect(yield* documents.ensure(path)).toBeUndefined(); |
| 241 | expect(worker).toHaveBeenCalledTimes(1); |
| 242 | expect(NodeFS.existsSync(path + ".txt")).toBe(false); |
| 243 | yield* TestClock.adjust("1 second"); |
| 244 | expect(yield* documents.ensure(path)).toBe(path + ".txt"); |
| 245 | expect(NodeFS.readFileSync(path + ".txt", "utf8")).toBe("Available"); |
| 246 | expect(worker).toHaveBeenCalledTimes(2); |
| 247 | expect(yield* documents.ensure(path)).toBe(path + ".txt"); |
| 248 | expect(worker).toHaveBeenCalledTimes(2); |
| 249 | }).pipe(Effect.provide(layer)), |
| 250 | (worker) => Effect.sync(() => worker.mockRestore()), |
| 251 | ), |
| 252 | ); |
| 253 | |
| 254 | it.effect.each([ |
| 255 | ["ipynb", "{"], |
| 256 | ["rtf", "invalid document"], |
| 257 | ["rtf", String.raw`{\rtf1}`], |
| 258 | ["rtf", String.raw`{\rtf1\ansi\ansicpg99999 Unsupported}`], |
| 259 | ] as const)("caches unchanged document rejections: %s %s", ([extension, source]) => |
| 260 | Effect.acquireUseRelease( |
| 261 | Effect.sync(() => vi.spyOn(NodeChildProcess, "execFile")), |
| 262 | (worker) => |
| 263 | Effect.gen(function* () { |
| 264 | const config = yield* ServerConfig.ServerConfig; |
| 265 | const path = NodePath.join(config.attachmentsDir, `invalid.${extension}`); |
| 266 | NodeFS.writeFileSync(path, source); |
| 267 | const documents = yield* DocumentText.DocumentText; |
| 268 | for (const seconds of [0, 1, 60]) { |
| 269 | yield* TestClock.adjust(`${seconds} seconds`); |
| 270 | expect(yield* documents.ensure(path)).toBeUndefined(); |
| 271 | expect(worker).toHaveBeenCalledTimes(1); |
| 272 | expect(NodeFS.existsSync(path + ".txt")).toBe(false); |
| 273 | } |
| 274 | }).pipe(Effect.provide(layer)), |
| 275 | (worker) => Effect.sync(() => worker.mockRestore()), |
| 276 | ), |
| 277 | ); |
| 278 | |
| 279 | it.effect.each([ |
| 280 | [ |
| 281 | "page-specific headers and footers", |
| 282 | String.raw`{\rtf1\ansi\facingp\sectd\titlepg |
| 283 | {\headerf First-page header}{\headerl Left-page header}{\headerr Right-page header} |
| 284 | {\footerf First-page footer}{\footerl Left-page footer}{\footerr Right-page footer} |
| 285 | \pard Approved delivery date: 24 November.\par |
| 286 | }`, |
| 287 | "Approved delivery date: 24 November.", |
| 288 | ], |
| 289 | [ |
| 290 | "text symbols", |
| 291 | String.raw`{\rtf1\ansi Pages 1\endash 5\emdash\lquote left\rquote\ldblquote right\rdblquote\bullet\b End\b0\unknown{\*\secret\bullet hidden}}`, |
| 292 | "Pages 1–5—‘left’“right”•End", |
| 293 | ], |
| 294 | [ |
| 295 | "space and hyphen symbols", |
| 296 | String.raw`{\rtf1\ansi A\enspace B\emspace C\qmspace D\~E\_F\-G}`, |
| 297 | "A\u2002B\u2003C\u2005D\u00a0E\u2011FG", |
| 298 | ], |
| 299 | [ |
| 300 | "joiner and direction marks", |
| 301 | String.raw`{\rtf1\ansi A\zwj B\zwnj C\ltrmark D\rtlmark E}`, |
| 302 | "A\u200dB\u200cC\u200eD\u200fE", |
| 303 | ], |
| 304 | [ |
| 305 | "text symbol fallback", |
| 306 | String.raw`{\rtf1\ansi\uc10\u945\endash\emdash\enspace\emspace\qmspace\lquote\rquote\ldblquote\rdblquote\bullet Z}`, |
| 307 | "αZ", |
| 308 | ], |
| 309 | ["hyphen symbol fallback", String.raw`{\rtf1\ansi\uc3\u945\~\_\-Z}`, "αZ"], |
| 310 | ["control word", String.raw`{\rtf1\ansi A\u9\tab X}`, "A\tX"], |
| 311 | ["line break paragraphs", "{\\rtf1\\ansi First\\\nSecond\\\r\nThird}", "First\nSecond\nThird"], |
| 312 | [ |
| 313 | "tracked deletion", |
| 314 | String.raw`{\rtf1\ansi The fee is {\deleted\revauthdel1\revdttmdel1 $500}{\revised\revauth1 $700}.}`, |
| 315 | "The fee is $700.", |
| 316 | ], |
| 317 | [ |
| 318 | "tracked deletion end", |
| 319 | String.raw`{\rtf1\ansi{\header Header\plain Plain}A\deleted B\deleted0 C\deleted D\plain E}`, |
| 320 | "ACE", |
| 321 | ], |
| 322 | [ |
| 323 | "tracked deletion fallback", |
| 324 | String.raw`{\rtf1\ansi A{\deleted\uc1\u945\deleted0 B}\deleted\u946?\deleted0 C}`, |
| 325 | "AC", |
| 326 | ], |
| 327 | [ |
| 328 | "tracked deletion table", |
| 329 | String.raw`{\rtf1\ansi\trowd\cellx1000\cellx2000\intbl A\deleted\cell\deleted0 B\cell\deleted\row\deleted0 C\cell}`, |
| 330 | "A\tB\t\nC", |
| 331 | ], |
| 332 | [ |
| 333 | "document breaks", |
| 334 | String.raw`{\rtf1\ansi\sectd\cols2\pard North quota: 120\column South quota: 240\page East quota: 360\sect\sectd\pard West quota: 480\par}`, |
| 335 | "North quota: 120\nSouth quota: 240\nEast quota: 360\nWest quota: 480", |
| 336 | ], |
| 337 | ["group close", String.raw`{\rtf1\ansi{\u945}Z}`, "αZ"], |
| 338 | ["group open", String.raw`{\rtf1\ansi\u945{Z}}`, "αZ"], |
| 339 | ["hex byte", String.raw`{\rtf1\ansi\u945\'3fZ}`, "αZ"], |
| 340 | ["escaped symbol", String.raw`{\rtf1\ansi\u945\{Z}`, "αZ"], |
| 341 | ["Macintosh text", String.raw`{\rtf1\mac Caf\'8e Cr\'8fme}`, "Café Crème"], |
| 342 | [ |
| 343 | "explicit Macintosh code page", |
| 344 | String.raw`{\rtf1\mac\ansicpg10000 Caf\'8e Cr\'8fme}`, |
| 345 | "Café Crème", |
| 346 | ], |
| 347 | ["destination symbol", String.raw`{\rtf1\ansi\u945\*Z}`, "αZ"], |
| 348 | ["multiple units", String.raw`{\rtf1\ansi\uc4\u945?\tab\'3f\~Z}`, "αZ"], |
| 349 | ["binary unit", String.raw`{\rtf1\ansi\u945\bin3 abcZ}`, "αZ"], |
| 350 | ["large fallback", String.raw`{\rtf1\ansi\uc20\u945????????????????????Z}`, "αZ"], |
| 351 | ["no fallback", String.raw`{\rtf1\ansi\uc0\u945Z}`, "αZ"], |
| 352 | ["scoped count", String.raw`{\rtf1\ansi{\uc2\u945??}\u946?Z}`, "αβZ"], |
| 353 | ] as const)("extracts RTF text: %s", ([name, source, expected]) => |
| 354 | Effect.gen(function* () { |
| 355 | const config = yield* ServerConfig.ServerConfig; |
| 356 | const path = NodePath.join(config.attachmentsDir, `${name}.rtf`); |
| 357 | NodeFS.writeFileSync(path, source); |
| 358 | expect(yield* Effect.promise(() => extractDocumentText(path, ".rtf"))).toBe(expected); |
| 359 | expect(yield* (yield* DocumentText.DocumentText).ensure(path)).toBe(path + ".txt"); |
| 360 | expect(NodeFS.readFileSync(path + ".txt", "utf8")).toBe(expected); |
| 361 | }).pipe(Effect.provide(layer)), |
| 362 | ); |
| 363 | |
| 364 | it.effect.each([false, true])("limits the child heap for standalone=%s", (standalone) => |
| 365 | Effect.acquireUseRelease( |
| 366 | Effect.promise(async () => { |
| 367 | const { execFile } = await vi.importActual<typeof NodeChildProcess>("node:child_process"); |
| 368 | const probe = vi.spyOn(NodeChildProcess, "execFile"); |
| 369 | probe.mockImplementation((( |
| 370 | file: string, |
| 371 | _args: readonly string[], |
| 372 | options: NodeChildProcess.ExecFileOptionsWithStringEncoding, |
| 373 | callback: ( |
| 374 | error: NodeChildProcess.ExecFileException | null, |
| 375 | stdout: string, |
| 376 | stderr: string, |
| 377 | ) => void, |
| 378 | ) => |
| 379 | execFile( |
| 380 | file, |
| 381 | [ |
| 382 | "-e", |
| 383 | 'process.stdout.write(String(require("node:v8").getHeapStatistics().heap_size_limit))', |
| 384 | ], |
| 385 | options, |
| 386 | callback, |
| 387 | )) as typeof NodeChildProcess.execFile); |
| 388 | return probe; |
| 389 | }), |
| 390 | () => |
| 391 | Effect.gen(function* () { |
| 392 | const config = yield* ServerConfig.ServerConfig; |
| 393 | const path = NodePath.join(config.attachmentsDir, "heap.rtf"); |
| 394 | NodeFS.writeFileSync(path, String.raw`{\rtf1 hello}`); |
| 395 | const extracted = yield* Effect.gen(function* () { |
| 396 | return yield* (yield* DocumentText.DocumentText).ensure(path); |
| 397 | }).pipe( |
| 398 | Effect.provide(Layer.fresh(DocumentText.layer)), |
| 399 | Effect.provideService(HostProcess.IsExecutable, standalone), |
| 400 | Effect.provideService(HostProcess.Environment, { |
| 401 | ...process.env, |
| 402 | NODE_OPTIONS: "--max-old-space-size=1024", |
| 403 | }), |
| 404 | ); |
| 405 | expect(extracted).toBe(path + ".txt"); |
| 406 | const heapLimit = Number(NodeFS.readFileSync(path + ".txt", "utf8")); |
| 407 | expect(heapLimit).toBeGreaterThan(192 * 1024 * 1024); |
| 408 | // Newer V8 versions reserve up to 192 MiB for the young generation as well. |
| 409 | expect(heapLimit).toBeLessThanOrEqual(384 * 1024 * 1024); |
| 410 | }).pipe(Effect.provide(layer)), |
| 411 | (probe) => Effect.sync(() => probe.mockRestore()), |
| 412 | ), |
| 413 | ); |
| 414 | |
| 415 | it.effect("extracts notebook cells and outputs once and keeps both readable paths", () => |
| 416 | Effect.gen(function* () { |
| 417 | const config = yield* ServerConfig.ServerConfig; |
| 418 | const file = { |
| 419 | ...attachment, |
| 420 | id: attachment.id.replace(/txt$/, "ipynb"), |
| 421 | name: "survey.ipynb", |
| 422 | mimeType: "application/x-ipynb+json", |
| 423 | }; |
| 424 | const path = NodePath.join(config.attachmentsDir, `${file.id}.ipynb`); |
| 425 | const rawSource = String.raw`\begin{tabular}{lr} North & 120 \\ \end{tabular}`; |
| 426 | const notebook = JSON.stringify({ |
| 427 | cells: [ |
| 428 | { cell_type: "markdown", source: ["# Survey\n", "Vessel: Larkspur"] }, |
| 429 | { |
| 430 | cell_type: "code", |
| 431 | source: ["print(43 * 7)"], |
| 432 | outputs: [ |
| 433 | { output_type: "stream", name: "stdout", text: "\u001b[32m301\u001b[0m\n" }, |
| 434 | { |
| 435 | output_type: "display_data", |
| 436 | data: { "text/plain": ["\u001b[", "33m42\u001b[0m\n", "Next line"] }, |
| 437 | metadata: {}, |
| 438 | }, |
| 439 | { |
| 440 | output_type: "error", |
| 441 | ename: "ZeroDivisionError", |
| 442 | evalue: "division by zero", |
| 443 | traceback: [ |
| 444 | "\u001b[32mCell In[1], line 1\u001b[0m", |
| 445 | "\u001b[31m----> 1 1 / 0\u001b[0m", |
| 446 | "ZeroDivisionError: division by zero", |
| 447 | ], |
| 448 | }, |
| 449 | ], |
| 450 | }, |
| 451 | { cell_type: "raw", source: [rawSource] }, |
| 452 | ], |
| 453 | }); |
| 454 | NodeFS.writeFileSync(path, notebook); |
| 455 | const answers = yield* appendUserInputAttachmentPaths({ |
| 456 | answers: { q: "Read this" }, |
| 457 | attachmentsDir: config.attachmentsDir, |
| 458 | attachmentsByQuestionId: { q: [file] }, |
| 459 | }); |
| 460 | expect(answers.q).toContain(`text extracted to: ${JSON.stringify(path + ".txt")}`); |
| 461 | expect(NodeFS.readFileSync(`${path}.txt`, "utf8")).toContain("Vessel: Larkspur"); |
| 462 | const extracted = NodeFS.readFileSync(`${path}.txt`, "utf8"); |
| 463 | expect(extracted).toContain("Output:\n301\n"); |
| 464 | expect(extracted).toContain("42\nNext line"); |
| 465 | expect(extracted).toContain(`## Cell 3 (raw)\n${rawSource}`); |
| 466 | expect(extracted).toContain( |
| 467 | "ZeroDivisionError: division by zero\nCell In[1], line 1\n----> 1 1 / 0\nZeroDivisionError: division by zero", |
| 468 | ); |
| 469 | expect(extracted).not.toContain("\u001b"); |
| 470 | expect(NodeFS.readFileSync(path, "utf8")).toBe(notebook); |
| 471 | const before = NodeFS.statSync(path + ".txt").mtimeMs; |
| 472 | // A changed binary would fail parsing if the cached extraction were not reused. |
| 473 | NodeFS.writeFileSync(path, "invalid notebook"); |
| 474 | const documents = yield* DocumentText.DocumentText; |
| 475 | const results = yield* Effect.all([documents.ensure(path), documents.ensure(path)], { |
| 476 | concurrency: "unbounded", |
| 477 | }); |
| 478 | expect(results).toEqual([path + ".txt", path + ".txt"]); |
| 479 | expect(NodeFS.statSync(path + ".txt").mtimeMs).toBe(before); |
| 480 | yield* Effect.gen(function* () { |
| 481 | const cleanup = yield* Cleanup.ResourceCleanupService; |
| 482 | yield* cleanup.cleanupAttachments([file.id]); |
| 483 | expect(NodeFS.existsSync(path)).toBe(false); |
| 484 | expect(NodeFS.existsSync(path + ".txt")).toBe(false); |
| 485 | NodeFS.writeFileSync(path + ".txt", "orphaned extraction"); |
| 486 | yield* cleanup.cleanupAttachments([file.id]); |
| 487 | expect(NodeFS.existsSync(path + ".txt")).toBe(false); |
| 488 | }).pipe( |
| 489 | Effect.provide( |
| 490 | Cleanup.layer.pipe( |
| 491 | Layer.provide( |
| 492 | Layer.mergeAll( |
| 493 | Layer.mock(TerminalManager.TerminalManager)({}), |
| 494 | Layer.mock(PreviewManager.PreviewManager)({}), |
| 495 | Layer.mock(DeviceService.DeviceService)({}), |
| 496 | Layer.mock(ScheduledTaskService.ScheduledTaskService)({}), |
| 497 | ), |
| 498 | ), |
| 499 | ), |
| 500 | ), |
| 501 | ); |
| 502 | }).pipe(Effect.provide(layer)), |
| 503 | ); |
| 504 | |
| 505 | it.effect( |
| 506 | "preserves corrupt documents, PDFs and unknown binaries without an extracted path", |
| 507 | () => |
| 508 | Effect.gen(function* () { |
| 509 | const config = yield* ServerConfig.ServerConfig; |
| 510 | for (const extension of ["docx", "ipynb", "zip", "pdf"]) { |
| 511 | const file = { |
| 512 | ...attachment, |
| 513 | id: attachment.id.replace(/txt$/, extension), |
| 514 | name: `broken.${extension}`, |
| 515 | }; |
| 516 | const path = NodePath.join(config.attachmentsDir, `${file.id}.${extension}`); |
| 517 | NodeFS.writeFileSync(path, "invalid document"); |
| 518 | const answers = yield* appendUserInputAttachmentPaths({ |
| 519 | answers: {}, |
| 520 | attachmentsDir: config.attachmentsDir, |
| 521 | attachmentsByQuestionId: { q: [file] }, |
| 522 | }); |
| 523 | expect(answers.q).toContain(JSON.stringify(path)); |
| 524 | expect(answers.q).not.toContain("text extracted"); |
| 525 | expect(NodeFS.existsSync(`${path}.txt`)).toBe(false); |
| 526 | expect(NodeFS.readFileSync(path, "utf8")).toBe("invalid document"); |
| 527 | } |
| 528 | }).pipe(Effect.provide(layer)), |
| 529 | ); |
| 530 | |
| 531 | it.effect("preserves office structure and EPUB reading order with bounded archives", () => |
| 532 | Effect.gen(function* () { |
| 533 | const config = yield* ServerConfig.ServerConfig; |
| 534 | const cases = [ |
| 535 | // Word stores object layout and field data outside the visible document text. |
| 536 | { |
| 537 | ext: "docx", |
| 538 | bounded: true, |
| 539 | parts: { |
| 540 | "word/document.xml": |
| 541 | '<w:document xmlns:w="urn:w" xmlns:wp="urn:wp" xmlns:wp14="urn:wp14" xmlns:mc="urn:mc"><w:body><w:p><w:r><w:t>Total cost is </w:t><w:drawing><wp:anchor><wp:positionH><wp:posOffset>1905000</wp:posOffset></wp:positionH><wp:positionV><wp:posOffset>-635</wp:posOffset></wp:positionV><wp14:sizeRelH><wp14:pctWidth>0</wp14:pctWidth></wp14:sizeRelH><wp14:sizeRelV><wp14:pctHeight>0</wp14:pctHeight></wp14:sizeRelV></wp:anchor></w:drawing><w:t>42 EUR.</w:t></w:r></w:p></w:body></w:document>', |
| 542 | }, |
| 543 | expected: ["Total cost is 42 EUR."], |
| 544 | absent: ["1905000", "-635", "0042"], |
| 545 | }, |
| 546 | { |
| 547 | ext: "docx", |
| 548 | bounded: true, |
| 549 | parts: { |
| 550 | "word/document.xml": |
| 551 | '<w:document xmlns:w="urn:w" xmlns:wp="urn:wp" xmlns:wp14="urn:wp14" xmlns:mc="urn:mc"><w:body><w:p><mc:AlternateContent><mc:Choice Requires="cx"><w:drawing><wp:anchor><wp:positionH><wp:posOffset>1905000</wp:posOffset></wp:positionH><wp:positionV><wp:posOffset>-635</wp:posOffset></wp:positionV><wp14:sizeRelH><wp14:pctWidth>0</wp14:pctWidth></wp14:sizeRelH><wp14:sizeRelV><wp14:pctHeight>0</wp14:pctHeight></wp14:sizeRelV></wp:anchor></w:drawing><cx:chart/></mc:Choice><mc:Fallback><w:pict><v:shape><v:textbox><w:txbxContent><w:p><w:r><w:t>Chart caption</w:t></w:r></w:p></w:txbxContent></v:textbox></v:shape></w:pict></mc:Fallback></mc:AlternateContent></w:p></w:body></w:document>', |
| 552 | }, |
| 553 | expected: ["Chart caption"], |
| 554 | absent: ["1905000", "-635"], |
| 555 | }, |
| 556 | { |
| 557 | ext: "docx", |
| 558 | bounded: true, |
| 559 | parts: { |
| 560 | "word/document.xml": |
| 561 | '<w:document xmlns:w="urn:w" xmlns:wp="urn:wp" xmlns:wp14="urn:wp14" xmlns:mc="urn:mc"><w:body><w:p><w:r><w:t>As shown </w:t><w:fldChar><w:fldData>RU5ETk9URQ==</w:fldData></w:fldChar><w:t>(Smith 2020) earlier.</w:t></w:r></w:p></w:body></w:document>', |
| 562 | }, |
| 563 | expected: ["As shown (Smith 2020) earlier."], |
| 564 | absent: ["RU5ETk9URQ=="], |
| 565 | }, |
| 566 | { |
| 567 | ext: "docx", |
| 568 | bounded: true, |
| 569 | parts: { |
| 570 | "word/document.xml": "<document><p>Readable despite embedded media</p></document>", |
| 571 | "word/media/black.bmp": "\0".repeat(64 * 1024), |
| 572 | "word/embeddings/unused.bin": "\0".repeat(33 * 1024 * 1024), |
| 573 | }, |
| 574 | expected: ["Readable despite embedded media"], |
| 575 | absent: ["[Extracted text truncated"], |
| 576 | }, |
| 577 | ...[false, true].map((deep) => ({ |
| 578 | ext: "docx", |
| 579 | bounded: true, |
| 580 | parts: { |
| 581 | "word/document.xml": `<document>${deep ? "<div>".repeat(129) + "Lost" + "</div>".repeat(129) : "<p/>".repeat(200_001)}</document>`, |
| 582 | }, |
| 583 | expected: ["[Extracted text truncated at document extraction limits.]"], |
| 584 | absent: ["Lost"], |
| 585 | })), |
| 586 | ...["ods", "odt", "odp", "docx"].flatMap((ext) => |
| 587 | [false, true].map((rows) => ({ |
| 588 | ext, |
| 589 | bounded: true, |
| 590 | parts: { |
| 591 | [ext === "docx" ? "word/document.xml" : "content.xml"]: |
| 592 | `<document><page><table><table-row${rows ? ' number-rows-repeated="10001"' : ""}><table-cell${rows ? "" : ' number-columns-repeated="513"'}><p>Repeated value</p></table-cell></table-row></table></page></document>`, |
| 593 | }, |
| 594 | expected: [ |
| 595 | "Repeated value", |
| 596 | "[Extracted text truncated at document extraction limits.]", |
| 597 | ], |
| 598 | occurrences: { text: "Repeated value", count: rows ? 10_000 : 512 }, |
| 599 | })), |
| 600 | ), |
| 601 | ...["ods", "odt", "odp", "docx"].map((ext) => ({ |
| 602 | ext, |
| 603 | bounded: true, |
| 604 | parts: { |
| 605 | [ext === "docx" ? "word/document.xml" : "content.xml"]: |
| 606 | `<document><page><table><table-row><table-cell><p>Kept</p></table-cell><table-cell number-columns-repeated="16383"> \n <p> \t </p> <s c="1048575"/> </table-cell></table-row><table-row number-rows-repeated="1048575"><table-cell> \n <p> \t </p> </table-cell></table-row></table></page></document>`, |
| 607 | }, |
| 608 | expected: ["Kept"], |
| 609 | absent: ["[Extracted text truncated", "\t"], |
| 610 | })), |
| 611 | ...["ods", "odt", "odp"].map((ext) => ({ |
| 612 | ext, |
| 613 | bounded: true, |
| 614 | parts: { |
| 615 | "content.xml": `<document><page><table><table-row> |
| 616 | <table-cell office:value-type="string" office:string-value="Harbor"/> |
| 617 | <table-cell office:value-type="boolean" office:boolean-value="false"/> |
| 618 | <table-cell office:value-type="date" office:date-value="2026-10-20"/> |
| 619 | <table-cell office:value-type="time" office:time-value="PT2H30M"/> |
| 620 | <table-cell office:value-type="currency" office:value="125.50" office:currency="USD"/> |
| 621 | <table-cell office:value-type="percentage" office:value="0.25"/> |
| 622 | <table-cell office:value-type="float" office:value="0"/> |
| 623 | <table-cell office:value-type="string" office:string-value="Fallback"><p> \n </p></table-cell> |
| 624 | <table-cell office:value-type="string" office:string-value="Unshown"><p>Displayed</p></table-cell> |
| 625 | <table-cell office:value-type="void" office:value="Unshown"/> |
| 626 | <table-cell office:value-type="currency" office:value=" " office:currency="Unshown"/> |
| 627 | <table-cell office:value-type="percentage" office:value=" "/> |
| 628 | </table-row></table></page></document>`, |
| 629 | }, |
| 630 | expected: [ |
| 631 | "Harbor\tfalse\t2026-10-20\tPT2H30M\t125.50 USD\t0.25 [ODF value-type=percentage]\t0\tFallback\tDisplayed", |
| 632 | ], |
| 633 | absent: ["Unshown", "[Extracted text truncated"], |
| 634 | })), |
| 635 | ...[false, true].map((rows) => ({ |
| 636 | ext: "xlsx", |
| 637 | bounded: true, |
| 638 | parts: { |
| 639 | "xl/workbook.xml": '<workbook><sheet name="Limited" r:id="a"/></workbook>', |
| 640 | "xl/_rels/workbook.xml.rels": |
| 641 | '<Relationships><Relationship Id="a" Target="worksheets/a.xml"/></Relationships>', |
| 642 | "xl/worksheets/a.xml": `<worksheet>${rows ? "<row><c><v>7</v></c></row>".repeat(10_001) : '<row><c r="A1"><v>7</v></c><c r="SS1"><v>Beyond</v></c></row>'}</worksheet>`, |
| 643 | }, |
| 644 | expected: ["7", "[Extracted text truncated at document extraction limits.]"], |
| 645 | absent: ["Beyond"], |
| 646 | })), |
| 647 | { |
| 648 | ext: "xlsx", |
| 649 | bounded: true, |
| 650 | parts: { |
| 651 | "xl/workbook.xml": '<workbook><sheet name="Padding" r:id="a"/></workbook>', |
| 652 | "xl/_rels/workbook.xml.rels": |
| 653 | '<Relationships><Relationship Id="a" Target="worksheets/a.xml"/></Relationships>', |
| 654 | "xl/sharedStrings.xml": "<sst><si><t> \n </t></si><si><t>Present</t></si></sst>", |
| 655 | "xl/worksheets/a.xml": `<worksheet>${'<row><c t="s"><v>0</v></c><c r="XFD1" t="inlineStr"><is><t> \t </t></is></c></row>'.repeat(10_001)}<row><c t="s"><v>0</v></c><c t="s"><v>1</v></c><c r="XFD2"><v> \n </v></c></row></worksheet>`, |
| 656 | }, |
| 657 | expected: ["Present"], |
| 658 | absent: ["[Extracted text truncated"], |
| 659 | }, |
| 660 | { |
| 661 | ext: "epub", |
| 662 | parts: { |
| 663 | "META-INF/container.xml": '<container><rootfile full-path="book.opf"/></container>', |
| 664 | "book.opf": |
| 665 | '<package><item id="bad" href="ch%ZZ.xhtml"/><item id="good" href="good.svg"/><itemref idref="bad"/><itemref idref="good"/></package>', |
| 666 | "good.svg": |
| 667 | '<svg xmlns="http://www.w3.org/2000/svg"><text>Readable SVG chapter</text></svg>', |
| 668 | }, |
| 669 | expected: [ |
| 670 | "Readable SVG chapter", |
| 671 | "[Skipped document part: chapter 1 (missing or invalid XML).]", |
| 672 | ], |
| 673 | absent: ["[Extracted text truncated"], |
| 674 | }, |
| 675 | ...[false, true].flatMap((bom) => |
| 676 | [false, true].map((bigEndian) => { |
| 677 | const encode = (source: string) => { |
| 678 | const data = Buffer.from(`${bom ? "\ufeff" : ""}${source}`, "utf16le"); |
| 679 | return bigEndian ? data.swap16() : data; |
| 680 | }; |
| 681 | return { |
| 682 | ext: "epub", |
| 683 | parts: { |
| 684 | "META-INF/container.xml": encode( |
| 685 | '<?xml version="1.0" encoding="UTF-16"?><container><rootfile full-path="book.opf"/></container>', |
| 686 | ), |
| 687 | "book.opf": encode( |
| 688 | '<?xml version="1.0" encoding="UTF-16"?><package><item id="a" href="a.xhtml"/><itemref idref="a"/></package>', |
| 689 | ), |
| 690 | "a.xhtml": encode( |
| 691 | '<?xml version="1.0" encoding="UTF-16"?><html><p>UTF-16 chapter: café 日本語</p></html>', |
| 692 | ), |
| 693 | }, |
| 694 | expected: ["UTF-16 chapter: café 日本語"], |
| 695 | absent: ["[Skipped document part", "[Extracted text truncated"], |
| 696 | }; |
| 697 | }), |
| 698 | ), |
| 699 | { |
| 700 | ext: "epub", |
| 701 | bounded: true, |
| 702 | parts: { |
| 703 | "META-INF/container.xml": '<container><rootfile full-path="book.opf"/></container>', |
| 704 | "book.opf": |
| 705 | '<package><item id="bad" href="bad.xhtml"/><item id="good" href="good.xhtml"/><itemref idref="bad"/><itemref idref="good"/></package>', |
| 706 | "bad.xhtml": "<html><p>Discard this broken chapter</html>", |
| 707 | "good.xhtml": |
| 708 | '<!DOCTYPE html PUBLIC "-//W3C//DTD XHTML 1.1//EN" "http://www.w3.org/TR/xhtml11/DTD/xhtml11.dtd"><html><p>North Harbor ≂̸ ∳ & <tag> "quoted"</p><p><![CDATA[Literal ]]></p><table><tr><td> </td></tr><tr><td>Kept cell</td></tr></table></html>', |
| 709 | }, |
| 710 | expected: [ |
| 711 | "## Chapter 2", |
| 712 | 'North\u00a0Harbor ≂̸ ∳ & <tag> "quoted"', |
| 713 | "Literal ", |
| 714 | "Kept cell", |
| 715 | "[Skipped document part: bad.xhtml (missing or invalid XML).]", |
| 716 | ], |
| 717 | absent: ["Discard this broken chapter", "[Extracted text truncated"], |
| 718 | }, |
| 719 | ...["pptx", "xlsx"].map((ext) => ({ |
| 720 | ext, |
| 721 | bounded: true, |
| 722 | parts: |
| 723 | ext === "pptx" |
| 724 | ? { |
| 725 | "ppt/presentation.xml": |
| 726 | '<presentation><sldIdLst><sldId r:id="bad"/><sldId r:id="good"/></sldIdLst></presentation>', |
| 727 | "ppt/_rels/presentation.xml.rels": |
| 728 | '<Relationships><Relationship Id="bad" Target="bad.xml"/><Relationship Id="good" Target="good.xml"/></Relationships>', |
| 729 | "ppt/bad.xml": "<slide><p>Discard</slide>", |
| 730 | "ppt/good.xml": "<slide><p>Healthy part</p></slide>", |
| 731 | } |
| 732 | : { |
| 733 | "xl/workbook.xml": |
| 734 | '<workbook><sheet name="Bad" r:id="bad"/><sheet name="Good" r:id="good"/></workbook>', |
| 735 | "xl/_rels/workbook.xml.rels": |
| 736 | '<Relationships><Relationship Id="bad" Target="bad.xml"/><Relationship Id="good" Target="good.xml"/></Relationships>', |
| 737 | "xl/bad.xml": "<worksheet><row>Discard</worksheet>", |
| 738 | "xl/good.xml": |
| 739 | '<worksheet><row><c t="inlineStr"><is><t>Healthy part</t></is></c></row></worksheet>', |
| 740 | }, |
| 741 | expected: [ |
| 742 | "Healthy part", |
| 743 | `[Skipped document part: ${ext === "pptx" ? "ppt" : "xl"}/bad.xml (missing or invalid XML).]`, |
| 744 | ], |
| 745 | absent: ["Discard", "[Extracted text truncated"], |
| 746 | })), |
| 747 | { |
| 748 | ext: "odt", |
| 749 | bounded: true, |
| 750 | parts: { "content.xml": '<document><p>Start<s c="1000001"/>Unneeded</p></document>' }, |
| 751 | expected: ["Start", "[Extracted text truncated at document extraction limits.]"], |
| 752 | absent: ["Unneeded"], |
| 753 | }, |
| 754 | { |
| 755 | ext: "docx", |
| 756 | bounded: true, |
| 757 | parts: { |
| 758 | "word/document.xml": |
| 759 | '<w:document xmlns:w="urn:w" xmlns:mc="urn:mc"><w:body><w:p><w:r><w:t>Body</w:t></w:r></w:p><w:p><w:r><mc:AlternateContent><mc:Choice Requires="wps"><w:drawing><wps:wsp><wps:txbx><w:txbxContent><w:p><w:r><w:t>Callout box</w:t></w:r></w:p></w:txbxContent></wps:txbx></wps:wsp></w:drawing></mc:Choice><mc:Fallback><w:pict><v:shape><v:textbox><w:txbxContent><w:p><w:r><w:t>Callout box</w:t></w:r></w:p></w:txbxContent></v:textbox></v:shape></w:pict></mc:Fallback></mc:AlternateContent></w:r></w:p></w:body></w:document>', |
| 760 | }, |
| 761 | expected: ["Body\nCallout box"], |
| 762 | occurrences: { text: "Callout box", count: 1 }, |
| 763 | }, |
| 764 | { |
| 765 | ext: "docx", |
| 766 | bounded: true, |
| 767 | parts: { |
| 768 | "word/document.xml": |
| 769 | '<w:document xmlns:w="urn:w" xmlns:mc="urn:mc"><w:body><w:p><w:r><mc:AlternateContent><mc:Choice Requires="cx"><w:drawing><cx:chart/></w:drawing></mc:Choice><mc:Fallback><w:pict><v:shape><v:textbox><w:txbxContent><w:p><w:r><w:t>Chart caption</w:t></w:r></w:p></w:txbxContent></v:textbox></v:shape></w:pict></mc:Fallback></mc:AlternateContent></w:r></w:p></w:body></w:document>', |
| 770 | }, |
| 771 | expected: ["Chart caption"], |
| 772 | }, |
| 773 | { |
| 774 | ext: "docx", |
| 775 | bounded: true, |
| 776 | parts: { |
| 777 | "word/document.xml": |
| 778 | '<w:document xmlns:w="urn:w"><w:body><w:p><w:pPr><w:pStyle w:val="Heading2"/></w:pPr><w:r><w:t>Report & totals</w:t></w:r></w:p><w:tbl><w:tr><w:tc><w:p><w:r><w:t>North</w:t></w:r></w:p></w:tc><w:tc><w:p><w:r><w:t>12</w:t></w:r></w:p><w:p><w:r><w:t>34</w:t></w:r></w:p></w:tc></w:tr><w:tr><w:tc><w:p><w:r><w:t>Before</w:t></w:r></w:p><w:tbl><w:tr><w:tc><w:p><w:r><w:t>Inner</w:t></w:r></w:p></w:tc></w:tr></w:tbl><w:p><w:r><w:t>After</w:t></w:r></w:p></w:tc><w:tc><w:p><w:r><w:t>Neighbor</w:t></w:r></w:p></w:tc><w:tc><w:p><w:r><w:noBreakHyphen/></w:r></w:p></w:tc></w:tr><w:tr><w:tc><w:p><w:r><w:noBreakHyphen/></w:r></w:p></w:tc></w:tr></w:tbl></w:body></w:document>', |
| 779 | }, |
| 780 | expected: [ |
| 781 | "## Report & totals", |
| 782 | "North\t12 34", |
| 783 | "Before Inner After\tNeighbor\t\u2011\n\u2011", |
| 784 | ], |
| 785 | }, |
| 786 | { |
| 787 | ext: "docx", |
| 788 | bounded: true, |
| 789 | parts: { |
| 790 | "word/document.xml": [ |
| 791 | '<w:document xmlns:w="urn:w" xmlns:mc="urn:mc"><w:body>', |
| 792 | '<w:p><w:r><w:t>First</w:t><w:cr/><w:t>Second</w:t><w:br/><w:t>Third</w:t><w:noBreakHyphen/><w:t>party</w:t><mc:AlternateContent><mc:Choice Requires="w"><w:noBreakHyphen/></mc:Choice><mc:Fallback><w:t>‑</w:t></mc:Fallback></mc:AlternateContent><w:t>End</w:t></w:r></w:p>', |
| 793 | '<w:p><w:r><w:t>Invoice 104</w:t><w:ptab w:alignment="right" w:relativeTo="margin" w:leader="none"/><w:t>$700.00</w:t></w:r></w:p>', |
| 794 | '<w:p><w:r><w:t>Invoice 105</w:t><mc:AlternateContent><mc:Choice Requires="w"><w:ptab w:alignment="right" w:relativeTo="margin" w:leader="none"/></mc:Choice><mc:Fallback><w:tab/></mc:Fallback></mc:AlternateContent><w:t>$800.00</w:t></w:r></w:p>', |
| 795 | '<w:p><w:r><w:t>Invoice 106</w:t><mc:AlternateContent><mc:Choice Requires="w"><w:ptab w:alignment="right" w:relativeTo="margin" w:leader="none"/></mc:Choice><mc:Fallback><w:t xml:space="preserve"> total </w:t></mc:Fallback></mc:AlternateContent><w:t>$900.00</w:t></w:r></w:p>', |
| 796 | "</w:body></w:document>", |
| 797 | ].join(""), |
| 798 | }, |
| 799 | expected: [ |
| 800 | "First\nSecond\nThird\u2011party\u2011End\nInvoice 104\t$700.00\nInvoice 105\t$800.00\nInvoice 106 total $900.00", |
| 801 | ], |
| 802 | }, |
| 803 | { |
| 804 | ext: "docx", |
| 805 | bounded: true, |
| 806 | parts: { |
| 807 | "word/document.xml": |
| 808 | '<w:document xmlns:w="urn:w"><w:body><w:p><w:r><w:t>Intro</w:t></w:r><w:r><w:fldChar w:fldCharType="begin"/></w:r><w:r><w:instrText> PAGEREF _Toc1 \\h </w:instrText></w:r><w:r><w:fldChar w:fldCharType="separate"/></w:r><w:r><w:t> 3</w:t></w:r><w:r><w:fldChar w:fldCharType="end"/></w:r></w:p><w:p><w:pPr><w:tabs><w:tab w:val="right" w:pos="9350"/></w:tabs></w:pPr><w:r><w:t xml:space="preserve">Fee </w:t></w:r><w:del><w:r><w:delText>$500</w:delText><w:noBreakHyphen/></w:r></w:del><w:ins><w:r><w:t>$700</w:t></w:r></w:ins><w:moveFrom><w:r><w:t>Moved</w:t></w:r></w:moveFrom><w:r><w:tab/><w:t>due</w:t></w:r></w:p></w:body></w:document>', |
| 809 | }, |
| 810 | expected: ["Intro 3\nFee $700\tdue"], |
| 811 | absent: ["PAGEREF", "$500", "Moved", "\u2011"], |
| 812 | }, |
| 813 | { |
| 814 | ext: "odt", |
| 815 | bounded: true, |
| 816 | parts: { |
| 817 | "content.xml": |
| 818 | "<document><text><tracked-changes><changed-region><deletion><change-info><creator>Reviewer</creator></change-info><p>$500</p></deletion></changed-region></tracked-changes><p>Fee <change/>$700</p></text></document>", |
| 819 | }, |
| 820 | expected: ["Fee $700"], |
| 821 | absent: ["Reviewer", "$500"], |
| 822 | }, |
| 823 | { |
| 824 | ext: "odt", |
| 825 | bounded: true, |
| 826 | parts: { |
| 827 | "content.xml": |
| 828 | '<document><text><p>Fee <office:annotation><dc:creator>Reviewer</dc:creator><dc:date>2026-01-01T00:00:00</dc:date><initials>JR</initials><p>Check</p></office:annotation>$700</p><table><table-row><table-cell office:value-type="float" office:value="150"><office:annotation><p>was 120</p></office:annotation></table-cell></table-row></table><p>Formula <math:math><math:semantics><math:mrow>x+y</math:mrow><math:annotation encoding="application/x-tex">x+y</math:annotation></math:semantics></math:math></p></text></document>', |
| 829 | }, |
| 830 | expected: ["Fee $700\n150\nFormula x+yx+y"], |
| 831 | absent: ["Reviewer", "2026-01-01", "JR", "Check", "was 120"], |
| 832 | }, |
| 833 | { |
| 834 | ext: "docx", |
| 835 | bounded: true, |
| 836 | parts: { |
| 837 | "word/document.xml": `<w:document><w:body><w:tbl><w:tr><w:tc><w:p><w:r><w:t>${"A".repeat(1_000_000)}</w:t></w:r></w:p></w:tc><w:tc><w:p><w:r><w:t>Later cell</w:t></w:r></w:p></w:tc></w:tr></w:tbl></w:body></w:document>`, |
| 838 | }, |
| 839 | expected: ["A".repeat(32), "[Extracted text truncated at document extraction limits.]"], |
| 840 | }, |
| 841 | { |
| 842 | ext: "xlsx", |
| 843 | bounded: true, |
| 844 | parts: { |
| 845 | "xl/workbook.xml": '<workbook><sheet name="Formulas" r:id="a"/></workbook>', |
| 846 | "xl/_rels/workbook.xml.rels": |
| 847 | '<Relationships><Relationship Id="a" Target="worksheets/a.xml"/></Relationships>', |
| 848 | "xl/styles.xml": '<styleSheet><cellXfs><xf numFmtId="9"/></cellXfs></styleSheet>', |
| 849 | "xl/worksheets/a.xml": |
| 850 | "<worksheet><row><c><f>SUM(11,22)</f><v/></c></row><row><c><f>SUM(33,44)</f><v> \n </v></c></row><row><c><f>SUM(55,66)</f></c></row><row><c><f>SUM(11,22)</f><v>33</v></c><c><f>SUM(0,0)</f><v>0</v></c></row></worksheet>", |
| 851 | }, |
| 852 | expected: [ |
| 853 | "SUM(11,22)\nSUM(33,44)\nSUM(55,66)", |
| 854 | "33 [Excel numFmtId=9; date system 1900]\t0 [Excel numFmtId=9; date system 1900]", |
| 855 | ], |
| 856 | absent: ["SUM(0,0)", "[Extracted text truncated"], |
| 857 | }, |
| 858 | { |
| 859 | ext: "xlsx", |
| 860 | bounded: true, |
| 861 | parts: { |
| 862 | "xl/workbook.xml": '<workbook><sheet name="Flags" r:id="a"/></workbook>', |
| 863 | "xl/_rels/workbook.xml.rels": |
| 864 | '<Relationships><Relationship Id="a" Target="worksheets/a.xml"/></Relationships>', |
| 865 | "xl/worksheets/a.xml": |
| 866 | '<worksheet><row><c r="A1" t="b"><v>1</v></c><c r="B1" t="b"><v>0</v></c><c r="C1" t="n"><v>1</v></c><c r="D1" t="b"><v>2</v></c></row></worksheet>', |
| 867 | }, |
| 868 | expected: ["true\tfalse\t1\t2"], |
| 869 | }, |
| 870 | { |
| 871 | ext: "xlsx", |
| 872 | parts: { |
| 873 | "xl/workbook.xml": |
| 874 | '<workbook><sheets><sheet name="Inventory" r:id="a"/><sheet name="Budget" r:id="b"/></sheets></workbook>', |
| 875 | "xl/_rels/workbook.xml.rels": |
| 876 | '<Relationships><Relationship Id="a" Target="worksheets/second.xml"/><Relationship Id="b" Target="worksheets/first.xml"/></Relationships>', |
| 877 | "xl/sharedStrings.xml": "<sst><si><t>Lantern</t></si></sst>", |
| 878 | "xl/styles.xml": |
| 879 | '<styleSheet><numFmts><numFmt numFmtId="164" formatCode="yyyy-mm-dd"/></numFmts><cellXfs><xf numFmtId="0"/><xf numFmtId="14"/><xf numFmtId="9"/><xf numFmtId="164"/></cellXfs></styleSheet>', |
| 880 | "xl/worksheets/second.xml": |
| 881 | '<worksheet><sheetData><row><c r="A1" t="s"><v>0</v></c><c r="C1"><v>37</v></c></row></sheetData></worksheet>', |
| 882 | "xl/worksheets/first.xml": |
| 883 | '<worksheet><sheetData><row><c t="inlineStr"><is><t>Ferry</t></is></c><c><v>628</v></c><c s="1"><v>45292</v></c><c s="2"><v>0.25</v></c><c s="3"><v>45293</v></c></row></sheetData></worksheet>', |
| 884 | }, |
| 885 | expected: [ |
| 886 | "## Sheet: Inventory", |
| 887 | "Lantern\t\t37", |
| 888 | "## Sheet: Budget", |
| 889 | "Ferry\t628", |
| 890 | "45292 [Excel numFmtId=14; date system 1900]", |
| 891 | "0.25 [Excel numFmtId=9; date system 1900]", |
| 892 | "45293 [Excel numFmtId=164; format=yyyy-mm-dd; date system 1900]", |
| 893 | ], |
| 894 | }, |
| 895 | { |
| 896 | ext: "xlsx", |
| 897 | bounded: true, |
| 898 | parts: { |
| 899 | "xl/workbook.xml": '<workbook><sheet name="Phonetics" r:id="a"/></workbook>', |
| 900 | "xl/_rels/workbook.xml.rels": |
| 901 | '<Relationships><Relationship Id="a" Target="worksheets/a.xml"/></Relationships>', |
| 902 | "xl/sharedStrings.xml": |
| 903 | '<sst><si><t>東京</t><rPh sb="0" eb="2"><t>とうきょう</t></rPh></si></sst>', |
| 904 | "xl/worksheets/a.xml": |
| 905 | '<worksheet><row><c t="s"><v>0</v></c><c t="inlineStr"><is><t>東京</t><rPh sb="0" eb="2"><t>とうきょう</t></rPh></is></c></row></worksheet>', |
| 906 | }, |
| 907 | expected: ["## Sheet: Phonetics", "東京\t東京"], |
| 908 | absent: ["とうきょう"], |
| 909 | }, |
| 910 | { |
| 911 | ext: "xlsx", |
| 912 | bounded: true, |
| 913 | parts: { |
| 914 | "xl/workbook.xml": '<workbook><sheet name="Escapes" r:id="a"/></workbook>', |
| 915 | "xl/_rels/workbook.xml.rels": |
| 916 | '<Relationships><Relationship Id="a" Target="worksheets/a.xml"/></Relationships>', |
| 917 | "xl/sharedStrings.xml": |
| 918 | "<sst><si><t>first_x000D_\nsecond</t></si><si><t>_x005F_x0041_</t></si><si><t>_x005F_x005F_x0041_</t></si><si><r><t>Bold_x000D_</t></r><r><t>_x005F_x000D_</t></r></si><si><t>&_xnothex_ cap_X0041_ _x000d_ tail</t></si></sst>", |
| 919 | "xl/worksheets/a.xml": |
| 920 | '<worksheet><row><c t="s"><v>0</v></c><c t="s"><v>1</v></c><c t="s"><v>2</v></c><c t="s"><v>3</v></c><c t="s"><v>4</v></c></row><row><c t="inlineStr"><is><t>_x005F_x005F_</t></is></c><c t="inlineStr"><is><t>first_x000D_second</t></is></c><c t="str"><f>A1</f><v>Total_x000D_due</v></c></row></worksheet>', |
| 921 | }, |
| 922 | expected: [ |
| 923 | "## Sheet: Escapes\nfirst second\t_x0041_\t_x005F_x0041_\tBold _x000D_\t&_xnothex_ cap_X0041_ tail\n_x005F_\tfirst second\tTotal due", |
| 924 | ], |
| 925 | }, |
| 926 | { |
| 927 | ext: "xlsx", |
| 928 | bounded: true, |
| 929 | parts: { |
| 930 | "xl/workbook.xml": |
| 931 | '<workbook><workbookPr date1904="true"/><sheets><sheet name="Dates" r:id="a"/></sheets></workbook>', |
| 932 | "xl/_rels/workbook.xml.rels": |
| 933 | '<Relationships><Relationship Id="a" Target="worksheets/dates.xml"/></Relationships>', |
| 934 | "xl/styles.xml": '<styleSheet><cellXfs><xf numFmtId="14"/></cellXfs></styleSheet>', |
| 935 | "xl/worksheets/dates.xml": |
| 936 | "<worksheet><sheetData><row><c><v>1</v></c></row></sheetData></worksheet>", |
| 937 | }, |
| 938 | expected: ["## Sheet: Dates", "1 [Excel numFmtId=14; date system 1904]"], |
| 939 | }, |
| 940 | { |
| 941 | ext: "xlsx", |
| 942 | bounded: true, |
| 943 | parts: { |
| 944 | "xl/workbook.xml": |
| 945 | '<workbook><sheets><sheet name="Repeated" r:id="a"/><sheet name="Unneeded" r:id="missing"/></sheets></workbook>', |
| 946 | "xl/_rels/workbook.xml.rels": |
| 947 | '<Relationships><Relationship Id="a" Target="worksheets/repeated.xml"/></Relationships>', |
| 948 | "xl/sharedStrings.xml": `<sst><si><t>${"A".repeat(1_000_000 - "## Sheet: Repeated\n".length)}</t></si></sst>`, |
| 949 | "xl/worksheets/repeated.xml": `<worksheet><sheetData>${'<row><c t="s"><v>0</v></c><c t="s"><v>0</v></c></row>'.repeat(1_500)}</sheetData></worksheet>`, |
| 950 | }, |
| 951 | expected: [ |
| 952 | "## Sheet: Repeated", |
| 953 | "A".repeat(32), |
| 954 | "[Extracted text truncated at document extraction limits.]", |
| 955 | ], |
| 956 | }, |
| 957 | { |
| 958 | ext: "pptx", |
| 959 | bounded: true, |
| 960 | parts: { |
| 961 | "ppt/presentation.xml": `<p:presentation xmlns:p="http://schemas.openxmlformats.org/presentationml/2006/main" |
| 962 | xmlns:r="http://schemas.openxmlformats.org/officeDocument/2006/relationships" |
| 963 | xmlns:p14="http://schemas.microsoft.com/office/powerpoint/2010/main"> |
| 964 | <p:sldIdLst><p:sldId id="256" r:id="b"/><p:sldId id="257" r:id="a"/></p:sldIdLst> |
| 965 | <p:extLst><p:ext uri="{521415D9-36F7-43E2-AB2F-B90AF26B5E84}"><p14:sectionLst> |
| 966 | <p14:section name="Schedule" id="{01F07B81-39E6-4BBB-9B89-66EA253FBD29}"> |
| 967 | <p14:sldIdLst><p14:sldId id="256"/><p14:sldId id="257"/></p14:sldIdLst> |
| 968 | </p14:section> |
| 969 | </p14:sectionLst></p:ext></p:extLst> |
| 970 | </p:presentation>`, |
| 971 | "ppt/_rels/presentation.xml.rels": |
| 972 | '<Relationships><Relationship Id="a" Target="slides/slide1.xml"/><Relationship Id="b" Target="slides/slide2.xml"/></Relationships>', |
| 973 | "ppt/slides/slide1.xml": "<p:sld><a:p><a:r><a:t>Departure</a:t></a:r></a:p></p:sld>", |
| 974 | "ppt/slides/slide2.xml": "<p:sld><a:p><a:r><a:t>Juniper Pier</a:t></a:r></a:p></p:sld>", |
| 975 | "ppt/slides/_rels/slide2.xml.rels": |
| 976 | '<Relationships><Relationship Type="http://schemas.openxmlformats.org/officeDocument/2006/relationships/slideLayout" Target="../notesSlides/ignored.xml"/><Relationship Type="http://schemas.openxmlformats.org/officeDocument/2006/relationships/notesSlide" TargetMode="External" Target="https://example.com/notes.xml"/><Relationship Type="http://schemas.openxmlformats.org/officeDocument/2006/relationships/notesSlide" Target="../notesSlides/notes7.xml"/></Relationships>', |
| 977 | "ppt/slides/_rels/slide1.xml.rels": |
| 978 | '<Relationships><Relationship Type="http://purl.oclc.org/ooxml/officeDocument/relationships/notesSlide" Target="/ppt/notesSlides/notes3.xml"/></Relationships>', |
| 979 | "ppt/notesSlides/notes7.xml": |
| 980 | "<p:notes><a:p><a:r><a:t>evacuation code ORCHID-81</a:t></a:r></a:p></p:notes>", |
| 981 | "ppt/notesSlides/notes3.xml": |
| 982 | "<p:notes><a:p><a:r><a:t>Board at gate 4</a:t></a:r></a:p></p:notes>", |
| 983 | "ppt/notesSlides/ignored.xml": "<p:notes><a:p>Unrelated notes</a:p></p:notes>", |
| 984 | }, |
| 985 | expected: [ |
| 986 | "## Slide 1", |
| 987 | "Juniper Pier\nNotes:\nevacuation code ORCHID-81", |
| 988 | "## Slide 2", |
| 989 | "Departure\nNotes:\nBoard at gate 4", |
| 990 | ], |
| 991 | occurrences: { text: "## Slide ", count: 2 }, |
| 992 | absent: ["Unrelated notes", "[Skipped document part", "[Extracted text truncated"], |
| 993 | }, |
| 994 | ...[false, true].map((missing) => ({ |
| 995 | ext: "pptx", |
| 996 | bounded: true, |
| 997 | parts: { |
| 998 | "ppt/presentation.xml": |
| 999 | '<presentation><sldIdLst><sldId r:id="a"/></sldIdLst></presentation>', |
| 1000 | "ppt/_rels/presentation.xml.rels": |
| 1001 | '<Relationships><Relationship Id="a" Target="slides/a.xml"/></Relationships>', |
| 1002 | "ppt/slides/a.xml": "<slide><p>Slide text</p></slide>", |
| 1003 | "ppt/slides/_rels/a.xml.rels": `<Relationships><Relationship Type="http://schemas.openxmlformats.org/officeDocument/2006/relationships/notesSlide" Target="../notesSlides/${missing ? "missing" : "bad"}.xml"/></Relationships>`, |
| 1004 | "ppt/notesSlides/bad.xml": "<notes><p>Discard broken notes</notes>", |
| 1005 | }, |
| 1006 | expected: [ |
| 1007 | "Slide text", |
| 1008 | `[Skipped document part: ppt/notesSlides/${missing ? "missing" : "bad"}.xml (missing or invalid XML).]`, |
| 1009 | ], |
| 1010 | absent: ["Discard broken notes", "[Extracted text truncated"], |
| 1011 | })), |
| 1012 | { |
| 1013 | ext: "pptx", |
| 1014 | bounded: true, |
| 1015 | parts: { |
| 1016 | "ppt/presentation.xml": |
| 1017 | '<presentation><sldIdLst><sldId r:id="a"/><sldId r:id="b"/></sldIdLst></presentation>', |
| 1018 | "ppt/_rels/presentation.xml.rels": |
| 1019 | '<Relationships><Relationship Id="a" Target="slides/a.xml"/><Relationship Id="b" Target="slides/b.xml"/></Relationships>', |
| 1020 | "ppt/slides/a.xml": `<slide><p>${"A".repeat(600_000)}</p></slide>`, |
| 1021 | "ppt/slides/b.xml": "<slide><p>Unneeded slide</p></slide>", |
| 1022 | "ppt/slides/_rels/a.xml.rels": |
| 1023 | '<Relationships><Relationship Type="http://schemas.openxmlformats.org/officeDocument/2006/relationships/notesSlide" Target="../notesSlides/a.xml"/></Relationships>', |
| 1024 | "ppt/notesSlides/a.xml": `<notes><p>evacuation code ORCHID-81 ${"B".repeat(600_000)} Unneeded notes</p></notes>`, |
| 1025 | }, |
| 1026 | expected: [ |
| 1027 | "## Slide 1", |
| 1028 | "A".repeat(32), |
| 1029 | "Notes:\nevacuation code ORCHID-81", |
| 1030 | "B".repeat(32), |
| 1031 | "[Extracted text truncated at document extraction limits.]", |
| 1032 | ], |
| 1033 | absent: ["Unneeded notes", "Unneeded slide", "## Slide 2"], |
| 1034 | }, |
| 1035 | { |
| 1036 | ext: "odt", |
| 1037 | parts: { |
| 1038 | "content.xml": |
| 1039 | '<office:document><text:h text:outline-level="2">\n Orchard\n</text:h><text:p>19<text:s text:c="3"/>violet<text:s/>crates <text:span>\n\t shipped</text:span> today<text:tab/>Dock<text:line-break/>Gate\u00a0B </text:p><text:p>Second</text:p><text:p>A<text:span><text:script>console.log(1);\n</text:script></text:span> B</text:p></office:document>', |
| 1040 | }, |
| 1041 | expected: [ |
| 1042 | "## Orchard\n19 violet crates shipped today\tDock\nGate\u00a0B\nSecond\nA B", |
| 1043 | ], |
| 1044 | }, |
| 1045 | { |
| 1046 | ext: "ods", |
| 1047 | parts: { |
| 1048 | "content.xml": |
| 1049 | '<office:document><table:table table:name="Cargo"><table:table-row><table:table-cell><text:p>Violet</text:p></table:table-cell><table:table-cell office:value="19"/><table:table-cell office:value="27"><text:p/></table:table-cell></table:table-row></table:table></office:document>', |
| 1050 | }, |
| 1051 | expected: ["## Sheet: Cargo", "Violet\t19\t27"], |
| 1052 | }, |
| 1053 | { |
| 1054 | ext: "ods", |
| 1055 | bounded: true, |
| 1056 | parts: { |
| 1057 | "content.xml": |
| 1058 | '<office:document><table:table table:name="Padded rows"><table:table-row><table:table-cell><text:p>Left</text:p></table:table-cell><table:table-cell table:number-columns-repeated="2"/><table:table-cell><text:p>Right</text:p></table:table-cell></table:table-row><table:table-row table:number-rows-repeated="1048575"><table:table-cell/></table:table-row></table:table></office:document>', |
| 1059 | }, |
| 1060 | expected: ["## Sheet: Padded rows", "Left\t\t\tRight"], |
| 1061 | absent: ["[Extracted text truncated"], |
| 1062 | }, |
| 1063 | { |
| 1064 | ext: "ods", |
| 1065 | bounded: true, |
| 1066 | parts: { |
| 1067 | "content.xml": |
| 1068 | '<office:document><table:table table:name="Padded"><table:table-row><table:table-cell><text:p>Populated</text:p></table:table-cell><table:table-cell table:number-columns-repeated="16383"/></table:table-row><table:table-row table:number-rows-repeated="1048575"><table:table-cell table:number-columns-repeated="16384"><text:p/></table:table-cell></table:table-row></table:table></office:document>', |
| 1069 | }, |
| 1070 | expected: ["## Sheet: Padded", "Populated"], |
| 1071 | absent: ["[Extracted text truncated", "\t"], |
| 1072 | }, |
| 1073 | { |
| 1074 | ext: "ods", |
| 1075 | bounded: true, |
| 1076 | parts: { |
| 1077 | "content.xml": `<office:document><table:table table:name="Repeated"><table:table-row><table:table-cell table:number-columns-repeated="512"><text:p>Lead ${"a".repeat(600_000)} Tail</text:p></table:table-cell></table:table-row></table:table></office:document>`, |
| 1078 | }, |
| 1079 | expected: [ |
| 1080 | "## Sheet: Repeated", |
| 1081 | "Lead ", |
| 1082 | " Tail\tLead ", |
| 1083 | "[Extracted text truncated at document extraction limits.]", |
| 1084 | ], |
| 1085 | }, |
| 1086 | { |
| 1087 | ext: "odp", |
| 1088 | parts: { |
| 1089 | "content.xml": |
| 1090 | "<office:document><draw:page><draw:frame><draw:text-box><text:p>\n First\n\t slide </text:p></draw:text-box></draw:frame></draw:page><draw:page><text:p>Second</text:p></draw:page></office:document>", |
| 1091 | }, |
| 1092 | expected: ["## Slide 1\nFirst slide\n\n## Slide 2\nSecond"], |
| 1093 | }, |
| 1094 | { |
| 1095 | ext: "ods", |
| 1096 | bounded: true, |
| 1097 | parts: { |
| 1098 | "content.xml": `<office:document>${Array.from( |
| 1099 | { length: 256 }, |
| 1100 | (_, index) => |
| 1101 | `<table:table table:name="${index === 0 ? "Shared" : "Unneeded"}"><table:table-row><table:table-cell table:number-columns-repeated="512"><text:p>${"B".repeat(4096)}</text:p></table:table-cell></table:table-row></table:table>`, |
| 1102 | ).join("")}</office:document>`, |
| 1103 | }, |
| 1104 | expected: [ |
| 1105 | "## Sheet: Shared", |
| 1106 | "B".repeat(32), |
| 1107 | "[Extracted text truncated at document extraction limits.]", |
| 1108 | ], |
| 1109 | absent: ["## Sheet: Unneeded"], |
| 1110 | }, |
| 1111 | { |
| 1112 | ext: "odt", |
| 1113 | bounded: true, |
| 1114 | parts: { |
| 1115 | "content.xml": `<office:document><text:p>Before whitespace</text:p><table:table><table:table-row table:number-rows-repeated="10000"><table:table-cell table:number-columns-repeated="512"><text:p>${" ".repeat(10_000)}</text:p></table:table-cell></table:table-row></table:table><text:p>Unneeded</text:p></office:document>`, |
| 1116 | }, |
| 1117 | expected: ["Before whitespace", "Unneeded"], |
| 1118 | absent: ["[Extracted text truncated"], |
| 1119 | }, |
| 1120 | ...["odt", "odp", "ods"].flatMap((ext) => |
| 1121 | [false, true].map((rows) => ({ |
| 1122 | ext, |
| 1123 | bounded: true, |
| 1124 | parts: { |
| 1125 | "content.xml": `<office:document><draw:page>${Array.from( |
| 1126 | { length: 256 }, |
| 1127 | () => |
| 1128 | `<table:table><table:table-row${rows ? ' table:number-rows-repeated="10000"' : ""}><table:table-cell table:number-columns-repeated="512"><text:p>${"Budget ".repeat(rows ? 160 : 585)}</text:p></table:table-cell></table:table-row></table:table>`, |
| 1129 | ).join( |
| 1130 | "", |
| 1131 | )}</draw:page><draw:page><text:p>Unneeded</text:p></draw:page></office:document>`, |
| 1132 | }, |
| 1133 | expected: ["Budget ", "[Extracted text truncated at document extraction limits.]"], |
| 1134 | absent: ["Unneeded"], |
| 1135 | })), |
| 1136 | ), |
| 1137 | { |
| 1138 | ext: "docx", |
| 1139 | bounded: true, |
| 1140 | parts: { |
| 1141 | "word/document.xml": `<w:document><w:p><w:t>${"A".repeat(1_000_100)}</w:t></w:p><w:p><w:t>Unneeded</w:t></w:p></w:document>`, |
| 1142 | }, |
| 1143 | expected: ["A".repeat(32), "[Extracted text truncated at document extraction limits.]"], |
| 1144 | absent: ["Unneeded"], |
| 1145 | }, |
| 1146 | { |
| 1147 | ext: "pptx", |
| 1148 | bounded: true, |
| 1149 | parts: { |
| 1150 | "ppt/presentation.xml": |
| 1151 | '<p:presentation><p:sldIdLst><p:sldId r:id="a"/><p:sldId r:id="missing"/></p:sldIdLst></p:presentation>', |
| 1152 | "ppt/_rels/presentation.xml.rels": |
| 1153 | '<Relationships><Relationship Id="a" Target="slides/slide.xml"/></Relationships>', |
| 1154 | "ppt/slides/slide.xml": `<p:sld><a:p><a:t>${"A".repeat(1_000_100)}</a:t></a:p></p:sld>`, |
| 1155 | }, |
| 1156 | expected: [ |
| 1157 | "## Slide 1", |
| 1158 | "A".repeat(32), |
| 1159 | "[Extracted text truncated at document extraction limits.]", |
| 1160 | ], |
| 1161 | absent: ["## Slide 2"], |
| 1162 | }, |
| 1163 | { |
| 1164 | ext: "epub", |
| 1165 | bounded: true, |
| 1166 | parts: { |
| 1167 | "META-INF/container.xml": '<container><rootfile full-path="book.opf"/></container>', |
| 1168 | "book.opf": |
| 1169 | '<package><item id="a" href="a.xhtml"/><itemref idref="a"/><itemref idref="missing"/></package>', |
| 1170 | "a.xhtml": `<html><body><p>${"A".repeat(1_000_100)}</p></body></html>`, |
| 1171 | }, |
| 1172 | expected: [ |
| 1173 | "## Chapter 1", |
| 1174 | "A".repeat(32), |
| 1175 | "[Extracted text truncated at document extraction limits.]", |
| 1176 | ], |
| 1177 | absent: ["## Chapter 2"], |
| 1178 | }, |
| 1179 | { |
| 1180 | ext: "epub", |
| 1181 | parts: { |
| 1182 | "META-INF/container.xml": |
| 1183 | '<container><rootfile full-path="OEBPS/book.opf"/></container>', |
| 1184 | "OEBPS/book.opf": |
| 1185 | '<package><manifest><item id="a" href="a.xhtml"/><item id="b" href="b.xhtml"/></manifest><spine><itemref idref="b"/><itemref idref="a"/></spine></package>', |
| 1186 | "OEBPS/a.xhtml": |
| 1187 | '<!DOCTYPE html PUBLIC "-//W3C//DTD XHTML 1.1//EN" "http://www.w3.org/TR/xhtml11/DTD/xhtml11.dtd"><html><body><p>Last chapter</p><p>Price <s>10</s> 8 <del>old</del></p></body></html>', |
| 1188 | "OEBPS/b.xhtml": |
| 1189 | '<!DOCTYPE html><html><head><title>My Novel</title><meta name="author" content="Hidden Author"/></head><body><h1>First chapter</h1><p>Larkspur</p><div>Direct div<section>Direct section<li>Direct item <em>emphasis</em> trailing</li></section></div><div>Account: A<br/>Balance: 731<line-break/>Currency: USD</div><table><thead><tr><th>Heading</th><th>Total</th></tr></thead><tbody><tr><td>Important value</td><td>731</td></tr><tr><td>Before<table><tr><td>Inner</td></tr></table>After</td><td>Neighbor</td></tr></tbody></table></body></html>', |
| 1190 | }, |
| 1191 | expected: [ |
| 1192 | "## Chapter 1", |
| 1193 | "\n# First chapter", |
| 1194 | "Larkspur", |
| 1195 | "Direct div", |
| 1196 | "Direct section", |
| 1197 | "Direct item emphasis trailing", |
| 1198 | "Account: A\nBalance: 731\nCurrency: USD", |
| 1199 | "Heading\tTotal", |
| 1200 | "Important value\t731", |
| 1201 | "Before Inner After\tNeighbor", |
| 1202 | "## Chapter 2", |
| 1203 | "Last chapter", |
| 1204 | "Price 10 8 old", |
| 1205 | ], |
| 1206 | absent: ["My Novel", "Hidden Author"], |
| 1207 | }, |
| 1208 | ]; |
| 1209 | for (const [caseIndex, item] of cases.entries()) { |
| 1210 | const local: Buffer[] = []; |
| 1211 | const central: Buffer[] = []; |
| 1212 | let offset = 0; |
| 1213 | for (const [name, content] of Object.entries(item.parts)) { |
| 1214 | const data = Buffer.from(content!); |
| 1215 | const compressed = /\.(bmp|bin)$/.test(name) ? NodeZlib.deflateRawSync(data) : data; |
| 1216 | const filename = Buffer.from(name); |
| 1217 | const header = Buffer.alloc(30); |
| 1218 | header.writeUInt32LE(0x04034b50); |
| 1219 | header.writeUInt16LE(20, 4); |
| 1220 | header.writeUInt16LE(compressed === data ? 0 : 8, 8); |
| 1221 | header.writeUInt32LE(NodeZlib.crc32(data), 14); |
| 1222 | header.writeUInt32LE(compressed.length, 18); |
| 1223 | header.writeUInt32LE(data.length, 22); |
| 1224 | header.writeUInt16LE(filename.length, 26); |
| 1225 | const directory = Buffer.alloc(46); |
| 1226 | directory.writeUInt32LE(0x02014b50); |
| 1227 | directory.writeUInt16LE(20, 4); |
| 1228 | directory.writeUInt16LE(20, 6); |
| 1229 | directory.writeUInt16LE(compressed === data ? 0 : 8, 10); |
| 1230 | directory.writeUInt32LE(NodeZlib.crc32(data), 16); |
| 1231 | directory.writeUInt32LE(compressed.length, 20); |
| 1232 | directory.writeUInt32LE(data.length, 24); |
| 1233 | directory.writeUInt16LE(filename.length, 28); |
| 1234 | directory.writeUInt32LE(offset, 42); |
| 1235 | local.push(header, filename, compressed); |
| 1236 | central.push(directory, filename); |
| 1237 | offset += header.length + filename.length + compressed.length; |
| 1238 | } |
| 1239 | const end = Buffer.alloc(22); |
| 1240 | end.writeUInt32LE(0x06054b50); |
| 1241 | end.writeUInt16LE(central.length / 2, 8); |
| 1242 | end.writeUInt16LE(central.length / 2, 10); |
| 1243 | end.writeUInt32LE(Buffer.concat(central).length, 12); |
| 1244 | end.writeUInt32LE(offset, 16); |
| 1245 | const path = NodePath.join(config.attachmentsDir, `structure-${caseIndex}.${item.ext}`); |
| 1246 | NodeFS.writeFileSync(path, Buffer.concat([...local, ...central, end])); |
| 1247 | const extracted = yield* Effect.promise(() => extractDocumentText(path, `.${item.ext}`)); |
| 1248 | let previous = -1; |
| 1249 | for (const expected of item.expected) { |
| 1250 | const index = extracted.indexOf(expected); |
| 1251 | expect(index, `${item.ext}: ${expected} in ${extracted.slice(0, 500)}`).toBeGreaterThan( |
| 1252 | previous, |
| 1253 | ); |
| 1254 | previous = index; |
| 1255 | } |
| 1256 | if ("occurrences" in item) |
| 1257 | expect(extracted.split(item.occurrences.text).length - 1).toBe(item.occurrences.count); |
| 1258 | if ("absent" in item) |
| 1259 | for (const absent of item.absent) expect(extracted).not.toContain(absent); |
| 1260 | if (("bounded" in item && item.bounded) || item.ext === "epub") { |
| 1261 | expect(extracted.length).toBeLessThan(1_000_100); |
| 1262 | expect(extracted.match(/\[Extracted text truncated/g)?.length ?? 0).toBeLessThanOrEqual( |
| 1263 | 1, |
| 1264 | ); |
| 1265 | const documents = yield* DocumentText.DocumentText; |
| 1266 | expect(yield* documents.ensure(path)).toBe(path + ".txt"); |
| 1267 | expect(NodeFS.readFileSync(path + ".txt", "utf8")).toBe(extracted); |
| 1268 | } |
| 1269 | if (item.ext === "docx") { |
| 1270 | end.writeUInt16LE(2049, 8); |
| 1271 | end.writeUInt16LE(2049, 10); |
| 1272 | NodeFS.writeFileSync(path, Buffer.concat([...local, ...central, end])); |
| 1273 | yield* Effect.promise(async () => { |
| 1274 | expect(await extractDocumentText(path, ".docx")).toContain("[Extracted text truncated"); |
| 1275 | }); |
| 1276 | end.writeUInt16LE(central.length / 2, 8); |
| 1277 | end.writeUInt16LE(central.length / 2, 10); |
| 1278 | central[0]!.writeUInt16LE(8, 10); |
| 1279 | central[0]!.writeUInt32LE(33 * 1024 * 1024, 24); |
| 1280 | NodeFS.writeFileSync(path, Buffer.concat([...local, ...central, end])); |
| 1281 | yield* Effect.promise(async () => { |
| 1282 | expect(await extractDocumentText(path, ".docx")).toContain("[Extracted text truncated"); |
| 1283 | }); |
| 1284 | } |
| 1285 | } |
| 1286 | const path = NodePath.join(config.attachmentsDir, "unicode.rtf"); |
| 1287 | NodeFS.writeFileSync( |
| 1288 | path, |
| 1289 | String.raw`{\rtf1\ansi{\fonttbl hidden}Hello\tab world\par Unicode \u945?{\*\secret hidden}}`, |
| 1290 | ); |
| 1291 | const extracted = yield* Effect.promise(() => extractDocumentText(path, ".rtf")); |
| 1292 | expect(extracted).toContain("Hello\tworld\nUnicode α"); |
| 1293 | expect(extracted).not.toContain("hidden"); |
| 1294 | |
| 1295 | for (const [name, content, expected] of [ |
| 1296 | [ |
| 1297 | "cyrillic-escaped", |
| 1298 | String.raw`{\rtf1\ansi\ansicpg1251 \'cf\'f0\'e8\'e2\'e5\'f2 {\ansicpg1252 \'e9} \'cc\'ee\'f1\'ea\'e2\'e0}`, |
| 1299 | "Привет é Москва", |
| 1300 | ], |
| 1301 | [ |
| 1302 | "cyrillic-raw", |
| 1303 | Buffer.concat([ |
| 1304 | Buffer.from(String.raw`{\rtf1\ansi\ansicpg1251 `), |
| 1305 | Buffer.from([0xcf, 0xf0, 0xe8, 0xe2, 0xe5, 0xf2]), |
| 1306 | Buffer.from("}"), |
| 1307 | ]), |
| 1308 | "Привет", |
| 1309 | ], |
| 1310 | ] as const) { |
| 1311 | const encodedPath = NodePath.join(config.attachmentsDir, `${name}.rtf`); |
| 1312 | NodeFS.writeFileSync(encodedPath, content); |
| 1313 | expect(yield* Effect.promise(() => extractDocumentText(encodedPath, ".rtf"))).toBe( |
| 1314 | expected, |
| 1315 | ); |
| 1316 | const documents = yield* DocumentText.DocumentText; |
| 1317 | expect(yield* documents.ensure(encodedPath)).toBe(encodedPath + ".txt"); |
| 1318 | expect(NodeFS.readFileSync(encodedPath + ".txt", "utf8")).toBe(expected); |
| 1319 | } |
| 1320 | const notebookPath = NodePath.join(config.attachmentsDir, "limited.ipynb"); |
| 1321 | NodeFS.writeFileSync( |
| 1322 | notebookPath, |
| 1323 | JSON.stringify({ cells: Array.from({ length: 10_001 }, () => ({ source: "Kept" })) }), |
| 1324 | ); |
| 1325 | const notebookLimited = yield* Effect.promise(() => |
| 1326 | extractDocumentText(notebookPath, ".ipynb"), |
| 1327 | ); |
| 1328 | expect(notebookLimited).toContain("## Cell 10000"); |
| 1329 | expect(notebookLimited).not.toContain("## Cell 10001"); |
| 1330 | expect(notebookLimited.match(/\[Extracted text truncated/g)).toHaveLength(1); |
| 1331 | expect(yield* (yield* DocumentText.DocumentText).ensure(notebookPath)).toBe( |
| 1332 | notebookPath + ".txt", |
| 1333 | ); |
| 1334 | expect(NodeFS.readFileSync(notebookPath + ".txt", "utf8")).toBe(notebookLimited); |
| 1335 | const errorPath = NodePath.join(config.attachmentsDir, "limited-error.ipynb"); |
| 1336 | NodeFS.writeFileSync( |
| 1337 | errorPath, |
| 1338 | JSON.stringify({ |
| 1339 | cells: [ |
| 1340 | { |
| 1341 | cell_type: "code", |
| 1342 | source: "1 / 0", |
| 1343 | outputs: [ |
| 1344 | { |
| 1345 | output_type: "error", |
| 1346 | ename: "ZeroDivisionError", |
| 1347 | evalue: "division by zero", |
| 1348 | traceback: ["\u001b[31m" + "trace".repeat(250_000) + "\u001b[0m", "After limit"], |
| 1349 | }, |
| 1350 | ], |
| 1351 | }, |
| 1352 | ], |
| 1353 | }), |
| 1354 | ); |
| 1355 | const errorLimited = yield* Effect.promise(() => extractDocumentText(errorPath, ".ipynb")); |
| 1356 | expect(errorLimited).toContain("ZeroDivisionError: division by zero\ntrace"); |
| 1357 | expect(errorLimited.length).toBeLessThan(1_000_100); |
| 1358 | expect(errorLimited.match(/\[Extracted text truncated/g)).toHaveLength(1); |
| 1359 | expect(errorLimited).not.toContain("After limit"); |
| 1360 | expect(errorLimited).not.toContain("\u001b"); |
| 1361 | const deepPath = NodePath.join(config.attachmentsDir, "deep.rtf"); |
| 1362 | NodeFS.writeFileSync( |
| 1363 | deepPath, |
| 1364 | String.raw`{\rtf1 Kept` + "{".repeat(128) + "Lost" + "}".repeat(129), |
| 1365 | ); |
| 1366 | const deepLimited = yield* Effect.promise(() => extractDocumentText(deepPath, ".rtf")); |
| 1367 | expect(deepLimited).toBe("Kept\n\n[Extracted text truncated at document extraction limits.]"); |
| 1368 | const unsupportedPath = NodePath.join(config.attachmentsDir, "unsupported-codepage.rtf"); |
| 1369 | NodeFS.writeFileSync(unsupportedPath, String.raw`{\rtf1\ansi\ansicpg932 \'82\'a0}`); |
| 1370 | yield* Effect.promise(async () => { |
| 1371 | await expect(extractDocumentText(unsupportedPath, ".rtf")).rejects.toThrow( |
| 1372 | "Unsupported RTF code page", |
| 1373 | ); |
| 1374 | }); |
| 1375 | expect(yield* (yield* DocumentText.DocumentText).ensure(unsupportedPath)).toBeUndefined(); |
| 1376 | expect(NodeFS.existsSync(unsupportedPath + ".txt")).toBe(false); |
| 1377 | NodeFS.writeFileSync( |
| 1378 | path, |
| 1379 | String.raw`{\rtf1 ` + "x".repeat(999_999) + String.raw`\u-10179?\u-8576? After}`, |
| 1380 | ); |
| 1381 | const unicodeTruncated = yield* Effect.promise(() => extractDocumentText(path, ".rtf")); |
| 1382 | expect(unicodeTruncated).toContain("[Extracted text truncated"); |
| 1383 | expect(unicodeTruncated.isWellFormed()).toBe(true); |
| 1384 | expect(unicodeTruncated.startsWith("x".repeat(999_999))).toBe(true); |
| 1385 | NodeFS.writeFileSync(path, String.raw`{\rtf1 First` + " ".repeat(1_000_100) + "Last}"); |
| 1386 | const whitespaceTruncated = yield* Effect.promise(() => extractDocumentText(path, ".rtf")); |
| 1387 | expect(whitespaceTruncated).toBe( |
| 1388 | "First\n\n[Extracted text truncated at document extraction limits.]", |
| 1389 | ); |
| 1390 | const documents = yield* DocumentText.DocumentText; |
| 1391 | expect(yield* documents.ensure(path)).toBe(path + ".txt"); |
| 1392 | expect(NodeFS.readFileSync(path + ".txt", "utf8")).toBe(whitespaceTruncated); |
| 1393 | NodeFS.truncateSync(path, 17 * 1024 * 1024); |
| 1394 | yield* Effect.promise(async () => { |
| 1395 | expect(await extractDocumentText(path, ".rtf")).toContain("[Extracted text truncated"); |
| 1396 | }); |
| 1397 | }).pipe(Effect.provide(layer)), |
| 1398 | ); |
| 1399 | it.effect("keeps selected values intact and appends an actual readable server path", () => |
| 1400 | Effect.gen(function* () { |
| 1401 | const config = yield* ServerConfig.ServerConfig; |
| 1402 | const path = NodePath.join(config.attachmentsDir, `${attachment.id}.txt`); |
| 1403 | NodeFS.writeFileSync(path, "spec"); |
| 1404 | const original = { q: ["First", "Second"], other: "No file" }; |
| 1405 | const answers = yield* appendUserInputAttachmentPaths({ |
| 1406 | answers: original, |
| 1407 | attachmentsDir: config.attachmentsDir, |
| 1408 | attachmentsByQuestionId: { q: [attachment] }, |
| 1409 | }); |
| 1410 | expect(answers.q).toEqual([ |
| 1411 | "First", |
| 1412 | "Second", |
| 1413 | `Attached file "spec \\"final\\".txt": ${JSON.stringify(path)}`, |
| 1414 | ]); |
| 1415 | expect(answers.other).toBe("No file"); |
| 1416 | expect(original.q).toEqual(["First", "Second"]); |
| 1417 | expect(NodeFS.readFileSync(path, "utf8")).toBe("spec"); |
| 1418 | const specialKey = yield* appendUserInputAttachmentPaths({ |
| 1419 | answers: {}, |
| 1420 | attachmentsDir: config.attachmentsDir, |
| 1421 | attachmentsByQuestionId: { ["__proto__"]: [attachment] }, |
| 1422 | }); |
| 1423 | expect(Object.keys(specialKey)).toEqual(["__proto__"]); |
| 1424 | expect(specialKey["__proto__"]).toContain(JSON.stringify(path)); |
| 1425 | }).pipe(Effect.provide(layer)), |
| 1426 | ); |
| 1427 | it.effect("does not send an unavailable attachment path", () => |
| 1428 | Effect.gen(function* () { |
| 1429 | const config = yield* ServerConfig.ServerConfig; |
| 1430 | const result = yield* appendUserInputAttachmentPaths({ |
| 1431 | answers: { q: "" }, |
| 1432 | attachmentsDir: config.attachmentsDir, |
| 1433 | attachmentsByQuestionId: { q: [attachment] }, |
| 1434 | }).pipe(Effect.result); |
| 1435 | expect(result._tag).toBe("Failure"); |
| 1436 | }).pipe(Effect.provide(layer)), |
| 1437 | ); |
| 1438 | }); |