@devries-cameron20 / t3code Mirror · read-only
Fork 0

The best way to control Claude Code, Codex and more from any computer or phone

Mirrored from https://github.com/Camo67/t3code.git

Synced Nov 21, 04:45 UTC

main
ts 786 lines 29.4 KB
Raw
1 // @effect-diagnostics nodeBuiltinImport:off -- isolated CLI worker; no server runtime is loaded here.
2 import * as NodeFSP from "node:fs/promises";
3 import * as NodePath from "node:path";
4 import * as NodeUtil from "node:util";
5 import { characterEntities } from "character-entities";
6 import { SaxesParser } from "saxes";
7 import * as Yauzl from "yauzl";
8
9 const MAX_INPUT = 16 * 1024 * 1024;
10 const MAX_EXPANDED = 32 * 1024 * 1024;
11 const MAX_OUTPUT = 1_000_000;
12 const CELL_START = Symbol("cell-start");
13 const CELL_END = Symbol("cell-end");
14 type TextPart = string | typeof CELL_START | typeof CELL_END | { fallback: string };
15
16 const TRUNCATED = "\n\n[Extracted text truncated at document extraction limits.]";
17 // Parts the document never displays: the XHTML head, tracked deletions and moves, field codes,
18 // phonetic guides, paragraph properties such as tab-stop definitions, drawing anchor layout
19 // values and encoded field data.
20 const HIDDEN = new Set([
21 "head",
22 "script",
23 "style",
24 "binary-data",
25 "rPh",
26 "delText",
27 "moveFrom",
28 "instrText",
29 "delInstrText",
30 "tracked-changes",
31 "pPr",
32 "positionH",
33 "positionV",
34 "sizeRelH",
35 "sizeRelV",
36 "fldData",
37 ]);
38
39 // @effect-diagnostics-next-line extendsNativeError:off - this worker loads no Effect runtime; Truncation is a local throw sentinel.
40 class Truncation extends Error {}
41
42 interface XmlNode {
43 name: string;
44 attrs: Record<string, string>;
45 children: Array<XmlNode | string>;
46 spaceCount?: number;
47 hasText?: boolean;
48 hasTextChoice?: boolean;
49 }
50
51 // ODF 1.3 Part 3 §6.1.2: paragraph character data turns TAB, CR and LF into spaces, collapses
52 // space runs and trims the ends; only <text:s>, <text:tab> and <text:line-break> keep literal ones.
53 function collapseOdfWhitespace(paragraph: XmlNode) {
54 let afterSpace = true;
55 let tail: { node: XmlNode; index: number; value: string } | undefined;
56 const visit = (node: XmlNode) => {
57 for (const [index, child] of node.children.entries()) {
58 if (typeof child !== "string") {
59 if (HIDDEN.has(child.name)) continue;
60 if (child.spaceCount === undefined && !["tab", "line-break", "p", "h"].includes(child.name))
61 visit(child);
62 else {
63 afterSpace = false;
64 tail = undefined;
65 }
66 continue;
67 }
68 let value = child.replaceAll(/[ \t\r\n]+/g, " ");
69 if (afterSpace && value.startsWith(" ")) value = value.slice(1);
70 if (value) {
71 afterSpace = value.endsWith(" ");
72 tail = { node, index, value };
73 }
74 node.children[index] = value;
75 }
76 };
77 visit(paragraph);
78 if (tail && afterSpace) tail.node.children[tail.index] = tail.value.slice(0, -1);
79 }
80
81 function xml(source: string, extension: string): XmlNode {
82 const odf = [".odt", ".ods", ".odp"].includes(extension);
83 const root: XmlNode = { name: "root", attrs: {}, children: [] };
84 const stack = [root];
85 let nodes = 0;
86 const parser = new SaxesParser();
87 if (extension === ".epub") Object.assign(parser.ENTITIES, characterEntities);
88 parser.on("opentag", (tag) => {
89 if (++nodes > 200_000 || stack.length > 128) throw new Truncation("XML structure limit");
90 const node: XmlNode = {
91 name: tag.name.split(":").at(-1)!,
92 attrs: Object.fromEntries(
93 Object.entries(tag.attributes).flatMap(([key, value]) => [
94 [key, String(value)],
95 [key.split(":").at(-1)!, String(value)],
96 ]),
97 ),
98 children: [],
99 };
100 if (odf && node.name === "s") node.spaceCount = repeatCount(node.attrs.c);
101 stack.at(-1)!.children.push(node);
102 stack.push(node);
103 });
104 parser.on("closetag", () => {
105 const node = stack.pop()!;
106 const parent = stack.at(-1)!;
107 if (odf && (node.name === "p" || node.name === "h")) collapseOdfWhitespace(node);
108 if (
109 (extension === ".docx" && node.name === "del") ||
110 (odf && node.name === "annotation" && parent.name !== "semantics")
111 ) {
112 parent.children.pop();
113 return;
114 }
115 if (node.name === "Fallback" && !retainCompatibilityFallback(parent, node)) return;
116 if (HIDDEN.has(node.name) || (!node.hasText && node.name !== "noBreakHyphen")) return;
117 parent.hasText = true;
118 if (node.name === "Choice") parent.hasTextChoice = true;
119 });
120 const text = (value: string) => {
121 const node = stack.at(-1)!;
122 node.children.push(value);
123 if (value.trim() !== "") node.hasText = true;
124 };
125 parser.on("text", text);
126 parser.on("cdata", text);
127 parser.write(source).close();
128 return root;
129 }
130
131 const children = (node: XmlNode | undefined) =>
132 node?.children.filter((child): child is XmlNode => typeof child !== "string") ?? [];
133
134 function retainCompatibilityFallback(parent: XmlNode, fallback: XmlNode): boolean {
135 // Markup-compatibility fallbacks repeat their mc:Choice content, e.g. Word text boxes.
136 // Keep one when no Choice carries text, such as a chart with a VML caption.
137 if (parent.hasTextChoice) {
138 parent.children.pop();
139 return false;
140 }
141 if (fallback.hasText || hasInlineTab(fallback, ["tab", "ptab"])) {
142 parent.children = parent.children.filter(
143 (child) =>
144 typeof child === "string" || child.name !== "Choice" || !hasInlineTab(child, ["ptab"]),
145 );
146 }
147 return true;
148 }
149
150 function hasInlineTab(node: XmlNode, names: readonly string[]): boolean {
151 return (
152 !HIDDEN.has(node.name) &&
153 (names.includes(node.name) || children(node).some((child) => hasInlineTab(child, names)))
154 );
155 }
156 function* all(node: XmlNode, name: string | readonly string[]): Generator<XmlNode, void> {
157 for (const child of children(node)) {
158 if (typeof name === "string" ? child.name === name : name.includes(child.name)) yield child;
159 else yield* all(child, name);
160 }
161 }
162 function* text(node: XmlNode): Generator<string, void> {
163 if (HIDDEN.has(node.name)) return;
164 if (["tab", "ptab", "cell"].includes(node.name)) yield "\t";
165 else if (["br", "line-break", "cr"].includes(node.name)) yield "\n";
166 else if (node.name === "noBreakHyphen") yield "\u2011";
167 else if (node.spaceCount !== undefined) {
168 yield " ".repeat(Math.min(MAX_OUTPUT, node.spaceCount));
169 if (node.spaceCount > MAX_OUTPUT) throw new Truncation("Space repeat limit");
170 } else
171 for (const child of node.children) {
172 if (typeof child === "string") yield child;
173 else yield* text(child);
174 }
175 }
176 // SpreadsheetML `t` and `v` values are ST_Xstring: `_xHHHH_` is one UTF-16 code unit, and an
177 // escaped lead underscore (`_x005F_xHHHH_`) keeps the literal `_xHHHH_`. Decoded in one pass.
178 function* xstringText(node: XmlNode): Generator<string, void> {
179 if (HIDDEN.has(node.name)) return;
180 if (node.name !== "t" && node.name !== "v") {
181 for (const child of children(node)) yield* xstringText(child);
182 return;
183 }
184 yield node.children
185 .filter((child) => typeof child === "string")
186 .join("")
187 .replaceAll(
188 /_x005[Ff]_(x[\dA-Fa-f]{4}_)|_x([\dA-Fa-f]{4})_/g,
189 (_, literal?: string, hex?: string) =>
190 literal ? `_${literal}` : String.fromCharCode(Number.parseInt(hex!, 16)),
191 );
192 }
193 function* paragraphs(node: XmlNode): Generator<TextPart, void> {
194 if (HIDDEN.has(node.name)) return;
195 for (const child of node.children) {
196 if (typeof child === "string") yield child;
197 else if (["br", "line-break", "cr"].includes(child.name)) yield "\n";
198 else if (["tbl", "table"].includes(child.name)) yield* table(child);
199 else if (["p", "h"].includes(child.name) || /^h[1-6]$/.test(child.name)) {
200 const style = all(child, "pStyle").next().value?.attrs.val ?? "";
201 const level =
202 child.name === "h"
203 ? Number(child.attrs["outline-level"]) || 1
204 : /^h[1-6]$/.test(child.name)
205 ? Number(child.name[1])
206 : Number(style.match(/^Heading([1-6])$/i)?.[1]) || 0;
207 if (level) yield `${"#".repeat(Math.min(6, level))} `;
208 yield* text(child);
209 yield "\n";
210 } else {
211 yield* paragraphs(child);
212 if (["li", "div", "section"].includes(child.name)) yield "\n";
213 }
214 }
215 }
216 function* inCell(parts: Iterable<TextPart>): Generator<TextPart, void> {
217 yield CELL_START;
218 yield* parts;
219 yield CELL_END;
220 }
221 function repeatCount(value: string | undefined): number {
222 if (value === undefined) return 1;
223 if (!/^[0-9]+$/.test(value) || Number(value) < 1) throw new Error("Invalid repeat count");
224 return Number(value);
225 }
226 function odfValue(cell: XmlNode): string {
227 const type = cell.attrs["office:value-type"] ?? cell.attrs["value-type"];
228 const attr = (name: string) => cell.attrs[`office:${name}`] ?? cell.attrs[name] ?? "";
229 switch (type) {
230 case "void":
231 return "";
232 case "string":
233 return attr("string-value");
234 case "boolean":
235 return attr("boolean-value");
236 case "date":
237 return attr("date-value");
238 case "time":
239 return attr("time-value");
240 case "currency":
241 return attr("value").trim() ? `${attr("value")} ${attr("currency")}`.trim() : "";
242 case "percentage":
243 return attr("value").trim() ? `${attr("value")} [ODF value-type=percentage]` : "";
244 default:
245 return attr("value");
246 }
247 }
248 function hasCellText(cell: XmlNode): boolean {
249 if (odfValue(cell).trim() || cell.name === "noBreakHyphen") return true;
250 // Space markers can have enormous repeat counts; emptiness never requires expanding them.
251 if (HIDDEN.has(cell.name) || cell.spaceCount !== undefined) return false;
252 return cell.children.some((child) =>
253 typeof child === "string" ? !!child.trim() : hasCellText(child),
254 );
255 }
256 function* table(node: XmlNode): Generator<TextPart, void> {
257 let rowCount = 0;
258 for (const row of all(node, ["tr", "table-row"])) {
259 const cells = children(row).filter((cell) =>
260 ["tc", "table-cell", "covered-table-cell", "td", "th"].includes(cell.name),
261 );
262 // ODS writers pad sheets with repeated empty rows and trailing cells.
263 while (cells.length && !hasCellText(cells.at(-1)!)) cells.pop();
264 if (!cells.length) continue;
265 const repeat = repeatCount(row.attrs["number-rows-repeated"]);
266 for (let index = 0; index < repeat; index++) {
267 if (++rowCount > 10_000) throw new Truncation("Row limit");
268 let column = 0;
269 for (const cell of cells) {
270 const count = repeatCount(cell.attrs["number-columns-repeated"]);
271 for (let i = 0; i < count; i++) {
272 if (column >= 512) throw new Truncation("Column limit");
273 if (column++) yield "\t";
274 yield CELL_START;
275 yield* paragraphs(cell);
276 yield { fallback: odfValue(cell) };
277 yield CELL_END;
278 }
279 }
280 yield "\n";
281 }
282 }
283 }
284
285 async function archive(path: string): Promise<Map<string, string>> {
286 const zip = await new Promise<Yauzl.ZipFile>((resolve, reject) =>
287 Yauzl.open(
288 path,
289 { lazyEntries: true, validateEntrySizes: true, strictFileNames: true },
290 (error, result) => (error || !result ? reject(error) : resolve(result)),
291 ),
292 );
293 const files = new Map<string, string>();
294 let expanded = 0;
295 let count = 0;
296 try {
297 await new Promise<void>((resolve, reject) => {
298 zip.on("error", reject);
299 zip.on("end", resolve);
300 zip.on("entry", (entry: Yauzl.Entry) => {
301 if (++count > 2048 || zip.entryCount > 2048) {
302 reject(new Truncation("Archive limit"));
303 return;
304 }
305 if (entry.generalPurposeBitFlag & 1) {
306 reject(new Error("Encrypted archive"));
307 return;
308 }
309 if (!/\.(xml|rels|xhtml|html|htm|opf|svg)$/i.test(entry.fileName)) {
310 zip.readEntry();
311 return;
312 }
313 if (
314 entry.uncompressedSize > MAX_EXPANDED ||
315 entry.uncompressedSize > Math.max(1024, entry.compressedSize * 200)
316 ) {
317 reject(new Truncation("Archive limit"));
318 return;
319 }
320 expanded += entry.uncompressedSize;
321 if (expanded > MAX_EXPANDED) {
322 reject(new Truncation("Expanded byte limit"));
323 return;
324 }
325 zip.openReadStream(entry, (error, stream) => {
326 if (error || !stream) {
327 reject(error);
328 return;
329 }
330 const buffers: Buffer[] = [];
331 let size = 0;
332 stream.on("error", reject);
333 stream.on("data", (chunk: Buffer) => {
334 size += chunk.length;
335 if (size > entry.uncompressedSize || size > MAX_EXPANDED) {
336 stream.destroy(new Truncation("Entry byte limit"));
337 return;
338 }
339 buffers.push(chunk);
340 });
341 stream.on("end", () => {
342 const data = Buffer.concat(buffers);
343 // XML's BOM and initial '<' bytes distinguish both UTF-16 byte orders.
344 const encoding =
345 (data[0] === 0xff && data[1] === 0xfe) || (data[0] === 0x3c && data[1] === 0)
346 ? "utf-16le"
347 : (data[0] === 0xfe && data[1] === 0xff) || (data[0] === 0 && data[1] === 0x3c)
348 ? "utf-16be"
349 : "utf-8";
350 files.set(entry.fileName, new TextDecoder(encoding).decode(data));
351 zip.readEntry();
352 });
353 });
354 });
355 zip.readEntry();
356 });
357 } finally {
358 zip.close();
359 }
360 return files;
361 }
362
363 const NOTEBOOK_CELL_TYPES = ["code", "markdown", "raw"];
364
365 function* notebook(source: string): Generator<string, void> {
366 const book: unknown = JSON.parse(source);
367 if (!book || typeof book !== "object" || !("cells" in book) || !Array.isArray(book.cells))
368 throw new Error("Invalid notebook");
369 function* lines(value: unknown): Generator<string, void> {
370 if (typeof value === "string") yield value;
371 else if (Array.isArray(value))
372 for (const line of value) {
373 if (typeof line === "string") yield line;
374 }
375 }
376 function outputText(value: unknown): string {
377 // Notebook fragments can split a terminal escape sequence.
378 return NodeUtil.stripVTControlCharacters([...lines(value)].join(""));
379 }
380 for (const [index, cell] of book.cells.entries()) {
381 if (index >= 10_000) throw new Truncation("Notebook cell limit");
382 if (!cell || typeof cell !== "object") continue;
383 const cellType = "cell_type" in cell ? cell.cell_type : undefined;
384 const kind = NOTEBOOK_CELL_TYPES.find((type) => type === cellType) ?? "markdown";
385 yield `## Cell ${index + 1} (${kind})\n`;
386 if ("source" in cell) yield* lines(cell.source);
387 if ("outputs" in cell && Array.isArray(cell.outputs)) {
388 yield "\nOutput:\n";
389 for (const output of cell.outputs) {
390 if (!output || typeof output !== "object") continue;
391 if ("text" in output) yield outputText(output.text);
392 else if (
393 "data" in output &&
394 output.data &&
395 typeof output.data === "object" &&
396 "text/plain" in output.data
397 )
398 yield outputText(output.data["text/plain"]);
399 else if ("output_type" in output && output.output_type === "error") {
400 if ("ename" in output) yield* lines(output.ename);
401 if ("evalue" in output) {
402 yield ": ";
403 yield* lines(output.evalue);
404 }
405 yield "\n";
406 if ("traceback" in output)
407 for (const line of lines(output.traceback)) {
408 yield NodeUtil.stripVTControlCharacters(line);
409 yield "\n";
410 }
411 }
412 yield "\n";
413 }
414 }
415 yield "\n\n";
416 }
417 }
418
419 function* rtf(source: string): Generator<string, void> {
420 if (!source.startsWith("{\\rtf")) throw new Error("Invalid RTF document");
421 let state = { skip: false, deleted: false, uc: 1, decoder: new TextDecoder("windows-1252") };
422 const stack: Array<typeof state> = [];
423 const hidden = () => state.skip || state.deleted;
424 const symbols = new Map([
425 ["endash", "–"],
426 ["emdash", characterEntities.mdash],
427 ["enspace", "\u2002"],
428 ["emspace", "\u2003"],
429 ["qmspace", "\u2005"],
430 ["lquote", "‘"],
431 ["rquote", "’"],
432 ["ldblquote", "“"],
433 ["rdblquote", "”"],
434 ["bullet", "•"],
435 ["zwj", "\u200d"],
436 ["zwnj", "\u200c"],
437 ["ltrmark", "\u200e"],
438 ["rtlmark", "\u200f"],
439 ["~", "\u00a0"],
440 ["_", "\u2011"],
441 ["-", ""],
442 ]);
443 let fallback = 0;
444 for (let i = 0; i < source.length; i++) {
445 const char = source[i]!;
446 if (char === "{" || char === "}") fallback = 0;
447 if (char === "{") {
448 stack.push({ ...state });
449 if (stack.length > 128) throw new Truncation("RTF depth limit");
450 } else if (char === "}") state = stack.pop() ?? state;
451 else if (char === "\\") {
452 const rest = source.slice(i + 1, i + 80);
453 const hex = /^'([0-9a-f]{2})/i.exec(rest);
454 const word = /^([a-z]+)(-?\d+)? ?/i.exec(rest);
455 if (hex) {
456 if (fallback) fallback--;
457 else if (!hidden()) yield state.decoder.decode(Uint8Array.of(Number.parseInt(hex[1]!, 16)));
458 i += 3;
459 } else if (word) {
460 i += word[0].length;
461 const control = word[1]!;
462 const value = Number(word[2]);
463 if (control === "bin") {
464 if (!Number.isSafeInteger(value) || value < 0)
465 throw new Error("Invalid RTF binary length");
466 i += value;
467 }
468 if (fallback) {
469 fallback--;
470 continue;
471 }
472 if (
473 [
474 "fonttbl",
475 "colortbl",
476 "stylesheet",
477 "info",
478 "pict",
479 "object",
480 "fldinst",
481 "header",
482 "headerf",
483 "headerl",
484 "headerr",
485 "footer",
486 "footerf",
487 "footerl",
488 "footerr",
489 ].includes(control)
490 )
491 state.skip = true;
492 if (control === "deleted") state.deleted = value !== 0;
493 else if (control === "plain") state.deleted = false;
494 if (control === "mac" || (control === "ansicpg" && value === 10000))
495 state.decoder = new TextDecoder("macintosh");
496 else if (control === "ansicpg") {
497 // Per-byte decoding only supports single-byte Windows code pages.
498 if (value !== 874 && (value < 1250 || value > 1258))
499 throw new Error("Unsupported RTF code page");
500 state.decoder = new TextDecoder(`windows-${value}`, { fatal: true });
501 } else if (control === "uc") {
502 if (!Number.isSafeInteger(value) || value < 0)
503 throw new Error("Invalid RTF fallback count");
504 state.uc = value;
505 } else if (control === "u") {
506 if (!hidden()) yield String.fromCharCode(value < 0 ? value + 65536 : value);
507 fallback = state.uc;
508 } else if (!hidden() && ["par", "line", "sect", "page", "column"].includes(control))
509 yield "\n";
510 else if (!hidden() && control === "tab") yield "\t";
511 // Table structure survives tracked deletions; only the deleted cell text is dropped.
512 else if (!state.skip && control === "row") yield "\n";
513 else if (!state.skip && control === "cell") yield "\t";
514 else if (!hidden() && symbols.has(control)) yield symbols.get(control)!;
515 } else {
516 i++;
517 if (fallback) fallback--;
518 else if (rest[0] === "*") state.skip = true;
519 // A backslash before a line break is \par; TextEdit ends paragraphs this way.
520 else if (!hidden() && (rest[0] === "\n" || rest[0] === "\r")) yield "\n";
521 else if (!hidden() && ["\\", "{", "}"].includes(rest[0]!)) yield rest[0]!;
522 else if (!hidden() && symbols.has(rest[0]!)) yield symbols.get(rest[0]!)!;
523 }
524 } else if (char !== "\n" && char !== "\r") {
525 if (fallback) fallback--;
526 else if (!hidden())
527 yield char.charCodeAt(0) < 128
528 ? char
529 : state.decoder.decode(Uint8Array.of(char.charCodeAt(0)));
530 }
531 }
532 }
533
534 async function* document(path: string, extension: string): AsyncGenerator<TextPart, void> {
535 if ((await NodeFSP.stat(path)).size > MAX_INPUT) throw new Truncation("Input byte limit");
536 if (extension === ".ipynb") yield* notebook(await NodeFSP.readFile(path, "utf8"));
537 else if (extension === ".rtf") yield* rtf((await NodeFSP.readFile(path)).toString("latin1"));
538 else {
539 const files = await archive(path);
540 const notes: string[] = [];
541 const skip = (name: string) =>
542 notes.push(`\n[Skipped document part: ${name} (missing or invalid XML).]\n`);
543 const read = (name: string): XmlNode => {
544 try {
545 const data = files.get(name);
546 if (!data) throw new Error("Missing document part");
547 return xml(data, extension);
548 } catch (error) {
549 if (error instanceof Truncation) throw error;
550 skip(name);
551 return { name: "root", attrs: {}, children: [] };
552 }
553 };
554 const relations = (name: string) =>
555 new Map(
556 [...all(read(name), "Relationship")]
557 .filter((rel) => rel.attrs.TargetMode !== "External")
558 .map((rel) => [rel.attrs.Id, rel.attrs.Target]),
559 );
560 if (extension === ".docx") yield* paragraphs(read("word/document.xml"));
561 else if ([".odt", ".ods", ".odp"].includes(extension)) {
562 const content = read("content.xml");
563 if (extension === ".odt") yield* paragraphs(content);
564 else {
565 let index = 0;
566 for (const part of all(content, extension === ".ods" ? "table" : "page")) {
567 yield extension === ".ods"
568 ? `\n\n## Sheet: ${part.attrs.name ?? "Untitled"}\n`
569 : `\n\n## Slide ${++index}\n`;
570 if (extension === ".ods") yield* table(part);
571 else yield* paragraphs(part);
572 }
573 }
574 } else if (extension === ".xlsx") {
575 const workbook = read("xl/workbook.xml");
576 const dateSystem = ["1", "true"].includes(
577 all(workbook, "workbookPr").next().value?.attrs.date1904 ?? "",
578 )
579 ? "1904"
580 : "1900";
581 const styles = files.has("xl/styles.xml") ? read("xl/styles.xml") : undefined;
582 const customFormats = new Map(
583 styles
584 ? [...all(styles, "numFmt")].map((node) => [node.attrs.numFmtId, node.attrs.formatCode])
585 : [],
586 );
587 const cellFormats = styles ? all(styles, "cellXfs").next().value : undefined;
588 const formats = cellFormats ? children(cellFormats) : [];
589 const strings = files.has("xl/sharedStrings.xml")
590 ? [...all(read("xl/sharedStrings.xml"), "si")]
591 : [];
592 const populatedStrings = strings.map(hasCellText);
593 const rels = relations("xl/_rels/workbook.xml.rels");
594 for (const sheet of all(workbook, "sheet")) {
595 yield `\n## Sheet: ${sheet.attrs.name}\n`;
596 const target = rels.get(sheet.attrs["r:id"] ?? sheet.attrs.id);
597 if (!target) {
598 skip(`sheet relationship ${sheet.attrs.name}`);
599 continue;
600 }
601 const name = target.startsWith("/")
602 ? target.slice(1)
603 : NodePath.posix.normalize(`xl/${target}`);
604 const populated = (cell: XmlNode) => {
605 const value = all(cell, "v").next().value;
606 if (cell.attrs.t === "s") {
607 const raw = value?.children.find((child) => typeof child === "string")?.trim();
608 return !!raw && !!populatedStrings[Number(raw)];
609 }
610 if (cell.attrs.t === "inlineStr") return hasCellText(cell);
611 if (value && hasCellText(value)) return true;
612 for (const formula of all(cell, "f")) if (hasCellText(formula)) return true;
613 return false;
614 };
615 let rowCount = 0;
616 for (const row of all(read(name), "row")) {
617 const cells: Array<XmlNode | undefined> = [];
618 let nextColumn = 0;
619 let columnsTruncated = false;
620 for (const cell of children(row).filter((node) => node.name === "c")) {
621 const letters = cell.attrs.r?.match(/^[A-Z]+/)?.[0];
622 const column = letters
623 ? [...letters].reduce((n, char) => n * 26 + char.charCodeAt(0) - 64, 0) - 1
624 : nextColumn;
625 nextColumn = column + 1;
626 if (!populated(cell)) continue;
627 if (column >= 512) {
628 columnsTruncated = true;
629 continue;
630 }
631 cells[column] = cell;
632 }
633 if (!cells.length && !columnsTruncated) continue;
634 if (++rowCount > 10_000) throw new Truncation("Row limit");
635 for (const [column, cell] of cells.entries()) {
636 if (column) yield "\t";
637 if (!cell) continue;
638 const value = all(cell, "v").next().value;
639 const raw = value?.children.find((child) => typeof child === "string") ?? "";
640 if (cell.attrs.t === "s") {
641 const shared = strings[Number(raw)];
642 if (shared) yield* inCell(xstringText(shared));
643 } else if (cell.attrs.t === "inlineStr") yield* inCell(xstringText(cell));
644 else if (cell.attrs.t === "b" && (raw.trim() === "0" || raw.trim() === "1"))
645 yield* inCell([raw.trim() === "1" ? "true" : "false"]);
646 else {
647 if (value && hasCellText(value)) yield* inCell(xstringText(value));
648 else for (const formula of all(cell, "f")) yield* inCell(text(formula));
649 const formatId = formats[Number(cell.attrs.s ?? 0)]?.attrs.numFmtId;
650 const format = formatId ? customFormats.get(formatId) : undefined;
651 if (
652 (cell.attrs.t === undefined || cell.attrs.t === "n") &&
653 raw.trim() &&
654 formatId &&
655 formatId !== "0"
656 )
657 yield ` [Excel numFmtId=${formatId}${format ? `; format=${format}` : ""}; date system ${dateSystem}]`;
658 }
659 }
660 yield "\n";
661 if (columnsTruncated) throw new Truncation("Column limit");
662 }
663 }
664 } else if (extension === ".pptx") {
665 const rels = relations("ppt/_rels/presentation.xml.rels");
666 const [presentation] = children(read("ppt/presentation.xml"));
667 const slideList = children(presentation).find((node) => node.name === "sldIdLst");
668 let index = 0;
669 for (const slide of children(slideList).filter((node) => node.name === "sldId")) {
670 yield `\n\n## Slide ${++index}\n`;
671 const target = rels.get(slide.attrs["r:id"] ?? slide.attrs.id);
672 if (!target) {
673 skip(`slide relationship ${index}`);
674 continue;
675 }
676 const name = target.startsWith("/")
677 ? target.slice(1)
678 : NodePath.posix.normalize(`ppt/${target}`);
679 yield* paragraphs(read(name));
680 const slideRels = NodePath.posix.join(
681 NodePath.posix.dirname(name),
682 "_rels",
683 `${NodePath.posix.basename(name)}.rels`,
684 );
685 if (!files.has(slideRels)) continue;
686 for (const rel of all(read(slideRels), "Relationship")) {
687 const target = rel.attrs.Target;
688 if (
689 !rel.attrs.Type?.endsWith("/notesSlide") ||
690 rel.attrs.TargetMode === "External" ||
691 !target
692 )
693 continue;
694 yield "Notes:\n";
695 yield* paragraphs(
696 read(
697 target.startsWith("/")
698 ? target.slice(1)
699 : NodePath.posix.join(NodePath.posix.dirname(name), target),
700 ),
701 );
702 }
703 }
704 } else if (extension === ".epub") {
705 const rootfile = all(read("META-INF/container.xml"), "rootfile").next().value?.attrs[
706 "full-path"
707 ];
708 if (!rootfile) {
709 if (!notes.length) skip("EPUB package");
710 yield* notes;
711 return;
712 }
713 const packageXml = read(rootfile);
714 const items = new Map(
715 [...all(packageXml, "item")].map((item) => [item.attrs.id, item.attrs.href]),
716 );
717 let index = 0;
718 for (const item of all(packageXml, "itemref")) {
719 yield `\n\n## Chapter ${++index}\n`;
720 const href = items.get(item.attrs.idref);
721 if (!href) {
722 skip(`chapter ${index}`);
723 continue;
724 }
725 let decoded: string;
726 try {
727 decoded = decodeURIComponent(href.split("#")[0]!);
728 } catch {
729 skip(`chapter ${index}`);
730 continue;
731 }
732 const name = NodePath.posix.normalize(
733 NodePath.posix.join(NodePath.posix.dirname(rootfile), decoded),
734 );
735 yield* paragraphs(read(name));
736 }
737 } else throw new Error("Unsupported document");
738 yield* notes;
739 }
740 }
741
742 export async function extractDocumentText(path: string, extension: string): Promise<string> {
743 let result = "";
744 let remaining = MAX_OUTPUT;
745 let cellDepth = 0;
746 let started = false;
747 let space = false;
748 let truncated = false;
749 try {
750 // Charge raw text before normalization so repeated whitespace also exhausts the budget.
751 for await (const part of document(path, extension)) {
752 if (part === CELL_START) {
753 if (cellDepth++ === 0) {
754 started = false;
755 space = false;
756 } else if (started) space = true;
757 continue;
758 }
759 if (part === CELL_END) {
760 cellDepth--;
761 if (cellDepth > 0 && started) space = true;
762 continue;
763 }
764 if (typeof part !== "string" && started) continue;
765 const raw = typeof part === "string" ? part : part.fallback;
766 let value = raw.slice(0, remaining).replaceAll("\0", "");
767 remaining -= Math.min(raw.length, remaining);
768 if (cellDepth) {
769 value = value.replaceAll(/\s+/g, " ");
770 const trimmed = value.trim();
771 if (trimmed) {
772 result += (started && (space || value.startsWith(" ")) ? " " : "") + trimmed;
773 started = true;
774 }
775 space = value.endsWith(" ") || (!value && space);
776 } else result += value;
777 if (remaining === 0) throw new Truncation("Output budget");
778 }
779 } catch (error) {
780 if (!(error instanceof Truncation)) throw error;
781 truncated = true;
782 }
783 result = result.replaceAll(/\n(?:[ \t]*\n){2,}/g, "\n\n").trim();
784 if (!result && !truncated) throw new Error("No text extracted");
785 return truncated ? result.replace(/[\uD800-\uDBFF]$/, "").trimEnd() + TRUNCATED : result;
786 }