The best way to control Claude Code, Codex and more from any computer or phone
Mirrored from https://github.com/Camo67/t3code.git
Synced Nov 21, 04:45 UTC
| 1 | // @effect-diagnostics nodeBuiltinImport:off -- isolated CLI worker; no server runtime is loaded here. |
| 2 | import * as NodeFSP from "node:fs/promises"; |
| 3 | import * as NodePath from "node:path"; |
| 4 | import * as NodeUtil from "node:util"; |
| 5 | import { characterEntities } from "character-entities"; |
| 6 | import { SaxesParser } from "saxes"; |
| 7 | import * as Yauzl from "yauzl"; |
| 8 | |
| 9 | const MAX_INPUT = 16 * 1024 * 1024; |
| 10 | const MAX_EXPANDED = 32 * 1024 * 1024; |
| 11 | const MAX_OUTPUT = 1_000_000; |
| 12 | const CELL_START = Symbol("cell-start"); |
| 13 | const CELL_END = Symbol("cell-end"); |
| 14 | type TextPart = string | typeof CELL_START | typeof CELL_END | { fallback: string }; |
| 15 | |
| 16 | const TRUNCATED = "\n\n[Extracted text truncated at document extraction limits.]"; |
| 17 | // Parts the document never displays: the XHTML head, tracked deletions and moves, field codes, |
| 18 | // phonetic guides, paragraph properties such as tab-stop definitions, drawing anchor layout |
| 19 | // values and encoded field data. |
| 20 | const HIDDEN = new Set([ |
| 21 | "head", |
| 22 | "script", |
| 23 | "style", |
| 24 | "binary-data", |
| 25 | "rPh", |
| 26 | "delText", |
| 27 | "moveFrom", |
| 28 | "instrText", |
| 29 | "delInstrText", |
| 30 | "tracked-changes", |
| 31 | "pPr", |
| 32 | "positionH", |
| 33 | "positionV", |
| 34 | "sizeRelH", |
| 35 | "sizeRelV", |
| 36 | "fldData", |
| 37 | ]); |
| 38 | |
| 39 | // @effect-diagnostics-next-line extendsNativeError:off - this worker loads no Effect runtime; Truncation is a local throw sentinel. |
| 40 | class Truncation extends Error {} |
| 41 | |
| 42 | interface XmlNode { |
| 43 | name: string; |
| 44 | attrs: Record<string, string>; |
| 45 | children: Array<XmlNode | string>; |
| 46 | spaceCount?: number; |
| 47 | hasText?: boolean; |
| 48 | hasTextChoice?: boolean; |
| 49 | } |
| 50 | |
| 51 | // ODF 1.3 Part 3 §6.1.2: paragraph character data turns TAB, CR and LF into spaces, collapses |
| 52 | // space runs and trims the ends; only <text:s>, <text:tab> and <text:line-break> keep literal ones. |
| 53 | function collapseOdfWhitespace(paragraph: XmlNode) { |
| 54 | let afterSpace = true; |
| 55 | let tail: { node: XmlNode; index: number; value: string } | undefined; |
| 56 | const visit = (node: XmlNode) => { |
| 57 | for (const [index, child] of node.children.entries()) { |
| 58 | if (typeof child !== "string") { |
| 59 | if (HIDDEN.has(child.name)) continue; |
| 60 | if (child.spaceCount === undefined && !["tab", "line-break", "p", "h"].includes(child.name)) |
| 61 | visit(child); |
| 62 | else { |
| 63 | afterSpace = false; |
| 64 | tail = undefined; |
| 65 | } |
| 66 | continue; |
| 67 | } |
| 68 | let value = child.replaceAll(/[ \t\r\n]+/g, " "); |
| 69 | if (afterSpace && value.startsWith(" ")) value = value.slice(1); |
| 70 | if (value) { |
| 71 | afterSpace = value.endsWith(" "); |
| 72 | tail = { node, index, value }; |
| 73 | } |
| 74 | node.children[index] = value; |
| 75 | } |
| 76 | }; |
| 77 | visit(paragraph); |
| 78 | if (tail && afterSpace) tail.node.children[tail.index] = tail.value.slice(0, -1); |
| 79 | } |
| 80 | |
| 81 | function xml(source: string, extension: string): XmlNode { |
| 82 | const odf = [".odt", ".ods", ".odp"].includes(extension); |
| 83 | const root: XmlNode = { name: "root", attrs: {}, children: [] }; |
| 84 | const stack = [root]; |
| 85 | let nodes = 0; |
| 86 | const parser = new SaxesParser(); |
| 87 | if (extension === ".epub") Object.assign(parser.ENTITIES, characterEntities); |
| 88 | parser.on("opentag", (tag) => { |
| 89 | if (++nodes > 200_000 || stack.length > 128) throw new Truncation("XML structure limit"); |
| 90 | const node: XmlNode = { |
| 91 | name: tag.name.split(":").at(-1)!, |
| 92 | attrs: Object.fromEntries( |
| 93 | Object.entries(tag.attributes).flatMap(([key, value]) => [ |
| 94 | [key, String(value)], |
| 95 | [key.split(":").at(-1)!, String(value)], |
| 96 | ]), |
| 97 | ), |
| 98 | children: [], |
| 99 | }; |
| 100 | if (odf && node.name === "s") node.spaceCount = repeatCount(node.attrs.c); |
| 101 | stack.at(-1)!.children.push(node); |
| 102 | stack.push(node); |
| 103 | }); |
| 104 | parser.on("closetag", () => { |
| 105 | const node = stack.pop()!; |
| 106 | const parent = stack.at(-1)!; |
| 107 | if (odf && (node.name === "p" || node.name === "h")) collapseOdfWhitespace(node); |
| 108 | if ( |
| 109 | (extension === ".docx" && node.name === "del") || |
| 110 | (odf && node.name === "annotation" && parent.name !== "semantics") |
| 111 | ) { |
| 112 | parent.children.pop(); |
| 113 | return; |
| 114 | } |
| 115 | if (node.name === "Fallback" && !retainCompatibilityFallback(parent, node)) return; |
| 116 | if (HIDDEN.has(node.name) || (!node.hasText && node.name !== "noBreakHyphen")) return; |
| 117 | parent.hasText = true; |
| 118 | if (node.name === "Choice") parent.hasTextChoice = true; |
| 119 | }); |
| 120 | const text = (value: string) => { |
| 121 | const node = stack.at(-1)!; |
| 122 | node.children.push(value); |
| 123 | if (value.trim() !== "") node.hasText = true; |
| 124 | }; |
| 125 | parser.on("text", text); |
| 126 | parser.on("cdata", text); |
| 127 | parser.write(source).close(); |
| 128 | return root; |
| 129 | } |
| 130 | |
| 131 | const children = (node: XmlNode | undefined) => |
| 132 | node?.children.filter((child): child is XmlNode => typeof child !== "string") ?? []; |
| 133 | |
| 134 | function retainCompatibilityFallback(parent: XmlNode, fallback: XmlNode): boolean { |
| 135 | // Markup-compatibility fallbacks repeat their mc:Choice content, e.g. Word text boxes. |
| 136 | // Keep one when no Choice carries text, such as a chart with a VML caption. |
| 137 | if (parent.hasTextChoice) { |
| 138 | parent.children.pop(); |
| 139 | return false; |
| 140 | } |
| 141 | if (fallback.hasText || hasInlineTab(fallback, ["tab", "ptab"])) { |
| 142 | parent.children = parent.children.filter( |
| 143 | (child) => |
| 144 | typeof child === "string" || child.name !== "Choice" || !hasInlineTab(child, ["ptab"]), |
| 145 | ); |
| 146 | } |
| 147 | return true; |
| 148 | } |
| 149 | |
| 150 | function hasInlineTab(node: XmlNode, names: readonly string[]): boolean { |
| 151 | return ( |
| 152 | !HIDDEN.has(node.name) && |
| 153 | (names.includes(node.name) || children(node).some((child) => hasInlineTab(child, names))) |
| 154 | ); |
| 155 | } |
| 156 | function* all(node: XmlNode, name: string | readonly string[]): Generator<XmlNode, void> { |
| 157 | for (const child of children(node)) { |
| 158 | if (typeof name === "string" ? child.name === name : name.includes(child.name)) yield child; |
| 159 | else yield* all(child, name); |
| 160 | } |
| 161 | } |
| 162 | function* text(node: XmlNode): Generator<string, void> { |
| 163 | if (HIDDEN.has(node.name)) return; |
| 164 | if (["tab", "ptab", "cell"].includes(node.name)) yield "\t"; |
| 165 | else if (["br", "line-break", "cr"].includes(node.name)) yield "\n"; |
| 166 | else if (node.name === "noBreakHyphen") yield "\u2011"; |
| 167 | else if (node.spaceCount !== undefined) { |
| 168 | yield " ".repeat(Math.min(MAX_OUTPUT, node.spaceCount)); |
| 169 | if (node.spaceCount > MAX_OUTPUT) throw new Truncation("Space repeat limit"); |
| 170 | } else |
| 171 | for (const child of node.children) { |
| 172 | if (typeof child === "string") yield child; |
| 173 | else yield* text(child); |
| 174 | } |
| 175 | } |
| 176 | // SpreadsheetML `t` and `v` values are ST_Xstring: `_xHHHH_` is one UTF-16 code unit, and an |
| 177 | // escaped lead underscore (`_x005F_xHHHH_`) keeps the literal `_xHHHH_`. Decoded in one pass. |
| 178 | function* xstringText(node: XmlNode): Generator<string, void> { |
| 179 | if (HIDDEN.has(node.name)) return; |
| 180 | if (node.name !== "t" && node.name !== "v") { |
| 181 | for (const child of children(node)) yield* xstringText(child); |
| 182 | return; |
| 183 | } |
| 184 | yield node.children |
| 185 | .filter((child) => typeof child === "string") |
| 186 | .join("") |
| 187 | .replaceAll( |
| 188 | /_x005[Ff]_(x[\dA-Fa-f]{4}_)|_x([\dA-Fa-f]{4})_/g, |
| 189 | (_, literal?: string, hex?: string) => |
| 190 | literal ? `_${literal}` : String.fromCharCode(Number.parseInt(hex!, 16)), |
| 191 | ); |
| 192 | } |
| 193 | function* paragraphs(node: XmlNode): Generator<TextPart, void> { |
| 194 | if (HIDDEN.has(node.name)) return; |
| 195 | for (const child of node.children) { |
| 196 | if (typeof child === "string") yield child; |
| 197 | else if (["br", "line-break", "cr"].includes(child.name)) yield "\n"; |
| 198 | else if (["tbl", "table"].includes(child.name)) yield* table(child); |
| 199 | else if (["p", "h"].includes(child.name) || /^h[1-6]$/.test(child.name)) { |
| 200 | const style = all(child, "pStyle").next().value?.attrs.val ?? ""; |
| 201 | const level = |
| 202 | child.name === "h" |
| 203 | ? Number(child.attrs["outline-level"]) || 1 |
| 204 | : /^h[1-6]$/.test(child.name) |
| 205 | ? Number(child.name[1]) |
| 206 | : Number(style.match(/^Heading([1-6])$/i)?.[1]) || 0; |
| 207 | if (level) yield `${"#".repeat(Math.min(6, level))} `; |
| 208 | yield* text(child); |
| 209 | yield "\n"; |
| 210 | } else { |
| 211 | yield* paragraphs(child); |
| 212 | if (["li", "div", "section"].includes(child.name)) yield "\n"; |
| 213 | } |
| 214 | } |
| 215 | } |
| 216 | function* inCell(parts: Iterable<TextPart>): Generator<TextPart, void> { |
| 217 | yield CELL_START; |
| 218 | yield* parts; |
| 219 | yield CELL_END; |
| 220 | } |
| 221 | function repeatCount(value: string | undefined): number { |
| 222 | if (value === undefined) return 1; |
| 223 | if (!/^[0-9]+$/.test(value) || Number(value) < 1) throw new Error("Invalid repeat count"); |
| 224 | return Number(value); |
| 225 | } |
| 226 | function odfValue(cell: XmlNode): string { |
| 227 | const type = cell.attrs["office:value-type"] ?? cell.attrs["value-type"]; |
| 228 | const attr = (name: string) => cell.attrs[`office:${name}`] ?? cell.attrs[name] ?? ""; |
| 229 | switch (type) { |
| 230 | case "void": |
| 231 | return ""; |
| 232 | case "string": |
| 233 | return attr("string-value"); |
| 234 | case "boolean": |
| 235 | return attr("boolean-value"); |
| 236 | case "date": |
| 237 | return attr("date-value"); |
| 238 | case "time": |
| 239 | return attr("time-value"); |
| 240 | case "currency": |
| 241 | return attr("value").trim() ? `${attr("value")} ${attr("currency")}`.trim() : ""; |
| 242 | case "percentage": |
| 243 | return attr("value").trim() ? `${attr("value")} [ODF value-type=percentage]` : ""; |
| 244 | default: |
| 245 | return attr("value"); |
| 246 | } |
| 247 | } |
| 248 | function hasCellText(cell: XmlNode): boolean { |
| 249 | if (odfValue(cell).trim() || cell.name === "noBreakHyphen") return true; |
| 250 | // Space markers can have enormous repeat counts; emptiness never requires expanding them. |
| 251 | if (HIDDEN.has(cell.name) || cell.spaceCount !== undefined) return false; |
| 252 | return cell.children.some((child) => |
| 253 | typeof child === "string" ? !!child.trim() : hasCellText(child), |
| 254 | ); |
| 255 | } |
| 256 | function* table(node: XmlNode): Generator<TextPart, void> { |
| 257 | let rowCount = 0; |
| 258 | for (const row of all(node, ["tr", "table-row"])) { |
| 259 | const cells = children(row).filter((cell) => |
| 260 | ["tc", "table-cell", "covered-table-cell", "td", "th"].includes(cell.name), |
| 261 | ); |
| 262 | // ODS writers pad sheets with repeated empty rows and trailing cells. |
| 263 | while (cells.length && !hasCellText(cells.at(-1)!)) cells.pop(); |
| 264 | if (!cells.length) continue; |
| 265 | const repeat = repeatCount(row.attrs["number-rows-repeated"]); |
| 266 | for (let index = 0; index < repeat; index++) { |
| 267 | if (++rowCount > 10_000) throw new Truncation("Row limit"); |
| 268 | let column = 0; |
| 269 | for (const cell of cells) { |
| 270 | const count = repeatCount(cell.attrs["number-columns-repeated"]); |
| 271 | for (let i = 0; i < count; i++) { |
| 272 | if (column >= 512) throw new Truncation("Column limit"); |
| 273 | if (column++) yield "\t"; |
| 274 | yield CELL_START; |
| 275 | yield* paragraphs(cell); |
| 276 | yield { fallback: odfValue(cell) }; |
| 277 | yield CELL_END; |
| 278 | } |
| 279 | } |
| 280 | yield "\n"; |
| 281 | } |
| 282 | } |
| 283 | } |
| 284 | |
| 285 | async function archive(path: string): Promise<Map<string, string>> { |
| 286 | const zip = await new Promise<Yauzl.ZipFile>((resolve, reject) => |
| 287 | Yauzl.open( |
| 288 | path, |
| 289 | { lazyEntries: true, validateEntrySizes: true, strictFileNames: true }, |
| 290 | (error, result) => (error || !result ? reject(error) : resolve(result)), |
| 291 | ), |
| 292 | ); |
| 293 | const files = new Map<string, string>(); |
| 294 | let expanded = 0; |
| 295 | let count = 0; |
| 296 | try { |
| 297 | await new Promise<void>((resolve, reject) => { |
| 298 | zip.on("error", reject); |
| 299 | zip.on("end", resolve); |
| 300 | zip.on("entry", (entry: Yauzl.Entry) => { |
| 301 | if (++count > 2048 || zip.entryCount > 2048) { |
| 302 | reject(new Truncation("Archive limit")); |
| 303 | return; |
| 304 | } |
| 305 | if (entry.generalPurposeBitFlag & 1) { |
| 306 | reject(new Error("Encrypted archive")); |
| 307 | return; |
| 308 | } |
| 309 | if (!/\.(xml|rels|xhtml|html|htm|opf|svg)$/i.test(entry.fileName)) { |
| 310 | zip.readEntry(); |
| 311 | return; |
| 312 | } |
| 313 | if ( |
| 314 | entry.uncompressedSize > MAX_EXPANDED || |
| 315 | entry.uncompressedSize > Math.max(1024, entry.compressedSize * 200) |
| 316 | ) { |
| 317 | reject(new Truncation("Archive limit")); |
| 318 | return; |
| 319 | } |
| 320 | expanded += entry.uncompressedSize; |
| 321 | if (expanded > MAX_EXPANDED) { |
| 322 | reject(new Truncation("Expanded byte limit")); |
| 323 | return; |
| 324 | } |
| 325 | zip.openReadStream(entry, (error, stream) => { |
| 326 | if (error || !stream) { |
| 327 | reject(error); |
| 328 | return; |
| 329 | } |
| 330 | const buffers: Buffer[] = []; |
| 331 | let size = 0; |
| 332 | stream.on("error", reject); |
| 333 | stream.on("data", (chunk: Buffer) => { |
| 334 | size += chunk.length; |
| 335 | if (size > entry.uncompressedSize || size > MAX_EXPANDED) { |
| 336 | stream.destroy(new Truncation("Entry byte limit")); |
| 337 | return; |
| 338 | } |
| 339 | buffers.push(chunk); |
| 340 | }); |
| 341 | stream.on("end", () => { |
| 342 | const data = Buffer.concat(buffers); |
| 343 | // XML's BOM and initial '<' bytes distinguish both UTF-16 byte orders. |
| 344 | const encoding = |
| 345 | (data[0] === 0xff && data[1] === 0xfe) || (data[0] === 0x3c && data[1] === 0) |
| 346 | ? "utf-16le" |
| 347 | : (data[0] === 0xfe && data[1] === 0xff) || (data[0] === 0 && data[1] === 0x3c) |
| 348 | ? "utf-16be" |
| 349 | : "utf-8"; |
| 350 | files.set(entry.fileName, new TextDecoder(encoding).decode(data)); |
| 351 | zip.readEntry(); |
| 352 | }); |
| 353 | }); |
| 354 | }); |
| 355 | zip.readEntry(); |
| 356 | }); |
| 357 | } finally { |
| 358 | zip.close(); |
| 359 | } |
| 360 | return files; |
| 361 | } |
| 362 | |
| 363 | const NOTEBOOK_CELL_TYPES = ["code", "markdown", "raw"]; |
| 364 | |
| 365 | function* notebook(source: string): Generator<string, void> { |
| 366 | const book: unknown = JSON.parse(source); |
| 367 | if (!book || typeof book !== "object" || !("cells" in book) || !Array.isArray(book.cells)) |
| 368 | throw new Error("Invalid notebook"); |
| 369 | function* lines(value: unknown): Generator<string, void> { |
| 370 | if (typeof value === "string") yield value; |
| 371 | else if (Array.isArray(value)) |
| 372 | for (const line of value) { |
| 373 | if (typeof line === "string") yield line; |
| 374 | } |
| 375 | } |
| 376 | function outputText(value: unknown): string { |
| 377 | // Notebook fragments can split a terminal escape sequence. |
| 378 | return NodeUtil.stripVTControlCharacters([...lines(value)].join("")); |
| 379 | } |
| 380 | for (const [index, cell] of book.cells.entries()) { |
| 381 | if (index >= 10_000) throw new Truncation("Notebook cell limit"); |
| 382 | if (!cell || typeof cell !== "object") continue; |
| 383 | const cellType = "cell_type" in cell ? cell.cell_type : undefined; |
| 384 | const kind = NOTEBOOK_CELL_TYPES.find((type) => type === cellType) ?? "markdown"; |
| 385 | yield `## Cell ${index + 1} (${kind})\n`; |
| 386 | if ("source" in cell) yield* lines(cell.source); |
| 387 | if ("outputs" in cell && Array.isArray(cell.outputs)) { |
| 388 | yield "\nOutput:\n"; |
| 389 | for (const output of cell.outputs) { |
| 390 | if (!output || typeof output !== "object") continue; |
| 391 | if ("text" in output) yield outputText(output.text); |
| 392 | else if ( |
| 393 | "data" in output && |
| 394 | output.data && |
| 395 | typeof output.data === "object" && |
| 396 | "text/plain" in output.data |
| 397 | ) |
| 398 | yield outputText(output.data["text/plain"]); |
| 399 | else if ("output_type" in output && output.output_type === "error") { |
| 400 | if ("ename" in output) yield* lines(output.ename); |
| 401 | if ("evalue" in output) { |
| 402 | yield ": "; |
| 403 | yield* lines(output.evalue); |
| 404 | } |
| 405 | yield "\n"; |
| 406 | if ("traceback" in output) |
| 407 | for (const line of lines(output.traceback)) { |
| 408 | yield NodeUtil.stripVTControlCharacters(line); |
| 409 | yield "\n"; |
| 410 | } |
| 411 | } |
| 412 | yield "\n"; |
| 413 | } |
| 414 | } |
| 415 | yield "\n\n"; |
| 416 | } |
| 417 | } |
| 418 | |
| 419 | function* rtf(source: string): Generator<string, void> { |
| 420 | if (!source.startsWith("{\\rtf")) throw new Error("Invalid RTF document"); |
| 421 | let state = { skip: false, deleted: false, uc: 1, decoder: new TextDecoder("windows-1252") }; |
| 422 | const stack: Array<typeof state> = []; |
| 423 | const hidden = () => state.skip || state.deleted; |
| 424 | const symbols = new Map([ |
| 425 | ["endash", "–"], |
| 426 | ["emdash", characterEntities.mdash], |
| 427 | ["enspace", "\u2002"], |
| 428 | ["emspace", "\u2003"], |
| 429 | ["qmspace", "\u2005"], |
| 430 | ["lquote", "‘"], |
| 431 | ["rquote", "’"], |
| 432 | ["ldblquote", "“"], |
| 433 | ["rdblquote", "”"], |
| 434 | ["bullet", "•"], |
| 435 | ["zwj", "\u200d"], |
| 436 | ["zwnj", "\u200c"], |
| 437 | ["ltrmark", "\u200e"], |
| 438 | ["rtlmark", "\u200f"], |
| 439 | ["~", "\u00a0"], |
| 440 | ["_", "\u2011"], |
| 441 | ["-", ""], |
| 442 | ]); |
| 443 | let fallback = 0; |
| 444 | for (let i = 0; i < source.length; i++) { |
| 445 | const char = source[i]!; |
| 446 | if (char === "{" || char === "}") fallback = 0; |
| 447 | if (char === "{") { |
| 448 | stack.push({ ...state }); |
| 449 | if (stack.length > 128) throw new Truncation("RTF depth limit"); |
| 450 | } else if (char === "}") state = stack.pop() ?? state; |
| 451 | else if (char === "\\") { |
| 452 | const rest = source.slice(i + 1, i + 80); |
| 453 | const hex = /^'([0-9a-f]{2})/i.exec(rest); |
| 454 | const word = /^([a-z]+)(-?\d+)? ?/i.exec(rest); |
| 455 | if (hex) { |
| 456 | if (fallback) fallback--; |
| 457 | else if (!hidden()) yield state.decoder.decode(Uint8Array.of(Number.parseInt(hex[1]!, 16))); |
| 458 | i += 3; |
| 459 | } else if (word) { |
| 460 | i += word[0].length; |
| 461 | const control = word[1]!; |
| 462 | const value = Number(word[2]); |
| 463 | if (control === "bin") { |
| 464 | if (!Number.isSafeInteger(value) || value < 0) |
| 465 | throw new Error("Invalid RTF binary length"); |
| 466 | i += value; |
| 467 | } |
| 468 | if (fallback) { |
| 469 | fallback--; |
| 470 | continue; |
| 471 | } |
| 472 | if ( |
| 473 | [ |
| 474 | "fonttbl", |
| 475 | "colortbl", |
| 476 | "stylesheet", |
| 477 | "info", |
| 478 | "pict", |
| 479 | "object", |
| 480 | "fldinst", |
| 481 | "header", |
| 482 | "headerf", |
| 483 | "headerl", |
| 484 | "headerr", |
| 485 | "footer", |
| 486 | "footerf", |
| 487 | "footerl", |
| 488 | "footerr", |
| 489 | ].includes(control) |
| 490 | ) |
| 491 | state.skip = true; |
| 492 | if (control === "deleted") state.deleted = value !== 0; |
| 493 | else if (control === "plain") state.deleted = false; |
| 494 | if (control === "mac" || (control === "ansicpg" && value === 10000)) |
| 495 | state.decoder = new TextDecoder("macintosh"); |
| 496 | else if (control === "ansicpg") { |
| 497 | // Per-byte decoding only supports single-byte Windows code pages. |
| 498 | if (value !== 874 && (value < 1250 || value > 1258)) |
| 499 | throw new Error("Unsupported RTF code page"); |
| 500 | state.decoder = new TextDecoder(`windows-${value}`, { fatal: true }); |
| 501 | } else if (control === "uc") { |
| 502 | if (!Number.isSafeInteger(value) || value < 0) |
| 503 | throw new Error("Invalid RTF fallback count"); |
| 504 | state.uc = value; |
| 505 | } else if (control === "u") { |
| 506 | if (!hidden()) yield String.fromCharCode(value < 0 ? value + 65536 : value); |
| 507 | fallback = state.uc; |
| 508 | } else if (!hidden() && ["par", "line", "sect", "page", "column"].includes(control)) |
| 509 | yield "\n"; |
| 510 | else if (!hidden() && control === "tab") yield "\t"; |
| 511 | // Table structure survives tracked deletions; only the deleted cell text is dropped. |
| 512 | else if (!state.skip && control === "row") yield "\n"; |
| 513 | else if (!state.skip && control === "cell") yield "\t"; |
| 514 | else if (!hidden() && symbols.has(control)) yield symbols.get(control)!; |
| 515 | } else { |
| 516 | i++; |
| 517 | if (fallback) fallback--; |
| 518 | else if (rest[0] === "*") state.skip = true; |
| 519 | // A backslash before a line break is \par; TextEdit ends paragraphs this way. |
| 520 | else if (!hidden() && (rest[0] === "\n" || rest[0] === "\r")) yield "\n"; |
| 521 | else if (!hidden() && ["\\", "{", "}"].includes(rest[0]!)) yield rest[0]!; |
| 522 | else if (!hidden() && symbols.has(rest[0]!)) yield symbols.get(rest[0]!)!; |
| 523 | } |
| 524 | } else if (char !== "\n" && char !== "\r") { |
| 525 | if (fallback) fallback--; |
| 526 | else if (!hidden()) |
| 527 | yield char.charCodeAt(0) < 128 |
| 528 | ? char |
| 529 | : state.decoder.decode(Uint8Array.of(char.charCodeAt(0))); |
| 530 | } |
| 531 | } |
| 532 | } |
| 533 | |
| 534 | async function* document(path: string, extension: string): AsyncGenerator<TextPart, void> { |
| 535 | if ((await NodeFSP.stat(path)).size > MAX_INPUT) throw new Truncation("Input byte limit"); |
| 536 | if (extension === ".ipynb") yield* notebook(await NodeFSP.readFile(path, "utf8")); |
| 537 | else if (extension === ".rtf") yield* rtf((await NodeFSP.readFile(path)).toString("latin1")); |
| 538 | else { |
| 539 | const files = await archive(path); |
| 540 | const notes: string[] = []; |
| 541 | const skip = (name: string) => |
| 542 | notes.push(`\n[Skipped document part: ${name} (missing or invalid XML).]\n`); |
| 543 | const read = (name: string): XmlNode => { |
| 544 | try { |
| 545 | const data = files.get(name); |
| 546 | if (!data) throw new Error("Missing document part"); |
| 547 | return xml(data, extension); |
| 548 | } catch (error) { |
| 549 | if (error instanceof Truncation) throw error; |
| 550 | skip(name); |
| 551 | return { name: "root", attrs: {}, children: [] }; |
| 552 | } |
| 553 | }; |
| 554 | const relations = (name: string) => |
| 555 | new Map( |
| 556 | [...all(read(name), "Relationship")] |
| 557 | .filter((rel) => rel.attrs.TargetMode !== "External") |
| 558 | .map((rel) => [rel.attrs.Id, rel.attrs.Target]), |
| 559 | ); |
| 560 | if (extension === ".docx") yield* paragraphs(read("word/document.xml")); |
| 561 | else if ([".odt", ".ods", ".odp"].includes(extension)) { |
| 562 | const content = read("content.xml"); |
| 563 | if (extension === ".odt") yield* paragraphs(content); |
| 564 | else { |
| 565 | let index = 0; |
| 566 | for (const part of all(content, extension === ".ods" ? "table" : "page")) { |
| 567 | yield extension === ".ods" |
| 568 | ? `\n\n## Sheet: ${part.attrs.name ?? "Untitled"}\n` |
| 569 | : `\n\n## Slide ${++index}\n`; |
| 570 | if (extension === ".ods") yield* table(part); |
| 571 | else yield* paragraphs(part); |
| 572 | } |
| 573 | } |
| 574 | } else if (extension === ".xlsx") { |
| 575 | const workbook = read("xl/workbook.xml"); |
| 576 | const dateSystem = ["1", "true"].includes( |
| 577 | all(workbook, "workbookPr").next().value?.attrs.date1904 ?? "", |
| 578 | ) |
| 579 | ? "1904" |
| 580 | : "1900"; |
| 581 | const styles = files.has("xl/styles.xml") ? read("xl/styles.xml") : undefined; |
| 582 | const customFormats = new Map( |
| 583 | styles |
| 584 | ? [...all(styles, "numFmt")].map((node) => [node.attrs.numFmtId, node.attrs.formatCode]) |
| 585 | : [], |
| 586 | ); |
| 587 | const cellFormats = styles ? all(styles, "cellXfs").next().value : undefined; |
| 588 | const formats = cellFormats ? children(cellFormats) : []; |
| 589 | const strings = files.has("xl/sharedStrings.xml") |
| 590 | ? [...all(read("xl/sharedStrings.xml"), "si")] |
| 591 | : []; |
| 592 | const populatedStrings = strings.map(hasCellText); |
| 593 | const rels = relations("xl/_rels/workbook.xml.rels"); |
| 594 | for (const sheet of all(workbook, "sheet")) { |
| 595 | yield `\n## Sheet: ${sheet.attrs.name}\n`; |
| 596 | const target = rels.get(sheet.attrs["r:id"] ?? sheet.attrs.id); |
| 597 | if (!target) { |
| 598 | skip(`sheet relationship ${sheet.attrs.name}`); |
| 599 | continue; |
| 600 | } |
| 601 | const name = target.startsWith("/") |
| 602 | ? target.slice(1) |
| 603 | : NodePath.posix.normalize(`xl/${target}`); |
| 604 | const populated = (cell: XmlNode) => { |
| 605 | const value = all(cell, "v").next().value; |
| 606 | if (cell.attrs.t === "s") { |
| 607 | const raw = value?.children.find((child) => typeof child === "string")?.trim(); |
| 608 | return !!raw && !!populatedStrings[Number(raw)]; |
| 609 | } |
| 610 | if (cell.attrs.t === "inlineStr") return hasCellText(cell); |
| 611 | if (value && hasCellText(value)) return true; |
| 612 | for (const formula of all(cell, "f")) if (hasCellText(formula)) return true; |
| 613 | return false; |
| 614 | }; |
| 615 | let rowCount = 0; |
| 616 | for (const row of all(read(name), "row")) { |
| 617 | const cells: Array<XmlNode | undefined> = []; |
| 618 | let nextColumn = 0; |
| 619 | let columnsTruncated = false; |
| 620 | for (const cell of children(row).filter((node) => node.name === "c")) { |
| 621 | const letters = cell.attrs.r?.match(/^[A-Z]+/)?.[0]; |
| 622 | const column = letters |
| 623 | ? [...letters].reduce((n, char) => n * 26 + char.charCodeAt(0) - 64, 0) - 1 |
| 624 | : nextColumn; |
| 625 | nextColumn = column + 1; |
| 626 | if (!populated(cell)) continue; |
| 627 | if (column >= 512) { |
| 628 | columnsTruncated = true; |
| 629 | continue; |
| 630 | } |
| 631 | cells[column] = cell; |
| 632 | } |
| 633 | if (!cells.length && !columnsTruncated) continue; |
| 634 | if (++rowCount > 10_000) throw new Truncation("Row limit"); |
| 635 | for (const [column, cell] of cells.entries()) { |
| 636 | if (column) yield "\t"; |
| 637 | if (!cell) continue; |
| 638 | const value = all(cell, "v").next().value; |
| 639 | const raw = value?.children.find((child) => typeof child === "string") ?? ""; |
| 640 | if (cell.attrs.t === "s") { |
| 641 | const shared = strings[Number(raw)]; |
| 642 | if (shared) yield* inCell(xstringText(shared)); |
| 643 | } else if (cell.attrs.t === "inlineStr") yield* inCell(xstringText(cell)); |
| 644 | else if (cell.attrs.t === "b" && (raw.trim() === "0" || raw.trim() === "1")) |
| 645 | yield* inCell([raw.trim() === "1" ? "true" : "false"]); |
| 646 | else { |
| 647 | if (value && hasCellText(value)) yield* inCell(xstringText(value)); |
| 648 | else for (const formula of all(cell, "f")) yield* inCell(text(formula)); |
| 649 | const formatId = formats[Number(cell.attrs.s ?? 0)]?.attrs.numFmtId; |
| 650 | const format = formatId ? customFormats.get(formatId) : undefined; |
| 651 | if ( |
| 652 | (cell.attrs.t === undefined || cell.attrs.t === "n") && |
| 653 | raw.trim() && |
| 654 | formatId && |
| 655 | formatId !== "0" |
| 656 | ) |
| 657 | yield ` [Excel numFmtId=${formatId}${format ? `; format=${format}` : ""}; date system ${dateSystem}]`; |
| 658 | } |
| 659 | } |
| 660 | yield "\n"; |
| 661 | if (columnsTruncated) throw new Truncation("Column limit"); |
| 662 | } |
| 663 | } |
| 664 | } else if (extension === ".pptx") { |
| 665 | const rels = relations("ppt/_rels/presentation.xml.rels"); |
| 666 | const [presentation] = children(read("ppt/presentation.xml")); |
| 667 | const slideList = children(presentation).find((node) => node.name === "sldIdLst"); |
| 668 | let index = 0; |
| 669 | for (const slide of children(slideList).filter((node) => node.name === "sldId")) { |
| 670 | yield `\n\n## Slide ${++index}\n`; |
| 671 | const target = rels.get(slide.attrs["r:id"] ?? slide.attrs.id); |
| 672 | if (!target) { |
| 673 | skip(`slide relationship ${index}`); |
| 674 | continue; |
| 675 | } |
| 676 | const name = target.startsWith("/") |
| 677 | ? target.slice(1) |
| 678 | : NodePath.posix.normalize(`ppt/${target}`); |
| 679 | yield* paragraphs(read(name)); |
| 680 | const slideRels = NodePath.posix.join( |
| 681 | NodePath.posix.dirname(name), |
| 682 | "_rels", |
| 683 | `${NodePath.posix.basename(name)}.rels`, |
| 684 | ); |
| 685 | if (!files.has(slideRels)) continue; |
| 686 | for (const rel of all(read(slideRels), "Relationship")) { |
| 687 | const target = rel.attrs.Target; |
| 688 | if ( |
| 689 | !rel.attrs.Type?.endsWith("/notesSlide") || |
| 690 | rel.attrs.TargetMode === "External" || |
| 691 | !target |
| 692 | ) |
| 693 | continue; |
| 694 | yield "Notes:\n"; |
| 695 | yield* paragraphs( |
| 696 | read( |
| 697 | target.startsWith("/") |
| 698 | ? target.slice(1) |
| 699 | : NodePath.posix.join(NodePath.posix.dirname(name), target), |
| 700 | ), |
| 701 | ); |
| 702 | } |
| 703 | } |
| 704 | } else if (extension === ".epub") { |
| 705 | const rootfile = all(read("META-INF/container.xml"), "rootfile").next().value?.attrs[ |
| 706 | "full-path" |
| 707 | ]; |
| 708 | if (!rootfile) { |
| 709 | if (!notes.length) skip("EPUB package"); |
| 710 | yield* notes; |
| 711 | return; |
| 712 | } |
| 713 | const packageXml = read(rootfile); |
| 714 | const items = new Map( |
| 715 | [...all(packageXml, "item")].map((item) => [item.attrs.id, item.attrs.href]), |
| 716 | ); |
| 717 | let index = 0; |
| 718 | for (const item of all(packageXml, "itemref")) { |
| 719 | yield `\n\n## Chapter ${++index}\n`; |
| 720 | const href = items.get(item.attrs.idref); |
| 721 | if (!href) { |
| 722 | skip(`chapter ${index}`); |
| 723 | continue; |
| 724 | } |
| 725 | let decoded: string; |
| 726 | try { |
| 727 | decoded = decodeURIComponent(href.split("#")[0]!); |
| 728 | } catch { |
| 729 | skip(`chapter ${index}`); |
| 730 | continue; |
| 731 | } |
| 732 | const name = NodePath.posix.normalize( |
| 733 | NodePath.posix.join(NodePath.posix.dirname(rootfile), decoded), |
| 734 | ); |
| 735 | yield* paragraphs(read(name)); |
| 736 | } |
| 737 | } else throw new Error("Unsupported document"); |
| 738 | yield* notes; |
| 739 | } |
| 740 | } |
| 741 | |
| 742 | export async function extractDocumentText(path: string, extension: string): Promise<string> { |
| 743 | let result = ""; |
| 744 | let remaining = MAX_OUTPUT; |
| 745 | let cellDepth = 0; |
| 746 | let started = false; |
| 747 | let space = false; |
| 748 | let truncated = false; |
| 749 | try { |
| 750 | // Charge raw text before normalization so repeated whitespace also exhausts the budget. |
| 751 | for await (const part of document(path, extension)) { |
| 752 | if (part === CELL_START) { |
| 753 | if (cellDepth++ === 0) { |
| 754 | started = false; |
| 755 | space = false; |
| 756 | } else if (started) space = true; |
| 757 | continue; |
| 758 | } |
| 759 | if (part === CELL_END) { |
| 760 | cellDepth--; |
| 761 | if (cellDepth > 0 && started) space = true; |
| 762 | continue; |
| 763 | } |
| 764 | if (typeof part !== "string" && started) continue; |
| 765 | const raw = typeof part === "string" ? part : part.fallback; |
| 766 | let value = raw.slice(0, remaining).replaceAll("\0", ""); |
| 767 | remaining -= Math.min(raw.length, remaining); |
| 768 | if (cellDepth) { |
| 769 | value = value.replaceAll(/\s+/g, " "); |
| 770 | const trimmed = value.trim(); |
| 771 | if (trimmed) { |
| 772 | result += (started && (space || value.startsWith(" ")) ? " " : "") + trimmed; |
| 773 | started = true; |
| 774 | } |
| 775 | space = value.endsWith(" ") || (!value && space); |
| 776 | } else result += value; |
| 777 | if (remaining === 0) throw new Truncation("Output budget"); |
| 778 | } |
| 779 | } catch (error) { |
| 780 | if (!(error instanceof Truncation)) throw error; |
| 781 | truncated = true; |
| 782 | } |
| 783 | result = result.replaceAll(/\n(?:[ \t]*\n){2,}/g, "\n\n").trim(); |
| 784 | if (!result && !truncated) throw new Error("No text extracted"); |
| 785 | return truncated ? result.replace(/[\uD800-\uDBFF]$/, "").trimEnd() + TRUNCATED : result; |
| 786 | } |