The best way to control Claude Code, Codex and more from any computer or phone
Mirrored from https://github.com/Camo67/t3code.git
Synced Nov 21, 08:30 UTC
fix(server): keep truncated document text valid Unicode (#38171)
Co-authored-by: Simone <185146821+Lucenx9@users.noreply.github.com> Co-authored-by: Julius Marminge <julius0216@outlook.com>
Simone committed
Nov 17, 2026 at 16:30 UTC
cc8593e06d8d3a9209949b07e75ef71d9b73eb0e
2 files changed
+8
-4
apps/server/src/provider/documentTextWorker.ts
+1
-1
@@ -738,5 +738,5 @@ export async function extractDocumentText(path: string, extension: string): Prom
738
}
739
result = result.replaceAll(/\n(?:[ \t]*\n){2,}/g, "\n\n").trim();
740
if (!result && !truncated) throw new Error("No text extracted");
741
- return result + (truncated ? TRUNCATED : "");
741
+ return truncated ? result.replace(/[\uD800-\uDBFF]$/, "").trimEnd() + TRUNCATED : result;
742
}
apps/server/src/provider/userInputAttachments.test.ts
+7
-3
@@ -1297,10 +1297,14 @@ describe("question answer paths", () => {
1297
});
1298
expect(yield* (yield* DocumentText.DocumentText).ensure(unsupportedPath)).toBeUndefined();
1299
expect(NodeFS.existsSync(unsupportedPath + ".txt")).toBe(false);
1300
- NodeFS.writeFileSync(path, String.raw`{\rtf1 ` + "x".repeat(1_000_100) + "}");
1301
- expect(yield* Effect.promise(() => extractDocumentText(path, ".rtf"))).toContain(
1302
- "[Extracted text truncated",
1300
+ NodeFS.writeFileSync(
1301
+ path,
1302
+ String.raw`{\rtf1 ` + "x".repeat(999_999) + String.raw`\u-10179?\u-8576? After}`,
1303
);
1304
+ const unicodeTruncated = yield* Effect.promise(() => extractDocumentText(path, ".rtf"));
1305
+ expect(unicodeTruncated).toContain("[Extracted text truncated");
1306
+ expect(unicodeTruncated.isWellFormed()).toBe(true);
1307
+ expect(unicodeTruncated.startsWith("x".repeat(999_999))).toBe(true);
1308
NodeFS.writeFileSync(path, String.raw`{\rtf1 First` + " ".repeat(1_000_100) + "Last}");
1309
const whitespaceTruncated = yield* Effect.promise(() => extractDocumentText(path, ".rtf"));
1310
expect(whitespaceTruncated).toBe(