@devries-cameron20 / t3code-1 / commits / 140de68f25
Fork 0

The best way to control Claude Code, Codex and more from any computer or phone

fix(mobile): images picked with choose files reach the agent as images (#21352)

maria committed Oct 25, 2026 at 19:04 UTC 140de68f25c5aeb3728596a7005635a6dfdf2989
4 files changed +276 -15
apps/mobile/src/lib/attachmentUpload.test.ts
+36 -1
@@ -1,4 +1,4 @@
1 -import { EnvironmentId } from "@t3tools/contracts";
1 +import { EnvironmentId, PROVIDER_SEND_TURN_MAX_IMAGE_BYTES } from "@t3tools/contracts";
2 import * as Option from "effect/Option";
3 import { beforeEach, describe, expect, it, vi } from "vite-plus/test";
4
@@ -410,6 +410,41 @@ describe("prepareTurnAttachments", () => {
410 });
411 });
412
413 + it("uploads an oversized document-picked image as a file", async () => {
414 + const largeImage = {
415 + ...documentPickedImage,
416 + mimeType: "image/png",
417 + sizeBytes: PROVIDER_SEND_TURN_MAX_IMAGE_BYTES + 1,
418 + };
419 + const prepared = await prepareTurnAttachments({ environmentId, attachments: [largeImage] });
420 +
421 + expect(mocks.runAtomCommand).toHaveBeenCalledWith(
422 + expect.anything(),
423 + mocks.createUploadUrl,
424 + expect.objectContaining({
425 + input: {
426 + type: "file",
427 + name: largeImage.name,
428 + mimeType: "image/png",
429 + sizeBytes: largeImage.sizeBytes,
430 + },
431 + }),
432 + expect.anything(),
433 + );
434 + expect(prepared.status).toBe("ready");
435 + if (prepared.status !== "ready") return;
436 + expect(prepared.attachments).toEqual([
437 + {
438 + type: "file",
439 + id: MINTED_ID,
440 + name: largeImage.name,
441 + mimeType: "image/png",
442 + sizeBytes: largeImage.sizeBytes,
443 + },
444 + ]);
445 + expect(mocks.readBase64).not.toHaveBeenCalled();
446 + });
447 +
448 it("uploads generic file bytes directly and keeps mixed attachment order", async () => {
449 const pastedFile = {
450 ...file,
apps/mobile/src/lib/composerFiles.test.ts
+165
@@ -90,6 +90,7 @@ vi.mock("./uuid", () => ({ uuidv4: () => "attachment-id" }));
90
91 import {
92 persistComposerAttachmentFile,
93 + isComposerImageAttachment,
94 pickComposerFiles,
95 pickComposerImages,
96 pickComposerMedia,
@@ -108,6 +109,7 @@ describe("composer file attachments", () => {
109 mocks.open.mockReset();
110 mocks.size.mockReset();
111 mocks.readBase64.mockReset();
112 + mocks.manipulate.mockReset();
113 mocks.size.mockImplementation((uri: string) => (uri.startsWith("content:") ? null : 42));
114 });
115
@@ -442,6 +444,169 @@ describe("composer file attachments", () => {
444 });
445 });
446
447 + describe("images chosen through the file picker", () => {
448 + it.each([
449 + { name: "photo.bin", mimeType: "image/jpeg", expected: "image/jpeg" },
450 + { name: "photo.png", mimeType: "image/png", expected: "image/png" },
451 + { name: "photo.gif", mimeType: "image/gif", expected: "image/gif" },
452 + { name: "photo.webp", mimeType: "image/webp", expected: "image/webp" },
453 + { name: "photo.JPG", mimeType: undefined, expected: "image/jpeg" },
454 + { name: "photo.PNG", mimeType: "application/octet-stream", expected: "image/png" },
455 + { name: "photo.gif", mimeType: "binary/octet-stream", expected: "image/gif" },
456 + { name: "photo.webp", mimeType: "application/unknown", expected: "image/webp" },
457 + { name: "photo.jpg", mimeType: "IMAGE/PNG; charset=binary", expected: "image/png" },
458 + ])("prepares $name ($mimeType) as $expected", async ({ name, mimeType, expected }) => {
459 + mocks.pickFile.mockResolvedValue({
460 + canceled: false,
461 + assets: [{ uri: `file:///picker/${name}`, name, mimeType, size: 3 }],
462 + });
463 + mocks.size.mockReturnValue(3);
464 + mocks.readBase64.mockResolvedValue("YWJj");
465 +
466 + const result = await pickComposerFiles({ existingCount: 0 });
467 +
468 + expect(result).toEqual({
469 + files: [
470 + {
471 + id: "attachment-id",
472 + type: "image",
473 + name,
474 + mimeType: expected,
475 + sizeBytes: 3,
476 + dataUrl: `data:${expected};base64,YWJj`,
477 + previewUri: `data:${expected};base64,YWJj`,
478 + },
479 + ],
480 + error: null,
481 + });
482 + expect(mocks.manipulate).not.toHaveBeenCalled();
483 + expect(mocks.delete).toHaveBeenCalledWith(
484 + `file:///documents/t3-composer-attachments/attachment-id-${name}`,
485 + );
486 + });
487 +
488 + it.each([
489 + { name: "diagram.svg", mimeType: "image/svg+xml" },
490 + { name: "photo.heic", mimeType: "image/heic" },
491 + { name: "photo.tiff", mimeType: "image/tiff" },
492 + { name: "photo.png", mimeType: "application/pdf" },
493 + { name: "unknown.bin", mimeType: "application/octet-stream" },
494 + ])("keeps $name ($mimeType) as a file", async ({ name, mimeType }) => {
495 + mocks.pickFile.mockResolvedValue({
496 + canceled: false,
497 + assets: [{ uri: `file:///picker/${name}`, name, mimeType, size: 42 }],
498 + });
499 +
500 + const result = await pickComposerFiles({ existingCount: 0 });
501 +
502 + expect(result.error).toBeNull();
503 + expect(result.files).toEqual([expect.objectContaining({ type: "file", name, mimeType })]);
504 + expect(mocks.readBase64).not.toHaveBeenCalled();
505 + expect(mocks.manipulate).not.toHaveBeenCalled();
506 + expect(mocks.delete).not.toHaveBeenCalled();
507 + });
508 +
509 + it.each([PROVIDER_SEND_TURN_MAX_IMAGE_BYTES, PROVIDER_SEND_TURN_MAX_IMAGE_BYTES + 1])(
510 + "classifies a measured %s byte PNG independently of picker metadata and file limit",
511 + async (size) => {
512 + mocks.pickFile.mockResolvedValue({
513 + canceled: false,
514 + assets: [
515 + { uri: "file:///picker/photo.png", name: "photo.png", mimeType: "image/png", size: 1 },
516 + ],
517 + });
518 + mocks.size.mockReturnValue(size);
519 + mocks.readBase64.mockResolvedValue("YWJj");
520 +
521 + const result = await pickComposerFiles({
522 + existingCount: 0,
523 + maxBytes: size > PROVIDER_SEND_TURN_MAX_IMAGE_BYTES ? size : 1024,
524 + });
525 +
526 + expect(result.error).toBeNull();
527 + expect(result.files[0]?.type).toBe(
528 + size > PROVIDER_SEND_TURN_MAX_IMAGE_BYTES ? "file" : "image",
529 + );
530 + expect(isComposerImageAttachment(result.files[0]!)).toBe(
531 + size <= PROVIDER_SEND_TURN_MAX_IMAGE_BYTES,
532 + );
533 + expect(mocks.readBase64).toHaveBeenCalledTimes(
534 + size > PROVIDER_SEND_TURN_MAX_IMAGE_BYTES ? 0 : 1,
535 + );
536 + },
537 + );
538 +
539 + it.each([false, true])("fits a large PNG with oversized output: %s", async (oversized) => {
540 + const header = Buffer.alloc(24);
541 + Buffer.from([0x89, 0x50, 0x4e, 0x47, 0x0d, 0x0a, 0x1a, 0x0a]).copy(header);
542 + header.write("IHDR", 12);
543 + header.writeUInt32BE(4000, 16);
544 + header.writeUInt32BE(3000, 20);
545 + const original = header.toString("base64");
546 + mocks.readBase64.mockResolvedValue(original);
547 + const resize = vi.fn();
548 + const renderAsync = vi.fn(async () => ({
549 + width: 4000,
550 + height: 3000,
551 + release: vi.fn(),
552 + saveAsync: vi.fn(async () => ({
553 + uri: "file:///cache/fitted.png",
554 + base64: oversized
555 + ? Buffer.alloc(PROVIDER_SEND_TURN_MAX_IMAGE_BYTES + 1).toString("base64")
556 + : "YWJj",
557 + })),
558 + }));
559 + mocks.manipulate.mockImplementation(() => {
560 + const context = {
561 + resize: (size: unknown) => {
562 + resize(size);
563 + return context;
564 + },
565 + renderAsync,
566 + };
567 + return context;
568 + });
569 + mocks.pickFile.mockResolvedValue({
570 + canceled: false,
571 + assets: [{ uri: "file:///picker/photo.png", name: "photo.png", mimeType: "image/png" }],
572 + });
573 +
574 + const result = await pickComposerFiles({ existingCount: 0 });
575 +
576 + expect(resize).toHaveBeenCalledWith({ width: 2000, height: 1500 });
577 + expect(result).toEqual({
578 + files: [
579 + expect.objectContaining({
580 + type: "image",
581 + mimeType: "image/png",
582 + sizeBytes: oversized ? header.length : 3,
583 + dataUrl: `data:image/png;base64,${oversized ? original : "YWJj"}`,
584 + previewUri: oversized
585 + ? `data:image/png;base64,${original}`
586 + : "file:///cache/fitted.png",
587 + }),
588 + ],
589 + error: null,
590 + });
591 + });
592 +
593 + it("releases the owned copy when reading an image fails", async () => {
594 + mocks.pickFile.mockResolvedValue({
595 + canceled: false,
596 + assets: [{ uri: "file:///picker/photo.png", name: "photo.png", mimeType: "image/png" }],
597 + });
598 + mocks.readBase64.mockRejectedValue(new Error("unreadable image"));
599 +
600 + expect(await pickComposerFiles({ existingCount: 0 })).toEqual({
601 + files: [],
602 + error: "unreadable image",
603 + });
604 + expect(mocks.delete).toHaveBeenCalledWith(
605 + "file:///documents/t3-composer-attachments/attachment-id-photo.png",
606 + );
607 + });
608 + });
609 +
610 it("copies picked files into app-owned storage without loading their contents", async () => {
611 mocks.pickFile.mockResolvedValue({
612 canceled: false,
apps/mobile/src/lib/composerImages.test.ts
+20 -1
@@ -1,5 +1,8 @@
1 import { beforeEach, describe, expect, it, vi } from "vite-plus/test";
2 -import { PROVIDER_SEND_TURN_MAX_ATTACHMENTS } from "@t3tools/contracts";
2 +import {
3 + PROVIDER_SEND_TURN_MAX_ATTACHMENTS,
4 + PROVIDER_SEND_TURN_MAX_IMAGE_BYTES,
5 +} from "@t3tools/contracts";
6
7 const files = new Map<string, { base64: string; deleted: boolean; text?: string }>();
8
@@ -263,4 +266,20 @@ describe("composerStripAttachments", () => {
266 "vid-1",
267 ]);
268 });
269 +
270 + it("keeps oversized document-picked images visible while sending them as files", async () => {
271 + const { composerStripAttachments, isComposerImageAttachment } =
272 + await import("./composerImages");
273 + const oversizedImage = {
274 + ...video,
275 + id: "large-image",
276 + name: "large.png",
277 + mimeType: "image/png",
278 + fileUri: "file:///large.png",
279 + sizeBytes: PROVIDER_SEND_TURN_MAX_IMAGE_BYTES + 1,
280 + };
281 +
282 + expect(composerStripAttachments([oversizedImage])).toEqual([oversizedImage]);
283 + expect(isComposerImageAttachment(oversizedImage)).toBe(false);
284 + });
285 });
apps/mobile/src/lib/composerImages.ts
+55 -13
@@ -82,15 +82,22 @@ export function composerStripAttachments(
82 attachments: ReadonlyArray<DraftComposerAttachment>,
83 ): ReadonlyArray<DraftComposerAttachment> {
84 return attachments.filter(
85 - (attachment) => isComposerImageAttachment(attachment) || videoMimeType(attachment) !== null,
85 + (attachment) =>
86 + attachment.type === "image" ||
87 + imageMimeType(attachment) !== null ||
88 + videoMimeType(attachment) !== null,
89 );
90 }
91
89 -/** Detect from the attachment; the document picker labels images as plain files. */
92 +/** Promote legacy document-picked images only when their originals fit the image contract. */
93 export function isComposerImageAttachment(
94 attachment: DraftComposerAttachment,
95 ): attachment is DraftComposerImageAttachment {
93 - return attachment.type === "image" || imageMimeType(attachment) !== null;
96 + return (
97 + attachment.type === "image" ||
98 + (attachment.sizeBytes <= PROVIDER_SEND_TURN_MAX_IMAGE_BYTES &&
99 + imageMimeType(attachment) !== null)
100 + );
101 }
102
103 export type FileBackedComposerAttachment = DraftComposerAttachment & { readonly fileUri: string };
@@ -251,7 +258,7 @@ export async function pickComposerFiles(input: {
258 readonly existingCount: number;
259 readonly maxBytes?: number;
260 }): Promise<{
254 - readonly files: ReadonlyArray<DraftComposerFileAttachment>;
261 + readonly files: ReadonlyArray<DraftComposerAttachment>;
262 readonly error: string | null;
263 }> {
264 const remainingSlots = PROVIDER_SEND_TURN_MAX_ATTACHMENTS - input.existingCount;
@@ -283,7 +290,7 @@ export async function pickComposerFiles(input: {
290 const maxBytes = clampFileAttachmentUploadBytes(
291 input.maxBytes ?? PROVIDER_SEND_TURN_MAX_FILE_BYTES,
292 );
286 - const attachments: DraftComposerFileAttachment[] = [];
293 + const attachments: DraftComposerAttachment[] = [];
294 let error: string | null = null;
295 let exceededAttachmentLimit = false;
296 for (const file of result.assets) {
@@ -294,15 +301,50 @@ export async function pickComposerFiles(input: {
301 // SAF can return blank display names, but the wire contract rejects them; fill before persisting.
302 const name = file.name.trim().length > 0 ? file.name : "file";
303 try {
297 - attachments.push(
298 - await createComposerFileAttachment({
299 - uri: file.uri,
304 + const mimeType = imageMimeType({ name, mimeType: file.mimeType ?? "" });
305 + const attachment = await createComposerFileAttachment({
306 + uri: file.uri,
307 + name,
308 + mimeType: file.mimeType || "application/octet-stream",
309 + sizeBytes: file.size ?? null,
310 + maxBytes:
311 + mimeType === null ? maxBytes : Math.max(maxBytes, PROVIDER_SEND_TURN_MAX_IMAGE_BYTES),
312 + });
313 + if (mimeType === null || attachment.sizeBytes > PROVIDER_SEND_TURN_MAX_IMAGE_BYTES) {
314 + attachments.push(attachment);
315 + continue;
316 + }
317 + try {
318 + const { File } = await import("expo-file-system");
319 + const original = await new File(attachment.fileUri).base64();
320 + const candidate = await fitComposerImage({
321 + uri: attachment.fileUri,
322 + mimeType,
323 + base64: original,
324 + });
325 + const fitted =
326 + candidate !== null &&
327 + estimateBase64ByteSize(candidate.base64) <= PROVIDER_SEND_TURN_MAX_IMAGE_BYTES
328 + ? candidate
329 + : null;
330 + const base64 = fitted?.base64 ?? original;
331 + const sizeBytes = estimateBase64ByteSize(base64);
332 + if (sizeBytes <= 0 || sizeBytes > PROVIDER_SEND_TURN_MAX_IMAGE_BYTES) {
333 + throw new Error(`'${name}' exceeds the 10 MB attachment limit.`);
334 + }
335 + const dataUrl = `data:${mimeType};base64,${base64}`;
336 + attachments.push({
337 + id: attachment.id,
338 + type: "image",
339 name,
301 - mimeType: file.mimeType || "application/octet-stream",
302 - sizeBytes: file.size ?? null,
303 - maxBytes,
304 - }),
305 - );
340 + mimeType,
341 + sizeBytes,
342 + dataUrl,
343 + previewUri: fitted?.uri ?? dataUrl,
344 + });
345 + } finally {
346 + await removePersistedComposerAttachmentFile(attachment.fileUri);
347 + }
348 } catch (cause) {
349 error = cause instanceof Error ? cause.message : `Could not read '${name}'.`;
350 }