@devries-cameron20 / t3code Mirror · read-only
Fork 0

The best way to control Claude Code, Codex and more from any computer or phone

Mirrored from https://github.com/Camo67/t3code.git

Synced Nov 21, 04:45 UTC

main
ts 560 lines 18.3 KB
Raw
1 import {
2 downscalableImageMimeType,
3 fitImageAttachmentSize,
4 IMAGE_ATTACHMENT_MAX_EDGE,
5 } from "@t3tools/client-runtime/image-attachment-size";
6 import {
7 IMAGE_DIMENSIONS_HEADER_BYTES,
8 readImageDimensions,
9 } from "@t3tools/shared/imageDimensions";
10 import { msg } from "gt-react";
11
12 import { m } from "../i18n/messages";
13
14 const MAX_DIMENSION = IMAGE_ATTACHMENT_MAX_EDGE;
15 export const MAX_STASH_IMAGE_DATA_URL_CHARS = 1_300_000;
16 /** Bound source decoding: enormous ImageBitmaps can OOM the tab before resizing. */
17 export const MAX_COMPRESSIBLE_SOURCE_BYTES = 50 * 1024 * 1024;
18 const MAX_HEIC_DECODE_PIXELS = 64_000_000;
19 const MAX_HEIC_METADATA_BYTES = 1024 * 1024;
20 const QUALITY_STEPS = [0.92, 0.85, 0.78, 0.68] as const;
21 const FALLBACK_SCALE_STEPS = [0.75, 0.55] as const;
22 const HEIC_IMAGE_MIME_TYPE = /^image\/hei(?:c|f)$/i;
23 const HEIC_IMAGE_EXTENSION = /\.(?:heic|heif)$/i;
24
25 type ImageSize = { width: number; height: number };
26
27 export interface CompressedStashImage {
28 imageSize?: ImageSize;
29 dataUrl: string;
30 mimeType: string;
31 sizeBytes: number;
32 recompressed: boolean;
33 }
34
35 export type ImageCompressionFailureReason = "too-large" | "unreadable";
36
37 export type CompressStashImageResult =
38 | { ok: true; image: CompressedStashImage }
39 | { ok: false; reason: ImageCompressionFailureReason };
40
41 export type CompressImageFileResult =
42 | { ok: true; file: File; recompressed: boolean; imageSize?: ImageSize }
43 | { ok: false; reason: ImageCompressionFailureReason };
44
45 /** Finder and some browsers omit the MIME type when dragging HEIC photos. */
46 export function isHeicImageFile(file: Pick<File, "name" | "type">): boolean {
47 if (HEIC_IMAGE_MIME_TYPE.test(file.type)) {
48 return true;
49 }
50 return (
51 (file.type === "" || file.type.toLowerCase() === "application/octet-stream") &&
52 HEIC_IMAGE_EXTENSION.test(file.name)
53 );
54 }
55
56 interface HeicMetadataBox {
57 payloadOffset: number;
58 endOffset: number;
59 }
60
61 function findHeicMetadataBox(
62 view: DataView,
63 startOffset: number,
64 endOffset: number,
65 type: number,
66 ): HeicMetadataBox | null {
67 let offset = startOffset;
68 while (offset + 8 <= endOffset) {
69 let size = view.getUint32(offset);
70 let headerSize = 8;
71 if (size === 1) {
72 if (offset + 16 > endOffset) return null;
73 const extendedSize = view.getBigUint64(offset + 8);
74 if (extendedSize > BigInt(Number.MAX_SAFE_INTEGER)) return null;
75 size = Number(extendedSize);
76 headerSize = 16;
77 } else if (size === 0) {
78 size = endOffset - offset;
79 }
80 if (size < headerSize || size > endOffset - offset) return null;
81
82 const nextOffset = offset + size;
83 if (view.getUint32(offset + 4) === type) {
84 return { payloadOffset: offset + headerSize, endOffset: nextOffset };
85 }
86 offset = nextOffset;
87 }
88 return null;
89 }
90
91 async function validateHeicImageDimensions(
92 file: File,
93 ): Promise<ImageCompressionFailureReason | null> {
94 const metadata = await file.slice(0, MAX_HEIC_METADATA_BYTES).arrayBuffer();
95 const view = new DataView(metadata);
96 const meta = findHeicMetadataBox(view, 0, view.byteLength, 0x6d657461);
97 if (!meta || meta.payloadOffset + 4 > meta.endOffset) return "unreadable";
98 const properties = findHeicMetadataBox(view, meta.payloadOffset + 4, meta.endOffset, 0x69707270);
99 if (!properties) return "unreadable";
100 const containers = findHeicMetadataBox(
101 view,
102 properties.payloadOffset,
103 properties.endOffset,
104 0x6970636f,
105 );
106 if (!containers) return "unreadable";
107
108 let offset = containers.payloadOffset;
109 let foundImageDimensions = false;
110 while (offset < containers.endOffset) {
111 const image = findHeicMetadataBox(view, offset, containers.endOffset, 0x69737065);
112 if (!image) break;
113 if (image.payloadOffset + 12 > image.endOffset) return "unreadable";
114 const width = view.getUint32(image.payloadOffset + 4);
115 const height = view.getUint32(image.payloadOffset + 8);
116 if (width === 0 || height === 0) return "unreadable";
117 if (width > MAX_HEIC_DECODE_PIXELS / height) return "too-large";
118 foundImageDimensions = true;
119 offset = image.endOffset;
120 }
121 return foundImageDimensions ? null : "unreadable";
122 }
123
124 export function readFileAsDataUrl(file: Blob): Promise<string> {
125 return new Promise((resolve, reject) => {
126 const reader = new FileReader();
127 reader.addEventListener("load", () => {
128 if (typeof reader.result === "string") {
129 resolve(reader.result);
130 return;
131 }
132 reject(new Error(m(msg("Could not read image data."))));
133 });
134 reader.addEventListener("error", () => {
135 reject(reader.error ?? new Error(m(msg("Failed to read image."))));
136 });
137 reader.readAsDataURL(file);
138 });
139 }
140
141 function dataUrlLength(blob: Blob): number {
142 return (
143 `data:${blob.type || "application/octet-stream"};base64,`.length + 4 * Math.ceil(blob.size / 3)
144 );
145 }
146
147 export function dataUrlToFile(dataUrl: string, name: string, mimeType: string): File {
148 const payload = dataUrl.slice(dataUrl.indexOf(",") + 1);
149 const binary = atob(payload);
150 const bytes = new Uint8Array(binary.length);
151 for (let index = 0; index < binary.length; index += 1) {
152 bytes[index] = binary.charCodeAt(index);
153 }
154 return new File([bytes], name, { type: mimeType });
155 }
156
157 function fileNameForMimeType(name: string, mimeType: string): string {
158 const extension = mimeType === "image/webp" ? ".webp" : ".jpg";
159 const dotIndex = name.lastIndexOf(".");
160 const base = dotIndex > 0 ? name.slice(0, dotIndex) : name;
161 return `${base}${extension}`;
162 }
163
164 function canRecompress(): boolean {
165 return (
166 typeof createImageBitmap === "function" &&
167 (typeof OffscreenCanvas === "function" || typeof document !== "undefined")
168 );
169 }
170
171 interface Canvas2D {
172 canvas: OffscreenCanvas | HTMLCanvasElement;
173 context: OffscreenCanvasRenderingContext2D | CanvasRenderingContext2D;
174 }
175
176 function createCanvas(width: number, height: number): Canvas2D | null {
177 if (typeof OffscreenCanvas === "function") {
178 const canvas = new OffscreenCanvas(width, height);
179 const context = canvas.getContext("2d");
180 if (!context) return null;
181 return { canvas, context };
182 }
183 if (typeof document === "undefined") return null;
184 const canvas = document.createElement("canvas");
185 canvas.width = width;
186 canvas.height = height;
187 const context = canvas.getContext("2d");
188 if (!context) return null;
189 return { canvas, context };
190 }
191
192 async function encodeCanvas(
193 canvas: OffscreenCanvas | HTMLCanvasElement,
194 quality: number,
195 mimeType: string,
196 ): Promise<Blob | null> {
197 let blob: Blob | null;
198 if (typeof HTMLCanvasElement !== "undefined" && canvas instanceof HTMLCanvasElement) {
199 blob = await new Promise<Blob | null>((resolve) => canvas.toBlob(resolve, mimeType, quality));
200 } else {
201 blob = await (canvas as OffscreenCanvas).convertToBlob({ type: mimeType, quality });
202 }
203 return blob?.type === mimeType ? blob : null;
204 }
205
206 const composerThumbnails = new WeakMap<File, Promise<string | null>>();
207 const settledComposerThumbnails = new WeakMap<File, string | null>();
208
209 function cacheComposerImageThumbnail(
210 file: File,
211 thumbnail: Promise<string | null>,
212 ): Promise<string | null> {
213 composerThumbnails.set(file, thumbnail);
214 void thumbnail.then((src) => settledComposerThumbnails.set(file, src));
215 return thumbnail;
216 }
217
218 /** `undefined` until the thumbnail settles, so a tile can paint a known thumbnail immediately. */
219 export function readComposerImageThumbnail(file: File): string | null | undefined {
220 return settledComposerThumbnails.get(file);
221 }
222
223 export function createComposerImageThumbnail(file: File): Promise<string | null> {
224 return (
225 composerThumbnails.get(file) ??
226 cacheComposerImageThumbnail(file, renderComposerImageThumbnail(file))
227 );
228 }
229
230 /**
231 * Pasted images are re-encoded after their placeholder already shows the source's thumbnail.
232 * Reusing it keeps the tile from flashing back to its fallback while the new file is decoded.
233 * Resolves once the target's thumbnail is readable, so a tile swapped to it paints at once even
234 * when compression finished before the source's thumbnail did.
235 */
236 export async function inheritComposerImageThumbnail(source: File, target: File): Promise<void> {
237 const sourceThumbnail = composerThumbnails.get(source);
238 if (source === target || !sourceThumbnail || composerThumbnails.has(target)) return;
239 cacheComposerImageThumbnail(
240 target,
241 sourceThumbnail.then((src) => src ?? renderComposerImageThumbnail(target)),
242 );
243 const src = await sourceThumbnail;
244 if (src) settledComposerThumbnails.set(target, src);
245 }
246
247 async function renderComposerImageThumbnail(file: File): Promise<string | null> {
248 if (isHeicImageFile(file)) return prepareHeicImage(file).thumbnail;
249 // Placeholders thumbnail the uncompressed source, which can be big enough to OOM the decode.
250 if (!canRecompress() || file.size > MAX_COMPRESSIBLE_SOURCE_BYTES) return null;
251 let bitmap: ImageBitmap | undefined;
252 try {
253 bitmap = await createImageBitmap(file);
254 return await renderComposerImageThumbnailFromBitmap(bitmap);
255 } catch {
256 return null;
257 } finally {
258 bitmap?.close();
259 }
260 }
261
262 async function renderComposerImageThumbnailFromBitmap(bitmap: ImageBitmap): Promise<string | null> {
263 try {
264 const side = Math.min(bitmap.width, bitmap.height);
265 if (side <= 0) return null;
266 const dimension = Math.min(256, side);
267 const surface = createCanvas(dimension, dimension);
268 if (!surface) return null;
269 surface.context.drawImage(
270 bitmap,
271 (bitmap.width - side) / 2,
272 (bitmap.height - side) / 2,
273 side,
274 side,
275 0,
276 0,
277 dimension,
278 dimension,
279 );
280 const blob = await encodeCanvas(surface.canvas, 1, "image/png");
281 return blob ? await readFileAsDataUrl(blob) : null;
282 } catch {
283 return null;
284 }
285 }
286
287 async function encodeWithinBudget(
288 bitmap: ImageBitmap,
289 maxDimension: number,
290 budgetChars: number,
291 preferredMimeType?: "image/jpeg",
292 ): Promise<{ blob: Blob; mimeType: string; imageSize: ImageSize } | null> {
293 const scale = Math.min(1, maxDimension / Math.max(bitmap.width, bitmap.height));
294 const width = Math.max(1, Math.round(bitmap.width * scale));
295 const height = Math.max(1, Math.round(bitmap.height * scale));
296 const target = createCanvas(width, height);
297 if (!target) return null;
298
299 // Probe the codec before drawing; JPEG needs a white matte because it has no alpha.
300 const mimeType =
301 preferredMimeType ??
302 ((await encodeCanvas(target.canvas, QUALITY_STEPS[0], "image/webp"))
303 ? "image/webp"
304 : "image/jpeg");
305
306 if (mimeType === "image/jpeg") {
307 target.context.fillStyle = "#ffffff";
308 target.context.fillRect(0, 0, width, height);
309 }
310 target.context.drawImage(bitmap, 0, 0, width, height);
311
312 for (const quality of QUALITY_STEPS) {
313 const encoded = await encodeCanvas(target.canvas, quality, mimeType);
314 if (!encoded) break;
315 if (dataUrlLength(encoded) <= budgetChars) {
316 return { blob: encoded, mimeType, imageSize: { width, height } };
317 }
318 }
319 return null;
320 }
321
322 type ReencodeResult =
323 | { ok: true; blob: Blob; mimeType: string; imageSize: ImageSize }
324 | { ok: false; reason: ImageCompressionFailureReason };
325
326 async function reencodeWithinBudget(
327 file: File,
328 budgetChars: number,
329 preferredMimeType?: "image/jpeg",
330 ): Promise<ReencodeResult> {
331 if (!canRecompress()) {
332 return { ok: false, reason: "too-large" };
333 }
334
335 let bitmap: ImageBitmap;
336 try {
337 bitmap = await createImageBitmap(file);
338 } catch {
339 return { ok: false, reason: "unreadable" };
340 }
341
342 try {
343 // Shrink the previous target; using MAX_DIMENSION leaves smaller images unchanged.
344 const baseDimension = Math.min(MAX_DIMENSION, Math.max(bitmap.width, bitmap.height));
345 let encodeFailed = false;
346 for (const dimensionScale of [1, ...FALLBACK_SCALE_STEPS]) {
347 const targetDimension = Math.max(1, Math.round(baseDimension * dimensionScale));
348 let encoded: Awaited<ReturnType<typeof encodeWithinBudget>>;
349 try {
350 encoded = await encodeWithinBudget(bitmap, targetDimension, budgetChars, preferredMimeType);
351 } catch {
352 encodeFailed = true;
353 continue;
354 }
355 encodeFailed = false;
356 if (encoded) {
357 return {
358 ok: true,
359 blob: encoded.blob,
360 mimeType: encoded.mimeType,
361 imageSize: encoded.imageSize,
362 };
363 }
364 }
365 return { ok: false, reason: encodeFailed ? "unreadable" : "too-large" };
366 } finally {
367 bitmap.close();
368 }
369 }
370
371 export async function compressImageForStash(
372 file: File,
373 budgetChars: number = MAX_STASH_IMAGE_DATA_URL_CHARS,
374 ): Promise<CompressStashImageResult> {
375 try {
376 if (dataUrlLength(file) <= budgetChars) {
377 return {
378 ok: true,
379 image: {
380 dataUrl: await readFileAsDataUrl(file),
381 mimeType: file.type,
382 sizeBytes: file.size,
383 recompressed: false,
384 },
385 };
386 }
387 const reencoded = await reencodeWithinBudget(file, budgetChars);
388 if (!reencoded.ok) {
389 return reencoded;
390 }
391 return {
392 ok: true,
393 image: {
394 dataUrl: await readFileAsDataUrl(reencoded.blob),
395 mimeType: reencoded.mimeType,
396 sizeBytes: reencoded.blob.size,
397 recompressed: true,
398 imageSize: reencoded.imageSize,
399 },
400 };
401 } catch {
402 return { ok: false, reason: "unreadable" };
403 }
404 }
405
406 async function fitImageToMaxEdge(file: File): Promise<{ file: File; imageSize: ImageSize } | null> {
407 const mimeType = downscalableImageMimeType(file.type);
408 if (!mimeType || !canRecompress() || file.size > MAX_COMPRESSIBLE_SOURCE_BYTES) return null;
409 let bitmap: ImageBitmap | undefined;
410 try {
411 const header = readImageDimensions(
412 new Uint8Array(await file.slice(0, IMAGE_DIMENSIONS_HEADER_BYTES).arrayBuffer()),
413 );
414 if (!header || !fitImageAttachmentSize(header)) return null;
415 bitmap = await createImageBitmap(file, { imageOrientation: "from-image" });
416 const imageSize = fitImageAttachmentSize(bitmap);
417 if (!imageSize) return null;
418 const target = createCanvas(imageSize.width, imageSize.height);
419 if (!target) return null;
420 target.context.imageSmoothingQuality = "high";
421 target.context.drawImage(bitmap, 0, 0, imageSize.width, imageSize.height);
422 const blob = await encodeCanvas(target.canvas, QUALITY_STEPS[0], mimeType);
423 if (!blob) return null;
424 return {
425 file: new File([blob], file.name, { type: mimeType, lastModified: file.lastModified }),
426 imageSize,
427 };
428 } catch {
429 return null;
430 } finally {
431 bitmap?.close();
432 }
433 }
434
435 export async function compressImageToByteLimit(
436 file: File,
437 maxBytes: number,
438 options?: { preferredMimeType?: "image/jpeg"; sourceSizeBytes?: number },
439 ): Promise<CompressImageFileResult> {
440 const fitted = await fitImageToMaxEdge(file);
441 if (fitted && fitted.file.size <= maxBytes) {
442 return { ok: true, file: fitted.file, recompressed: true, imageSize: fitted.imageSize };
443 }
444 if (!fitted && file.size <= maxBytes) {
445 return { ok: true, file, recompressed: false };
446 }
447 if ((options?.sourceSizeBytes ?? file.size) > MAX_COMPRESSIBLE_SOURCE_BYTES) {
448 return { ok: false, reason: "too-large" };
449 }
450 const budgetChars = Math.floor(maxBytes / 3) * 4;
451 const reencoded = await reencodeWithinBudget(
452 fitted?.file ?? file,
453 budgetChars,
454 options?.preferredMimeType,
455 );
456 if (!reencoded.ok) {
457 return reencoded;
458 }
459 return {
460 ok: true,
461 file: new File(
462 [reencoded.blob],
463 fileNameForMimeType(file.name || "image", reencoded.mimeType),
464 { type: reencoded.mimeType },
465 ),
466 recompressed: true,
467 imageSize: reencoded.imageSize,
468 };
469 }
470
471 const heicPreparations = new WeakMap<
472 File,
473 { image: Promise<CompressImageFileResult>; thumbnail: Promise<string | null> }
474 >();
475 // Preparing tiles share full-size decodes, so keep only one HEIC bitmap alive at a time.
476 let heicPreparationQueue = Promise.resolve();
477
478 function prepareHeicImage(file: File) {
479 const cached = heicPreparations.get(file);
480 if (cached) return cached;
481 let resolveThumbnail!: (src: string | null) => void;
482 const thumbnail = new Promise<string | null>((resolve) => {
483 resolveThumbnail = resolve;
484 });
485 const image = heicPreparationQueue
486 .then(() => convertHeicImage(file, resolveThumbnail))
487 .finally(() => resolveThumbnail(null));
488 heicPreparationQueue = image.then(
489 () => undefined,
490 () => undefined,
491 );
492 const preparation = { image, thumbnail };
493 heicPreparations.set(file, preparation);
494 return preparation;
495 }
496
497 async function convertHeicImage(
498 file: File,
499 onThumbnail: (src: string | null) => void,
500 ): Promise<CompressImageFileResult> {
501 if (file.size > MAX_COMPRESSIBLE_SOURCE_BYTES) {
502 return { ok: false, reason: "too-large" };
503 }
504
505 let bitmap: ImageBitmap | undefined;
506 try {
507 const dimensionError = await validateHeicImageDimensions(file);
508 if (dimensionError) {
509 return { ok: false, reason: dimensionError };
510 }
511 bitmap =
512 typeof createImageBitmap === "function"
513 ? await createImageBitmap(file).catch(() => undefined)
514 : undefined;
515 if (!bitmap) {
516 const { heicTo } = await import("heic-to/csp");
517 bitmap = await heicTo({ blob: file, type: "bitmap" });
518 }
519 const imageSize = fitImageAttachmentSize(bitmap) ?? {
520 width: bitmap.width,
521 height: bitmap.height,
522 };
523 const surface = createCanvas(imageSize.width, imageSize.height);
524 if (!surface) return { ok: false, reason: "unreadable" };
525 surface.context.drawImage(bitmap, 0, 0, imageSize.width, imageSize.height);
526 const [converted] = await Promise.all([
527 encodeCanvas(surface.canvas, QUALITY_STEPS[0], "image/jpeg"),
528 renderComposerImageThumbnailFromBitmap(bitmap).then(onThumbnail),
529 ]);
530 if (!converted) return { ok: false, reason: "unreadable" };
531 return {
532 ok: true,
533 file: new File([converted], fileNameForMimeType(file.name || "image", "image/jpeg"), {
534 type: "image/jpeg",
535 lastModified: file.lastModified,
536 }),
537 recompressed: true,
538 imageSize,
539 };
540 } catch {
541 return { ok: false, reason: "unreadable" };
542 } finally {
543 bitmap?.close();
544 }
545 }
546
547 export async function prepareImageForAttachment(
548 file: File,
549 maxBytes: number,
550 ): Promise<CompressImageFileResult> {
551 if (!isHeicImageFile(file)) return compressImageToByteLimit(file, maxBytes);
552 const prepared = await prepareHeicImage(file).image;
553 if (!prepared.ok || prepared.file.size <= maxBytes) return prepared;
554 const result = await compressImageToByteLimit(prepared.file, maxBytes, {
555 preferredMimeType: "image/jpeg",
556 sourceSizeBytes: file.size,
557 });
558
559 return result.ok ? { ...result, recompressed: true } : result;
560 }