[Flight Reply] Encode Typed Arrays and Blobs (#28819)
With the enableBinaryFlight flag on we should encode typed arrays and blobs in the Reply direction too for parity. It's already possible to pass Blobs inside FormData but you should be able to pass them inside objects too. We encode typed arrays as blobs and then unwrap them automatically to the right typed array type. Unlike the other protocol, I encode the type as a reference tag instead of row tag. Therefore I need to rename the tags to avoid conflicts with other tags in references. We are running out of characters though.
Sebastian Markbåge committed
Apr 15, 2024 at 22:32 UTC
17e920c00d314bfc6833e99ff4cb5c91fb3da254
5 files changed
+302
-46
packages/react-client/src/ReactFlightClient.js
+10
-10
@@ -1262,10 +1262,10 @@ function processFullRow(
1262
// We must always clone to extract it into a separate buffer instead of just a view.
1263
resolveBuffer(response, id, mergeBuffer(buffer, chunk).buffer);
1264
return;
1265
- case 67 /* "C" */:
1265
+ case 79 /* "O" */:
1266
resolveTypedArray(response, id, buffer, chunk, Int8Array, 1);
1267
return;
1268
- case 99 /* "c" */:
1268
+ case 111 /* "o" */:
1269
resolveBuffer(
1270
response,
1271
id,
@@ -1287,13 +1287,13 @@ function processFullRow(
1287
case 108 /* "l" */:
1288
resolveTypedArray(response, id, buffer, chunk, Uint32Array, 4);
1289
return;
1290
- case 70 /* "F" */:
1290
+ case 71 /* "G" */:
1291
resolveTypedArray(response, id, buffer, chunk, Float32Array, 4);
1292
return;
1293
- case 100 /* "d" */:
1293
+ case 103 /* "g" */:
1294
resolveTypedArray(response, id, buffer, chunk, Float64Array, 8);
1295
return;
1296
- case 78 /* "N" */:
1296
+ case 77 /* "M" */:
1297
resolveTypedArray(response, id, buffer, chunk, BigInt64Array, 8);
1298
return;
1299
case 109 /* "m" */:
@@ -1417,16 +1417,16 @@ export function processBinaryChunk(
1417
resolvedRowTag === 84 /* "T" */ ||
1418
(enableBinaryFlight &&
1419
(resolvedRowTag === 65 /* "A" */ ||
1420
- resolvedRowTag === 67 /* "C" */ ||
1421
- resolvedRowTag === 99 /* "c" */ ||
1420
+ resolvedRowTag === 79 /* "O" */ ||
1421
+ resolvedRowTag === 111 /* "o" */ ||
1422
resolvedRowTag === 85 /* "U" */ ||
1423
resolvedRowTag === 83 /* "S" */ ||
1424
resolvedRowTag === 115 /* "s" */ ||
1425
resolvedRowTag === 76 /* "L" */ ||
1426
resolvedRowTag === 108 /* "l" */ ||
1427
- resolvedRowTag === 70 /* "F" */ ||
1428
- resolvedRowTag === 100 /* "d" */ ||
1429
- resolvedRowTag === 78 /* "N" */ ||
1427
+ resolvedRowTag === 71 /* "G" */ ||
1428
+ resolvedRowTag === 103 /* "g" */ ||
1429
+ resolvedRowTag === 77 /* "M" */ ||
1430
resolvedRowTag === 109 /* "m" */ ||
1431
resolvedRowTag === 86)) /* "V" */
1432
) {
packages/react-client/src/ReactFlightReplyClient.js
+85
-1
@@ -17,7 +17,10 @@ import type {
17
import type {LazyComponent} from 'react/src/ReactLazy';
18
import type {TemporaryReferenceSet} from './ReactFlightTemporaryReferences';
19
20
-import {enableRenderableContext} from 'shared/ReactFeatureFlags';
20
+import {
21
+ enableRenderableContext,
22
+ enableBinaryFlight,
23
+} from 'shared/ReactFeatureFlags';
24
25
import {
26
REACT_ELEMENT_TYPE,
@@ -150,6 +153,10 @@ function serializeSetID(id: number): string {
153
return '$W' + id.toString(16);
154
}
155
156
+function serializeBlobID(id: number): string {
157
+ return '$B' + id.toString(16);
158
+}
159
+
160
function escapeStringValue(value: string): string {
161
if (value[0] === '$') {
162
// We need to escape $ prefixed strings since we use those to encode
@@ -171,6 +178,19 @@ export function processReply(
178
let pendingParts = 0;
179
let formData: null | FormData = null;
180
181
+ function serializeTypedArray(
182
+ tag: string,
183
+ typedArray: ArrayBuffer | $ArrayBufferView,
184
+ ): string {
185
+ const blob = new Blob([typedArray]);
186
+ const blobId = nextPartId++;
187
+ if (formData === null) {
188
+ formData = new FormData();
189
+ }
190
+ formData.append(formFieldPrefix + blobId, blob);
191
+ return '$' + tag + blobId.toString(16);
192
+ }
193
+
194
function resolveToJSON(
195
this:
196
| {+[key: string | number]: ReactServerValue}
@@ -362,6 +382,70 @@ export function processReply(
382
formData.append(formFieldPrefix + setId, partJSON);
383
return serializeSetID(setId);
384
}
385
+
386
+ if (enableBinaryFlight) {
387
+ if (value instanceof ArrayBuffer) {
388
+ return serializeTypedArray('A', value);
389
+ }
390
+ if (value instanceof Int8Array) {
391
+ // char
392
+ return serializeTypedArray('O', value);
393
+ }
394
+ if (value instanceof Uint8Array) {
395
+ // unsigned char
396
+ return serializeTypedArray('o', value);
397
+ }
398
+ if (value instanceof Uint8ClampedArray) {
399
+ // unsigned clamped char
400
+ return serializeTypedArray('U', value);
401
+ }
402
+ if (value instanceof Int16Array) {
403
+ // sort
404
+ return serializeTypedArray('S', value);
405
+ }
406
+ if (value instanceof Uint16Array) {
407
+ // unsigned short
408
+ return serializeTypedArray('s', value);
409
+ }
410
+ if (value instanceof Int32Array) {
411
+ // long
412
+ return serializeTypedArray('L', value);
413
+ }
414
+ if (value instanceof Uint32Array) {
415
+ // unsigned long
416
+ return serializeTypedArray('l', value);
417
+ }
418
+ if (value instanceof Float32Array) {
419
+ // float
420
+ return serializeTypedArray('G', value);
421
+ }
422
+ if (value instanceof Float64Array) {
423
+ // double
424
+ return serializeTypedArray('g', value);
425
+ }
426
+ if (value instanceof BigInt64Array) {
427
+ // number
428
+ return serializeTypedArray('M', value);
429
+ }
430
+ if (value instanceof BigUint64Array) {
431
+ // unsigned number
432
+ // We use "m" instead of "n" since JSON can start with "null"
433
+ return serializeTypedArray('m', value);
434
+ }
435
+ if (value instanceof DataView) {
436
+ return serializeTypedArray('V', value);
437
+ }
438
+ // TODO: Blob is not available in old Node/browsers. Remove the typeof check later.
439
+ if (typeof Blob === 'function' && value instanceof Blob) {
440
+ if (formData === null) {
441
+ formData = new FormData();
442
+ }
443
+ const blobId = nextPartId++;
444
+ formData.append(formFieldPrefix + blobId, value);
445
+ return serializeBlobID(blobId);
446
+ }
447
+ }
448
+
449
const iteratorFn = getIteratorFn(value);
450
if (iteratorFn) {
451
return Array.from((value: any));
packages/react-server-dom-webpack/src/__tests__/ReactFlightDOMReplyEdge-test.js
+93
@@ -5,6 +5,7 @@
5
* LICENSE file in the root directory of this source tree.
6
*
7
* @emails react-core
8
+ * @jest-environment ./scripts/jest/ReactDOMServerIntegrationEnvironment
9
*/
10
11
'use strict';
@@ -15,6 +16,13 @@ global.ReadableStream =
16
global.TextEncoder = require('util').TextEncoder;
17
global.TextDecoder = require('util').TextDecoder;
18
19
+if (typeof Blob === 'undefined') {
20
+ global.Blob = require('buffer').Blob;
21
+}
22
+if (typeof File === 'undefined') {
23
+ global.File = require('buffer').File;
24
+}
25
+
26
// let serverExports;
27
let webpackServerMap;
28
let ReactServerDOMServer;
@@ -36,6 +44,13 @@ describe('ReactFlightDOMReplyEdge', () => {
44
ReactServerDOMClient = require('react-server-dom-webpack/client.edge');
45
});
46
47
+ if (typeof FormData === 'undefined') {
48
+ // We can't test if we don't have a native FormData implementation because the JSDOM one
49
+ // is missing the arrayBuffer() method.
50
+ it('cannot test', () => {});
51
+ return;
52
+ }
53
+
54
it('can encode a reply', async () => {
55
const body = await ReactServerDOMClient.encodeReply({some: 'object'});
56
const decoded = await ReactServerDOMServer.decodeReply(
@@ -45,4 +60,82 @@ describe('ReactFlightDOMReplyEdge', () => {
60
61
expect(decoded).toEqual({some: 'object'});
62
});
63
+
64
+ // @gate enableBinaryFlight
65
+ it('should be able to serialize any kind of typed array', async () => {
66
+ const buffer = new Uint8Array([
67
+ 123, 4, 10, 5, 100, 255, 244, 45, 56, 67, 43, 124, 67, 89, 100, 20,
68
+ ]).buffer;
69
+ const buffers = [
70
+ buffer,
71
+ new Int8Array(buffer, 1),
72
+ new Uint8Array(buffer, 2),
73
+ new Uint8ClampedArray(buffer, 2),
74
+ new Int16Array(buffer, 2),
75
+ new Uint16Array(buffer, 2),
76
+ new Int32Array(buffer, 4),
77
+ new Uint32Array(buffer, 4),
78
+ new Float32Array(buffer, 4),
79
+ new Float64Array(buffer, 0),
80
+ new BigInt64Array(buffer, 0),
81
+ new BigUint64Array(buffer, 0),
82
+ new DataView(buffer, 3),
83
+ ];
84
+
85
+ const body = await ReactServerDOMClient.encodeReply(buffers);
86
+ const result = await ReactServerDOMServer.decodeReply(
87
+ body,
88
+ webpackServerMap,
89
+ );
90
+
91
+ expect(result).toEqual(buffers);
92
+ });
93
+
94
+ // @gate enableBinaryFlight
95
+ it('should be able to serialize a blob', async () => {
96
+ const bytes = new Uint8Array([
97
+ 123, 4, 10, 5, 100, 255, 244, 45, 56, 67, 43, 124, 67, 89, 100, 20,
98
+ ]);
99
+ const blob = new Blob([bytes, bytes], {
100
+ type: 'application/x-test',
101
+ });
102
+ const body = await ReactServerDOMClient.encodeReply(blob);
103
+ const result = await ReactServerDOMServer.decodeReply(
104
+ body,
105
+ webpackServerMap,
106
+ );
107
+ expect(result instanceof Blob).toBe(true);
108
+ expect(result.size).toBe(bytes.length * 2);
109
+ expect(await result.arrayBuffer()).toEqual(await blob.arrayBuffer());
110
+ });
111
+
112
+ it('can transport FormData (blobs)', async () => {
113
+ const bytes = new Uint8Array([
114
+ 123, 4, 10, 5, 100, 255, 244, 45, 56, 67, 43, 124, 67, 89, 100, 20,
115
+ ]);
116
+ const blob = new Blob([bytes, bytes], {
117
+ type: 'application/x-test',
118
+ });
119
+
120
+ const formData = new FormData();
121
+ formData.append('hi', 'world');
122
+ formData.append('file', blob, 'filename.test');
123
+
124
+ expect(formData.get('file') instanceof File).toBe(true);
125
+ expect(formData.get('file').name).toBe('filename.test');
126
+
127
+ const body = await ReactServerDOMClient.encodeReply(formData);
128
+ const result = await ReactServerDOMServer.decodeReply(
129
+ body,
130
+ webpackServerMap,
131
+ );
132
+
133
+ expect(result instanceof FormData).toBe(true);
134
+ expect(result.get('hi')).toBe('world');
135
+ const resultBlob = result.get('file');
136
+ expect(resultBlob instanceof Blob).toBe(true);
137
+ expect(resultBlob.name).toBe('filename.test'); // In this direction we allow file name to pass through but not other direction.
138
+ expect(resultBlob.size).toBe(bytes.length * 2);
139
+ expect(await resultBlob.arrayBuffer()).toEqual(await blob.arrayBuffer());
140
+ });
141
});
packages/react-server/src/ReactFlightReplyServer.js
+104
-25
@@ -25,6 +25,7 @@ import {
25
} from 'react-client/src/ReactFlightClientConfig';
26
27
import {createTemporaryReference} from './ReactFlightServerTemporaryReferences';
28
+import {enableBinaryFlight} from 'shared/ReactFeatureFlags';
29
30
export type JSONValue =
31
| number
@@ -378,9 +379,41 @@ function getOutlinedModel(response: Response, id: number): any {
379
return chunk.value;
380
}
381
381
-function parseModelString(
382
+function parseTypedArray(
383
response: Response,
384
+ reference: string,
385
+ constructor: any,
386
+ bytesPerElement: number,
387
parentObject: Object,
388
+ parentKey: string,
389
+): null {
390
+ const id = parseInt(reference.slice(2), 16);
391
+ const prefix = response._prefix;
392
+ const key = prefix + id;
393
+ // We should have this backingEntry in the store already because we emitted
394
+ // it before referencing it. It should be a Blob.
395
+ const backingEntry: Blob = (response._formData.get(key): any);
396
+
397
+ const promise =
398
+ constructor === ArrayBuffer
399
+ ? backingEntry.arrayBuffer()
400
+ : backingEntry.arrayBuffer().then(function (buffer) {
401
+ return new constructor(buffer);
402
+ });
403
+
404
+ // Since loading the buffer is an async operation we'll be blocking the parent
405
+ // chunk. TODO: This is not safe if the parent chunk needs a mapper like Map.
406
+ const parentChunk = initializingChunk;
407
+ promise.then(
408
+ createModelResolver(parentChunk, parentObject, parentKey),
409
+ createModelReject(parentChunk),
410
+ );
411
+ return null;
412
+}
413
+
414
+function parseModelString(
415
+ response: Response,
416
+ obj: Object,
417
key: string,
418
value: string,
419
): any {
@@ -407,7 +440,7 @@ function parseModelString(
440
metaData.id,
441
metaData.bound,
442
initializingChunk,
410
- parentObject,
443
+ obj,
444
key,
445
);
446
}
@@ -473,32 +506,78 @@ function parseModelString(
506
// BigInt
507
return BigInt(value.slice(2));
508
}
476
- default: {
477
- // We assume that anything else is a reference ID.
478
- const id = parseInt(value.slice(1), 16);
479
- const chunk = getChunk(response, id);
480
- switch (chunk.status) {
481
- case RESOLVED_MODEL:
482
- initializeModelChunk(chunk);
483
- break;
484
- }
485
- // The status might have changed after initialization.
486
- switch (chunk.status) {
487
- case INITIALIZED:
488
- return chunk.value;
489
- case PENDING:
490
- case BLOCKED:
491
- const parentChunk = initializingChunk;
492
- chunk.then(
493
- createModelResolver(parentChunk, parentObject, key),
494
- createModelReject(parentChunk),
495
- );
496
- return null;
497
- default:
498
- throw chunk.reason;
509
+ }
510
+ if (enableBinaryFlight) {
511
+ switch (value[1]) {
512
+ case 'A':
513
+ return parseTypedArray(response, value, ArrayBuffer, 1, obj, key);
514
+ case 'O':
515
+ return parseTypedArray(response, value, Int8Array, 1, obj, key);
516
+ case 'o':
517
+ return parseTypedArray(response, value, Uint8Array, 1, obj, key);
518
+ case 'U':
519
+ return parseTypedArray(
520
+ response,
521
+ value,
522
+ Uint8ClampedArray,
523
+ 1,
524
+ obj,
525
+ key,
526
+ );
527
+ case 'S':
528
+ return parseTypedArray(response, value, Int16Array, 2, obj, key);
529
+ case 's':
530
+ return parseTypedArray(response, value, Uint16Array, 2, obj, key);
531
+ case 'L':
532
+ return parseTypedArray(response, value, Int32Array, 4, obj, key);
533
+ case 'l':
534
+ return parseTypedArray(response, value, Uint32Array, 4, obj, key);
535
+ case 'G':
536
+ return parseTypedArray(response, value, Float32Array, 4, obj, key);
537
+ case 'g':
538
+ return parseTypedArray(response, value, Float64Array, 8, obj, key);
539
+ case 'M':
540
+ return parseTypedArray(response, value, BigInt64Array, 8, obj, key);
541
+ case 'm':
542
+ return parseTypedArray(response, value, BigUint64Array, 8, obj, key);
543
+ case 'V':
544
+ return parseTypedArray(response, value, DataView, 1, obj, key);
545
+ case 'B': {
546
+ // Blob
547
+ const id = parseInt(value.slice(2), 16);
548
+ const prefix = response._prefix;
549
+ const blobKey = prefix + id;
550
+ // We should have this backingEntry in the store already because we emitted
551
+ // it before referencing it. It should be a Blob.
552
+ const backingEntry: Blob = (response._formData.get(blobKey): any);
553
+ return backingEntry;
554
}
555
}
556
}
557
+
558
+ // We assume that anything else is a reference ID.
559
+ const id = parseInt(value.slice(1), 16);
560
+ const chunk = getChunk(response, id);
561
+ switch (chunk.status) {
562
+ case RESOLVED_MODEL:
563
+ initializeModelChunk(chunk);
564
+ break;
565
+ }
566
+ // The status might have changed after initialization.
567
+ switch (chunk.status) {
568
+ case INITIALIZED:
569
+ return chunk.value;
570
+ case PENDING:
571
+ case BLOCKED:
572
+ const parentChunk = initializingChunk;
573
+ chunk.then(
574
+ createModelResolver(parentChunk, obj, key),
575
+ createModelReject(parentChunk),
576
+ );
577
+ return null;
578
+ default:
579
+ throw chunk.reason;
580
+ }
581
}
582
return value;
583
}
packages/react-server/src/ReactFlightServer.js
+10
-10
@@ -1610,11 +1610,11 @@ function renderModelDestructive(
1610
}
1611
if (value instanceof Int8Array) {
1612
// char
1613
- return serializeTypedArray(request, 'C', value);
1613
+ return serializeTypedArray(request, 'O', value);
1614
}
1615
if (value instanceof Uint8Array) {
1616
// unsigned char
1617
- return serializeTypedArray(request, 'c', value);
1617
+ return serializeTypedArray(request, 'o', value);
1618
}
1619
if (value instanceof Uint8ClampedArray) {
1620
// unsigned clamped char
@@ -1638,15 +1638,15 @@ function renderModelDestructive(
1638
}
1639
if (value instanceof Float32Array) {
1640
// float
1641
- return serializeTypedArray(request, 'F', value);
1641
+ return serializeTypedArray(request, 'G', value);
1642
}
1643
if (value instanceof Float64Array) {
1644
// double
1645
- return serializeTypedArray(request, 'd', value);
1645
+ return serializeTypedArray(request, 'g', value);
1646
}
1647
if (value instanceof BigInt64Array) {
1648
// number
1649
- return serializeTypedArray(request, 'N', value);
1649
+ return serializeTypedArray(request, 'M', value);
1650
}
1651
if (value instanceof BigUint64Array) {
1652
// unsigned number
@@ -2158,11 +2158,11 @@ function renderConsoleValue(
2158
}
2159
if (value instanceof Int8Array) {
2160
// char
2161
- return serializeTypedArray(request, 'C', value);
2161
+ return serializeTypedArray(request, 'O', value);
2162
}
2163
if (value instanceof Uint8Array) {
2164
// unsigned char
2165
- return serializeTypedArray(request, 'c', value);
2165
+ return serializeTypedArray(request, 'o', value);
2166
}
2167
if (value instanceof Uint8ClampedArray) {
2168
// unsigned clamped char
@@ -2186,15 +2186,15 @@ function renderConsoleValue(
2186
}
2187
if (value instanceof Float32Array) {
2188
// float
2189
- return serializeTypedArray(request, 'F', value);
2189
+ return serializeTypedArray(request, 'G', value);
2190
}
2191
if (value instanceof Float64Array) {
2192
// double
2193
- return serializeTypedArray(request, 'd', value);
2193
+ return serializeTypedArray(request, 'g', value);
2194
}
2195
if (value instanceof BigInt64Array) {
2196
// number
2197
- return serializeTypedArray(request, 'N', value);
2197
+ return serializeTypedArray(request, 'M', value);
2198
}
2199
if (value instanceof BigUint64Array) {
2200
// unsigned number