A high-throughput and memory-efficient inference and serving engine for LLMs
Mirrored from https://github.com/tduncan-lum/Nvidia-Optimization-Stack---VLLM.git
Synced Nov 14, 04:15 UTC
Feature/video support in random mm dataset (#25963)
Signed-off-by: Eugene Khvedchenia <ekhvedchenia@nvidia.com> Signed-off-by: Eugene Khvedchenya <ekhvedchenia@nvidia.com> Co-authored-by: Roger Wang <hey@rogerw.io>
Eugene Khvedchenya committed
Oct 29, 2025 at 12:24 UTC
5e72216d176e7f47119263d45f42129ecb1d5b18
3 files changed
+601
-25
tests/benchmarks/test_random_dataset.py
+123
@@ -359,3 +359,126 @@ def test_random_mm_bucket_config_not_mutated(
359
assert len(mm_data) >= 1
360
for it in mm_data:
361
assert it.get("type") == "image_url"
362
+
363
+
364
+@pytest.mark.benchmark
365
+def test_random_mm_video_sampling(hf_tokenizer: PreTrainedTokenizerBase) -> None:
366
+ """Test video sampling functionality in RandomMultiModalDataset."""
367
+ ds = RandomMultiModalDataset(random_seed=42)
368
+
369
+ # Test with video bucket configuration
370
+ bucket_config = {
371
+ (64, 64, 1): 0.3, # Images
372
+ (64, 64, 8): 0.7, # Videos
373
+ }
374
+
375
+ limit_mm_per_prompt = {"image": 2, "video": 2}
376
+
377
+ samples = _collect_mm_samples(
378
+ ds,
379
+ hf_tokenizer,
380
+ num_requests=5,
381
+ base_items_per_request=1,
382
+ num_mm_items_range_ratio=0.0,
383
+ limit_mm_per_prompt=limit_mm_per_prompt,
384
+ bucket_config=bucket_config,
385
+ )
386
+
387
+ assert len(samples) == 5
388
+
389
+ # Check that we have both images and videos
390
+ video_count = 0
391
+ image_count = 0
392
+
393
+ for s in samples:
394
+ mm_data = cast(list[dict[str, Any]], s.multi_modal_data)
395
+ assert len(mm_data) == 1
396
+
397
+ item = mm_data[0]
398
+ if item.get("type") == "video_url":
399
+ video_count += 1
400
+ # Verify video URL format
401
+ url = item.get("video_url", {}).get("url", "")
402
+ assert url.startswith("data:video/mp4;base64,")
403
+ elif item.get("type") == "image_url":
404
+ image_count += 1
405
+ # Verify image URL format
406
+ url = item.get("image_url", {}).get("url", "")
407
+ assert url.startswith("data:image/jpeg;base64,")
408
+
409
+ # Should have some videos due to 0.7 probability
410
+ assert video_count > 0
411
+ assert image_count > 0
412
+
413
+
414
+@pytest.mark.benchmark
415
+def test_random_mm_video_only_sampling(hf_tokenizer: PreTrainedTokenizerBase) -> None:
416
+ """Test sampling with only video buckets."""
417
+ ds = RandomMultiModalDataset(random_seed=42)
418
+
419
+ bucket_config = {
420
+ (64, 64, 8): 1.0, # Only videos
421
+ }
422
+
423
+ limit_mm_per_prompt = {"image": 0, "video": 1}
424
+
425
+ samples = _collect_mm_samples(
426
+ ds,
427
+ hf_tokenizer,
428
+ num_requests=3,
429
+ base_items_per_request=1,
430
+ num_mm_items_range_ratio=0.0,
431
+ limit_mm_per_prompt=limit_mm_per_prompt,
432
+ bucket_config=bucket_config,
433
+ )
434
+
435
+ assert len(samples) == 3
436
+
437
+ for s in samples:
438
+ mm_data = cast(list[dict[str, Any]], s.multi_modal_data)
439
+ assert len(mm_data) == 1
440
+
441
+ item = mm_data[0]
442
+ assert item.get("type") == "video_url"
443
+ url = item.get("video_url", {}).get("url", "")
444
+ assert url.startswith("data:video/mp4;base64,")
445
+
446
+
447
+@pytest.mark.benchmark
448
+def test_random_mm_video_deterministic_sampling(
449
+ hf_tokenizer: PreTrainedTokenizerBase,
450
+) -> None:
451
+ """Test that video sampling is deterministic with same seed."""
452
+ seed = 123
453
+ ds_a = RandomMultiModalDataset(random_seed=seed)
454
+ ds_b = RandomMultiModalDataset(random_seed=seed)
455
+
456
+ bucket_config = {
457
+ (64, 64, 8): 1.0, # Only videos
458
+ }
459
+
460
+ limit_mm_per_prompt = {"image": 0, "video": 1}
461
+
462
+ a = _collect_mm_samples(
463
+ ds_a,
464
+ hf_tokenizer,
465
+ num_requests=3,
466
+ base_items_per_request=1,
467
+ num_mm_items_range_ratio=0.0,
468
+ limit_mm_per_prompt=limit_mm_per_prompt,
469
+ bucket_config=bucket_config,
470
+ )
471
+
472
+ b = _collect_mm_samples(
473
+ ds_b,
474
+ hf_tokenizer,
475
+ num_requests=3,
476
+ base_items_per_request=1,
477
+ num_mm_items_range_ratio=0.0,
478
+ limit_mm_per_prompt=limit_mm_per_prompt,
479
+ bucket_config=bucket_config,
480
+ )
481
+
482
+ fa = [_mm_fingerprint_sample(s) for s in a]
483
+ fb = [_mm_fingerprint_sample(s) for s in b]
484
+ assert fa == fb
tests/benchmarks/test_random_multimodal_dataset_video.py
new
+398
@@ -0,0 +1,398 @@
1
+# SPDX-License-Identifier: Apache-2.0
2
+# SPDX-FileCopyrightText: Copyright contributors to the vLLM project
3
+
4
+import base64
5
+import os
6
+from tempfile import NamedTemporaryFile
7
+from typing import Any, cast
8
+
9
+import cv2
10
+import pytest
11
+from transformers import AutoTokenizer, PreTrainedTokenizerBase
12
+
13
+from vllm.benchmarks.datasets import RandomMultiModalDataset, SampleRequest
14
+
15
+
16
+@pytest.fixture(scope="session")
17
+def hf_tokenizer() -> PreTrainedTokenizerBase:
18
+ """Use a small, commonly available tokenizer."""
19
+ return AutoTokenizer.from_pretrained("gpt2")
20
+
21
+
22
+@pytest.fixture
23
+def video_dataset() -> RandomMultiModalDataset:
24
+ """Create a RandomMultiModalDataset instance for testing."""
25
+ return RandomMultiModalDataset(random_seed=42)
26
+
27
+
28
+@pytest.mark.benchmark
29
+def test_generate_synthetic_video_different_seeds():
30
+ """Test that different seeds produce different videos."""
31
+ dataset1 = RandomMultiModalDataset(random_seed=123)
32
+ dataset2 = RandomMultiModalDataset(random_seed=456)
33
+
34
+ width, height, num_frames = 64, 48, 8
35
+
36
+ video1 = dataset1.generate_synthetic_video(width, height, num_frames)
37
+ video2 = dataset2.generate_synthetic_video(width, height, num_frames)
38
+
39
+ # Videos should be different due to different seeds
40
+ assert video1["bytes"] != video2["bytes"]
41
+
42
+
43
+@pytest.mark.benchmark
44
+def test_map_config_to_modality(video_dataset: RandomMultiModalDataset):
45
+ """Test modality mapping for different configurations."""
46
+ # Test image configuration (num_frames = 1)
47
+ assert video_dataset.map_config_to_modality((256, 256, 1)) == "image"
48
+ assert video_dataset.map_config_to_modality((720, 1280, 1)) == "image"
49
+
50
+ # Test video configurations (num_frames > 1)
51
+ assert video_dataset.map_config_to_modality((256, 256, 8)) == "video"
52
+ assert video_dataset.map_config_to_modality((720, 1280, 16)) == "video"
53
+ assert video_dataset.map_config_to_modality((64, 64, 32)) == "video"
54
+
55
+ # Test invalid configurations
56
+ with pytest.raises(ValueError, match="Invalid multimodal item configuration"):
57
+ video_dataset.map_config_to_modality((256, 256, 0))
58
+
59
+ with pytest.raises(ValueError, match="Invalid multimodal item configuration"):
60
+ video_dataset.map_config_to_modality((256, 256, -1))
61
+
62
+
63
+@pytest.mark.benchmark
64
+def test_generate_mm_item_video(video_dataset: RandomMultiModalDataset):
65
+ """Test generating multimodal items for video configurations."""
66
+ # Test video item generation
67
+ video_config = (64, 48, 8) # height, width, num_frames
68
+ result = video_dataset.generate_mm_item(video_config)
69
+
70
+ # Check the result structure matches OpenAI API format
71
+ assert isinstance(result, dict)
72
+ assert result["type"] == "video_url"
73
+ assert "video_url" in result
74
+ assert "url" in result["video_url"]
75
+
76
+ # Check that the URL is a data URL with base64 encoded video
77
+ url = result["video_url"]["url"]
78
+ assert url.startswith("data:video/mp4;base64,")
79
+
80
+ # Decode and verify the video content
81
+ base64_data = url.split(",")[1]
82
+ video_bytes = base64.b64decode(base64_data)
83
+ assert len(video_bytes) > 0
84
+
85
+ # Verify the video can be decoded
86
+ with NamedTemporaryFile(suffix=".mp4", delete=False) as temp_file:
87
+ temp_path = temp_file.name
88
+ temp_file.write(video_bytes)
89
+
90
+ try:
91
+ cap = cv2.VideoCapture(temp_path)
92
+ assert cap.isOpened()
93
+
94
+ frame_count = int(cap.get(cv2.CAP_PROP_FRAME_COUNT))
95
+ frame_width = int(cap.get(cv2.CAP_PROP_FRAME_WIDTH))
96
+ frame_height = int(cap.get(cv2.CAP_PROP_FRAME_HEIGHT))
97
+
98
+ assert frame_count == 8
99
+ assert frame_width == 48
100
+ assert frame_height == 64
101
+
102
+ cap.release()
103
+ finally:
104
+ if os.path.exists(temp_path):
105
+ os.unlink(temp_path)
106
+
107
+
108
+@pytest.mark.benchmark
109
+def test_generate_mm_item_image(video_dataset: RandomMultiModalDataset):
110
+ """Test generating multimodal items for image configurations."""
111
+ # Test image item generation
112
+ image_config = (64, 48, 1) # height, width, num_frames=1
113
+ result = video_dataset.generate_mm_item(image_config)
114
+
115
+ # Check the result structure matches OpenAI API format
116
+ assert isinstance(result, dict)
117
+ assert result["type"] == "image_url"
118
+ assert "image_url" in result
119
+ assert "url" in result["image_url"]
120
+
121
+ # Check that the URL is a data URL with base64 encoded image
122
+ url = result["image_url"]["url"]
123
+ assert url.startswith("data:image/jpeg;base64,")
124
+
125
+
126
+@pytest.mark.benchmark
127
+def test_generate_mm_item_invalid_config(video_dataset: RandomMultiModalDataset):
128
+ """Test error handling for invalid configurations."""
129
+ with pytest.raises(ValueError, match="Invalid multimodal item configuration"):
130
+ video_dataset.generate_mm_item((256, 256, 0))
131
+
132
+
133
+@pytest.mark.benchmark
134
+def test_sample_with_video_buckets(
135
+ video_dataset: RandomMultiModalDataset, hf_tokenizer: PreTrainedTokenizerBase
136
+):
137
+ """Test sampling with video bucket configurations."""
138
+ # Configure bucket with video probability > 0
139
+ bucket_config = {
140
+ (64, 64, 1): 0.3, # Images
141
+ (64, 64, 8): 0.7, # Videos
142
+ }
143
+
144
+ limit_mm_per_prompt = {"image": 5, "video": 3}
145
+
146
+ samples = video_dataset.sample(
147
+ tokenizer=hf_tokenizer,
148
+ num_requests=5,
149
+ base_items_per_request=2,
150
+ num_mm_items_range_ratio=0.0,
151
+ limit_mm_per_prompt=limit_mm_per_prompt,
152
+ bucket_config=bucket_config,
153
+ input_len=20,
154
+ output_len=5,
155
+ )
156
+
157
+ assert len(samples) == 5
158
+
159
+ # Check that samples contain both images and videos
160
+ video_count = 0
161
+ image_count = 0
162
+
163
+ for sample in samples:
164
+ assert isinstance(sample, SampleRequest)
165
+ assert sample.multi_modal_data is not None
166
+ assert isinstance(sample.multi_modal_data, list)
167
+
168
+ mm_data = cast(list[dict[str, Any]], sample.multi_modal_data)
169
+ assert len(mm_data) == 2 # base_items_per_request
170
+
171
+ for item in mm_data:
172
+ if item["type"] == "video_url":
173
+ video_count += 1
174
+ # Verify video URL format
175
+ url = item["video_url"]["url"]
176
+ assert url.startswith("data:video/mp4;base64,")
177
+ elif item["type"] == "image_url":
178
+ image_count += 1
179
+ # Verify image URL format
180
+ url = item["image_url"]["url"]
181
+ assert url.startswith("data:image/jpeg;base64,")
182
+
183
+ # Should have some videos due to 0.7 probability
184
+ assert video_count > 0
185
+ assert image_count > 0
186
+
187
+
188
+@pytest.mark.benchmark
189
+def test_sample_video_only_buckets(
190
+ video_dataset: RandomMultiModalDataset, hf_tokenizer: PreTrainedTokenizerBase
191
+):
192
+ """Test sampling with only video buckets."""
193
+ bucket_config = {
194
+ (64, 64, 8): 1.0, # Only videos
195
+ }
196
+
197
+ limit_mm_per_prompt = {"image": 0, "video": 2}
198
+
199
+ samples = video_dataset.sample(
200
+ tokenizer=hf_tokenizer,
201
+ num_requests=3,
202
+ base_items_per_request=1,
203
+ num_mm_items_range_ratio=0.0,
204
+ limit_mm_per_prompt=limit_mm_per_prompt,
205
+ bucket_config=bucket_config,
206
+ input_len=20,
207
+ output_len=5,
208
+ )
209
+
210
+ assert len(samples) == 3
211
+
212
+ for sample in samples:
213
+ assert isinstance(sample, SampleRequest)
214
+ assert sample.multi_modal_data is not None
215
+ assert isinstance(sample.multi_modal_data, list)
216
+
217
+ mm_data = cast(list[dict[str, Any]], sample.multi_modal_data)
218
+ assert len(mm_data) == 1
219
+
220
+ item = mm_data[0]
221
+ assert item["type"] == "video_url"
222
+ url = item["video_url"]["url"]
223
+ assert url.startswith("data:video/mp4;base64,")
224
+
225
+
226
+@pytest.mark.benchmark
227
+def test_sample_respects_video_limits(
228
+ video_dataset: RandomMultiModalDataset, hf_tokenizer: PreTrainedTokenizerBase
229
+):
230
+ """Test that sampling respects video limits per prompt."""
231
+ bucket_config = {
232
+ (64, 64, 8): 1.0, # Only videos
233
+ }
234
+
235
+ # Set very low video limit
236
+ limit_mm_per_prompt = {"image": 0, "video": 1}
237
+
238
+ samples = video_dataset.sample(
239
+ tokenizer=hf_tokenizer,
240
+ num_requests=3,
241
+ base_items_per_request=1,
242
+ num_mm_items_range_ratio=0.0,
243
+ limit_mm_per_prompt=limit_mm_per_prompt,
244
+ bucket_config=bucket_config,
245
+ input_len=20,
246
+ output_len=5,
247
+ )
248
+
249
+ assert len(samples) == 3
250
+
251
+ for sample in samples:
252
+ mm_data = cast(list[dict[str, Any]], sample.multi_modal_data)
253
+ assert len(mm_data) <= 1 # Should respect video limit
254
+
255
+
256
+@pytest.mark.benchmark
257
+def test_sample_mixed_buckets_with_zero_probability(
258
+ video_dataset: RandomMultiModalDataset, hf_tokenizer: PreTrainedTokenizerBase
259
+):
260
+ """Test sampling with mixed buckets including zero probability entries."""
261
+ bucket_config = {
262
+ (64, 64, 1): 0.5, # Images
263
+ (64, 64, 8): 0.5, # Videos
264
+ (128, 128, 16): 0.0, # Zero probability videos (should be ignored)
265
+ }
266
+
267
+ limit_mm_per_prompt = {"image": 2, "video": 2}
268
+
269
+ samples = video_dataset.sample(
270
+ tokenizer=hf_tokenizer,
271
+ num_requests=4,
272
+ base_items_per_request=2,
273
+ num_mm_items_range_ratio=0.0,
274
+ limit_mm_per_prompt=limit_mm_per_prompt,
275
+ bucket_config=bucket_config,
276
+ input_len=20,
277
+ output_len=5,
278
+ )
279
+
280
+ assert len(samples) == 4
281
+
282
+ # Should only see 64x64 videos, not 128x128 videos
283
+ for sample in samples:
284
+ mm_data = cast(list[dict[str, Any]], sample.multi_modal_data)
285
+ for item in mm_data:
286
+ if item["type"] == "video_url":
287
+ # Decode video to verify dimensions
288
+ url = item["video_url"]["url"]
289
+ base64_data = url.split(",")[1]
290
+ video_bytes = base64.b64decode(base64_data)
291
+
292
+ with NamedTemporaryFile(suffix=".mp4", delete=False) as temp_file: # noqa
293
+ temp_path = temp_file.name
294
+ temp_file.write(video_bytes)
295
+
296
+ try:
297
+ cap = cv2.VideoCapture(temp_path)
298
+ frame_width = int(cap.get(cv2.CAP_PROP_FRAME_WIDTH))
299
+ frame_height = int(cap.get(cv2.CAP_PROP_FRAME_HEIGHT))
300
+ cap.release()
301
+
302
+ # Should be 64x64, not 128x128
303
+ assert frame_width == 64
304
+ assert frame_height == 64
305
+ finally:
306
+ if os.path.exists(temp_path):
307
+ os.unlink(temp_path)
308
+
309
+
310
+@pytest.mark.benchmark
311
+def test_sample_deterministic_with_videos(hf_tokenizer: PreTrainedTokenizerBase):
312
+ """Test that sampling with videos is deterministic with same seed."""
313
+ dataset1 = RandomMultiModalDataset(random_seed=123)
314
+ dataset2 = RandomMultiModalDataset(random_seed=123)
315
+
316
+ bucket_config = {
317
+ (64, 64, 1): 0.3, # Images
318
+ (64, 64, 8): 0.7, # Videos
319
+ }
320
+
321
+ limit_mm_per_prompt = {"image": 2, "video": 2}
322
+
323
+ samples1 = dataset1.sample(
324
+ tokenizer=hf_tokenizer,
325
+ num_requests=3,
326
+ base_items_per_request=1,
327
+ num_mm_items_range_ratio=0.0,
328
+ limit_mm_per_prompt=limit_mm_per_prompt,
329
+ bucket_config=bucket_config,
330
+ input_len=20,
331
+ output_len=5,
332
+ )
333
+
334
+ samples2 = dataset2.sample(
335
+ tokenizer=hf_tokenizer,
336
+ num_requests=3,
337
+ base_items_per_request=1,
338
+ num_mm_items_range_ratio=0.0,
339
+ limit_mm_per_prompt=limit_mm_per_prompt,
340
+ bucket_config=bucket_config,
341
+ input_len=20,
342
+ output_len=5,
343
+ )
344
+
345
+ assert len(samples1) == len(samples2)
346
+
347
+ # Compare multimodal data
348
+ for s1, s2 in zip(samples1, samples2):
349
+ assert s1.multi_modal_data == s2.multi_modal_data
350
+
351
+
352
+@pytest.mark.benchmark
353
+def test_sample_different_seeds_produce_different_videos(
354
+ hf_tokenizer: PreTrainedTokenizerBase,
355
+):
356
+ """Test that different seeds produce different video content."""
357
+ dataset1 = RandomMultiModalDataset(random_seed=123)
358
+ dataset2 = RandomMultiModalDataset(random_seed=456)
359
+
360
+ bucket_config = {
361
+ (64, 64, 8): 1.0, # Only videos
362
+ }
363
+
364
+ limit_mm_per_prompt = {"image": 0, "video": 1}
365
+
366
+ samples1 = dataset1.sample(
367
+ tokenizer=hf_tokenizer,
368
+ num_requests=2,
369
+ base_items_per_request=1,
370
+ num_mm_items_range_ratio=0.0,
371
+ limit_mm_per_prompt=limit_mm_per_prompt,
372
+ bucket_config=bucket_config,
373
+ input_len=20,
374
+ output_len=5,
375
+ )
376
+
377
+ samples2 = dataset2.sample(
378
+ tokenizer=hf_tokenizer,
379
+ num_requests=2,
380
+ base_items_per_request=1,
381
+ num_mm_items_range_ratio=0.0,
382
+ limit_mm_per_prompt=limit_mm_per_prompt,
383
+ bucket_config=bucket_config,
384
+ input_len=20,
385
+ output_len=5,
386
+ )
387
+
388
+ # Video content should be different
389
+ for s1, s2 in zip(samples1, samples2):
390
+ mm_data1 = cast(list[dict[str, Any]], s1.multi_modal_data)
391
+ mm_data2 = cast(list[dict[str, Any]], s2.multi_modal_data)
392
+
393
+ assert len(mm_data1) == len(mm_data2) == 1
394
+
395
+ url1 = mm_data1[0]["video_url"]["url"]
396
+ url2 = mm_data2[0]["video_url"]["url"]
397
+
398
+ assert url1 != url2 # Different video content
vllm/benchmarks/datasets.py
+80
-25
@@ -27,8 +27,10 @@ from copy import deepcopy
27
from dataclasses import dataclass
28
from functools import cache
29
from io import BytesIO
30
+from tempfile import NamedTemporaryFile
31
from typing import Any, cast
32
33
+import cv2
34
import numpy as np
35
from PIL import Image
36
from transformers import PreTrainedTokenizerBase
@@ -498,9 +500,13 @@ class RandomDataset(BenchmarkDataset):
500
num_requests, range_ratio, input_len, output_len, tokenizer
501
)
502
501
- # Generate prefix once
502
- prefix_token_ids = self.get_prefix(tokenizer, prefix_len)
503
vocab_size = tokenizer.vocab_size
504
+ prohibited_tokens = tokenizer.all_special_ids
505
+ all_tokens = np.arange(vocab_size)
506
+ allowed_tokens = np.array(list(set(all_tokens) - set(prohibited_tokens)))
507
+
508
+ # Generate prefix once
509
+ prefix_token_ids = self.get_prefix(allowed_tokens, prefix_len)
510
511
requests = []
512
token_mismatch_total = 0
@@ -513,6 +519,7 @@ class RandomDataset(BenchmarkDataset):
519
input_len=int(input_lens[i]),
520
offset=int(offsets[i]),
521
index=i,
522
+ allowed_tokens=allowed_tokens,
523
)
524
token_mismatch_total += token_mismatch
525
requests.append(
@@ -553,13 +560,17 @@ class RandomDataset(BenchmarkDataset):
560
return requests
561
562
def get_prefix(
556
- self, tokenizer: PreTrainedTokenizerBase, prefix_len: int
563
+ self,
564
+ allowed_tokens: np.ndarray,
565
+ prefix_len: int,
566
) -> list[int]:
567
"""
568
Get the prefix for the dataset.
569
"""
570
return (
562
- self._rng.integers(0, tokenizer.vocab_size, size=prefix_len).tolist()
571
+ allowed_tokens[
572
+ self._rng.integers(0, len(allowed_tokens), size=prefix_len)
573
+ ].tolist()
574
if prefix_len > 0
575
else []
576
)
@@ -623,6 +634,7 @@ class RandomDataset(BenchmarkDataset):
634
input_len: int,
635
offset: int,
636
index: int,
637
+ allowed_tokens: np.ndarray,
638
) -> tuple[str, int, int]:
639
"""
640
Returns (prompt, total_input_len).
@@ -636,8 +648,11 @@ class RandomDataset(BenchmarkDataset):
648
To avoid uncontrolled change of the prompt length,
649
the encoded sequence is truncated before being decoded again.
650
"""
639
- # Build the inner sequence by sampling sequentially from the vocab
640
- inner_seq = ((offset + index + np.arange(input_len)) % vocab_size).tolist()
651
+ # Build the inner sequence by sampling
652
+ # sequentially from the allowed tokens
653
+ inner_seq = allowed_tokens[
654
+ (offset + index + np.arange(input_len)) % len(allowed_tokens)
655
+ ].tolist()
656
token_sequence = prefix_token_ids + inner_seq
657
658
# Decode, then re-encode and truncate to preserve token count invariants
@@ -772,7 +787,7 @@ class RandomMultiModalDataset(RandomDataset):
787
788
Status:
789
- Images: supported via synthetic RGB data.
775
- - Video: not yet supported (TODO: implement video generation method).
790
+ - Video: supported via synthetic RGB data.
791
- Audio: not yet supported.
792
793
Sampling overview:
@@ -782,7 +797,7 @@ class RandomMultiModalDataset(RandomDataset):
797
The maximum is further clamped to the sum of per-modality limits.
798
2) Each item’s modality and shape is sampled from `bucket_config`, a dict
799
mapping (height, width, num_frames) → probability. We treat
785
- `num_frames`=1 as image and and `num_frames` > 1 as video.
800
+ `num_frames`=1 as image and `num_frames` > 1 as video.
801
Entries with zero probability are removed and the rest are renormalized
802
to sum to 1.
803
3) Per-modality hard caps are enforced via `limit_mm_per_prompt`.
@@ -797,8 +812,7 @@ class RandomMultiModalDataset(RandomDataset):
812
"""
813
814
IS_MULTIMODAL = True
800
- # NOTE: video sampling is WIP. Setting it to 0.
801
- DEFAULT_LIMIT_MM_PER_PROMPT = {"image": 255, "video": 0}
815
+ DEFAULT_LIMIT_MM_PER_PROMPT = {"image": 255, "video": 1}
816
817
DEFAULT_BASE_ITEMS_PER_REQUEST = 1
818
DEFAULT_NUM_MM_ITEMS_RANGE_RATIO = 0.0
@@ -828,12 +842,47 @@ class RandomMultiModalDataset(RandomDataset):
842
)
843
return Image.fromarray(random_pixels)
844
831
- def generate_synthetic_video(self, width: int, height: int, num_frames: int) -> Any:
845
+ def generate_synthetic_video(
846
+ self, width: int, height: int, num_frames: int
847
+ ) -> dict:
848
"""Generate synthetic video with random values.
849
834
- TODO: Finish this method.
850
+ Creates a video with random pixel values, encodes it to MP4 format,
851
+ and returns the content as bytes.
852
"""
836
- raise NotImplementedError("Video sampling is WIP.")
853
+ random_pixels = self._rng.integers(
854
+ 0,
855
+ 256,
856
+ (num_frames, height, width, 3),
857
+ dtype=np.uint8,
858
+ )
859
+
860
+ # Create a temporary video file in memory
861
+ fourcc = cv2.VideoWriter_fourcc(*"mp4v")
862
+ fps = 30 # frames per second
863
+
864
+ with NamedTemporaryFile(suffix=".mp4", delete_on_close=False) as temp_file:
865
+ temp_path = temp_file.name
866
+
867
+ # Create video writer
868
+ video_writer = cv2.VideoWriter(
869
+ temp_path, fourcc=fourcc, fps=fps, frameSize=(width, height)
870
+ )
871
+
872
+ if not video_writer.isOpened():
873
+ raise RuntimeError("Failed to create video writer")
874
+
875
+ for frame in random_pixels:
876
+ video_writer.write(frame)
877
+
878
+ video_writer.release()
879
+ temp_file.close()
880
+
881
+ # Read the video file content
882
+ with open(temp_path, "rb") as f:
883
+ video_content = f.read()
884
+
885
+ return {"bytes": video_content}
886
887
def map_config_to_modality(self, config: tuple[int, int, int]) -> str:
888
"""Map the configuration to the modality."""
@@ -1044,16 +1093,6 @@ class RandomMultiModalDataset(RandomDataset):
1093
enable_multimodal_chat: bool = DEFAULT_ENABLE_MULTIMODAL_CHAT,
1094
**kwargs,
1095
) -> list[SampleRequest]:
1047
- # NOTE: Video sampling is WIP. Raise error if video is in bucket config
1048
- # and probability is non-zero.
1049
- if any(
1050
- self.map_config_to_modality(cfg) == "video" and p > 0
1051
- for cfg, p in bucket_config.items()
1052
- ):
1053
- raise NotImplementedError(
1054
- "Video sampling not implemented; set its probability to 0."
1055
- )
1056
-
1096
# Get the sampling parameters for the dataset
1097
input_lens, output_lens, offsets = self.get_sampling_params(
1098
num_requests, range_ratio, input_len, output_len, tokenizer
@@ -1071,9 +1110,24 @@ class RandomMultiModalDataset(RandomDataset):
1110
bucket_config,
1111
)
1112
1074
- # Generate prefix once
1075
- prefix_token_ids = self.get_prefix(tokenizer, prefix_len)
1113
vocab_size = tokenizer.vocab_size
1114
+ # Can't use tokenizer.all_special_ids since
1115
+ # it returns ONLY ids from special_tokens_map.json
1116
+ # We want to exclude placeholder tokens and all
1117
+ # tokens that indicate start/end of image as it
1118
+ # may break prompt replacement logic.
1119
+ prohibited_tokens = list(
1120
+ tok_id
1121
+ for tok_id, token in tokenizer.added_tokens_decoder.items()
1122
+ if token.special
1123
+ )
1124
+ all_tokens = np.arange(vocab_size)
1125
+ allowed_tokens = np.array(list(set(all_tokens) - set(prohibited_tokens)))
1126
+ logger.debug(
1127
+ "Sampling from %d out of %d (vocab size)", len(allowed_tokens), vocab_size
1128
+ )
1129
+ # Generate prefix once
1130
+ prefix_token_ids = self.get_prefix(allowed_tokens, prefix_len)
1131
# Add synthetic multimodal items to each request
1132
mm_requests = []
1133
token_mismatch_total = 0
@@ -1086,6 +1140,7 @@ class RandomMultiModalDataset(RandomDataset):
1140
input_len=int(input_lens[i]),
1141
offset=int(offsets[i]),
1142
index=i,
1143
+ allowed_tokens=allowed_tokens,
1144
)
1145
token_mismatch_total += token_mismatch
1146
# Get multimodal item iterator for a given request