A high-throughput and memory-efficient inference and serving engine for LLMs
Mirrored from https://github.com/tduncan-lum/Nvidia-Optimization-Stack---VLLM.git
Synced Nov 14, 05:30 UTC
[Model] Use `merge_by_field_config` for MM models (Llava family) (#26280)
Signed-off-by: DarkLight1337 <tlleungac@connect.ust.hk>
Cyrus Leung committed
Oct 6, 2025 at 17:45 UTC
19a00eb21099f7cd82fc4460718613df6f54fea1
9 files changed
+99
-173
examples/offline_inference/vision_language_multi_image.py
+53
-53
@@ -371,13 +371,14 @@ def load_internvl(question: str, image_urls: list[str]) -> ModelRequestData:
371
)
372
373
374
-def load_llama4(question: str, image_urls: list[str]) -> ModelRequestData:
375
- model_name = "meta-llama/Llama-4-Scout-17B-16E-Instruct"
374
+def load_keye_vl(question: str, image_urls: list[str]) -> ModelRequestData:
375
+ model_name = "Kwai-Keye/Keye-VL-8B-Preview"
376
377
engine_args = EngineArgs(
378
model=model_name,
379
- max_model_len=131072,
380
- tensor_parallel_size=8,
379
+ trust_remote_code=True,
380
+ max_model_len=8192,
381
+ max_num_seqs=5,
382
limit_mm_per_prompt={"image": len(image_urls)},
383
)
384
@@ -389,29 +390,32 @@ def load_llama4(question: str, image_urls: list[str]) -> ModelRequestData:
390
*placeholders,
391
{"type": "text", "text": question},
392
],
392
- }
393
+ },
394
]
395
395
- processor = AutoProcessor.from_pretrained(model_name)
396
+ processor = AutoProcessor.from_pretrained(model_name, trust_remote_code=True)
397
398
prompt = processor.apply_chat_template(
399
messages, tokenize=False, add_generation_prompt=True
400
)
401
402
+ image_data = [fetch_image(url) for url in image_urls]
403
+
404
return ModelRequestData(
405
engine_args=engine_args,
406
prompt=prompt,
404
- image_data=[fetch_image(url) for url in image_urls],
407
+ image_data=image_data,
408
)
409
410
408
-def load_llava(question: str, image_urls: list[str]) -> ModelRequestData:
409
- # NOTE: CAUTION! Original Llava models wasn't really trained on multi-image inputs,
410
- # it will generate poor response for multi-image inputs!
411
- model_name = "llava-hf/llava-1.5-7b-hf"
411
+def load_keye_vl1_5(question: str, image_urls: list[str]) -> ModelRequestData:
412
+ model_name = "Kwai-Keye/Keye-VL-1_5-8B"
413
+
414
engine_args = EngineArgs(
415
model=model_name,
414
- max_num_seqs=16,
416
+ trust_remote_code=True,
417
+ max_model_len=32768,
418
+ max_num_seqs=5,
419
limit_mm_per_prompt={"image": len(image_urls)},
420
)
421
@@ -423,28 +427,32 @@ def load_llava(question: str, image_urls: list[str]) -> ModelRequestData:
427
*placeholders,
428
{"type": "text", "text": question},
429
],
426
- }
430
+ },
431
]
432
429
- processor = AutoProcessor.from_pretrained(model_name)
433
+ processor = AutoProcessor.from_pretrained(model_name, trust_remote_code=True)
434
435
prompt = processor.apply_chat_template(
436
messages, tokenize=False, add_generation_prompt=True
437
)
438
439
+ image_data = [fetch_image(url) for url in image_urls]
440
+
441
return ModelRequestData(
442
engine_args=engine_args,
443
prompt=prompt,
438
- image_data=[fetch_image(url) for url in image_urls],
444
+ image_data=image_data,
445
)
446
447
442
-def load_llava_next(question: str, image_urls: list[str]) -> ModelRequestData:
443
- model_name = "llava-hf/llava-v1.6-mistral-7b-hf"
448
+def load_kimi_vl(question: str, image_urls: list[str]) -> ModelRequestData:
449
+ model_name = "moonshotai/Kimi-VL-A3B-Instruct"
450
+
451
engine_args = EngineArgs(
452
model=model_name,
446
- max_model_len=8192,
447
- max_num_seqs=16,
453
+ trust_remote_code=True,
454
+ max_model_len=4096,
455
+ max_num_seqs=4,
456
limit_mm_per_prompt={"image": len(image_urls)},
457
)
458
@@ -459,7 +467,7 @@ def load_llava_next(question: str, image_urls: list[str]) -> ModelRequestData:
467
}
468
]
469
462
- processor = AutoProcessor.from_pretrained(model_name)
470
+ processor = AutoProcessor.from_pretrained(model_name, trust_remote_code=True)
471
472
prompt = processor.apply_chat_template(
473
messages, tokenize=False, add_generation_prompt=True
@@ -472,12 +480,13 @@ def load_llava_next(question: str, image_urls: list[str]) -> ModelRequestData:
480
)
481
482
475
-def load_llava_onevision(question: str, image_urls: list[str]) -> ModelRequestData:
476
- model_name = "llava-hf/llava-onevision-qwen2-7b-ov-hf"
483
+def load_llama4(question: str, image_urls: list[str]) -> ModelRequestData:
484
+ model_name = "meta-llama/Llama-4-Scout-17B-16E-Instruct"
485
+
486
engine_args = EngineArgs(
487
model=model_name,
479
- max_model_len=16384,
480
- max_num_seqs=16,
488
+ max_model_len=131072,
489
+ tensor_parallel_size=8,
490
limit_mm_per_prompt={"image": len(image_urls)},
491
)
492
@@ -505,14 +514,13 @@ def load_llava_onevision(question: str, image_urls: list[str]) -> ModelRequestDa
514
)
515
516
508
-def load_keye_vl(question: str, image_urls: list[str]) -> ModelRequestData:
509
- model_name = "Kwai-Keye/Keye-VL-8B-Preview"
510
-
517
+def load_llava(question: str, image_urls: list[str]) -> ModelRequestData:
518
+ # NOTE: CAUTION! Original Llava models wasn't really trained on multi-image inputs,
519
+ # it will generate poor response for multi-image inputs!
520
+ model_name = "llava-hf/llava-1.5-7b-hf"
521
engine_args = EngineArgs(
522
model=model_name,
513
- trust_remote_code=True,
514
- max_model_len=8192,
515
- max_num_seqs=5,
523
+ max_num_seqs=16,
524
limit_mm_per_prompt={"image": len(image_urls)},
525
)
526
@@ -524,32 +532,28 @@ def load_keye_vl(question: str, image_urls: list[str]) -> ModelRequestData:
532
*placeholders,
533
{"type": "text", "text": question},
534
],
527
- },
535
+ }
536
]
537
530
- processor = AutoProcessor.from_pretrained(model_name, trust_remote_code=True)
538
+ processor = AutoProcessor.from_pretrained(model_name)
539
540
prompt = processor.apply_chat_template(
541
messages, tokenize=False, add_generation_prompt=True
542
)
543
536
- image_data = [fetch_image(url) for url in image_urls]
537
-
544
return ModelRequestData(
545
engine_args=engine_args,
546
prompt=prompt,
541
- image_data=image_data,
547
+ image_data=[fetch_image(url) for url in image_urls],
548
)
549
550
545
-def load_keye_vl1_5(question: str, image_urls: list[str]) -> ModelRequestData:
546
- model_name = "Kwai-Keye/Keye-VL-1_5-8B"
547
-
551
+def load_llava_next(question: str, image_urls: list[str]) -> ModelRequestData:
552
+ model_name = "llava-hf/llava-v1.6-mistral-7b-hf"
553
engine_args = EngineArgs(
554
model=model_name,
550
- trust_remote_code=True,
551
- max_model_len=32768,
552
- max_num_seqs=5,
555
+ max_model_len=8192,
556
+ max_num_seqs=16,
557
limit_mm_per_prompt={"image": len(image_urls)},
558
)
559
@@ -561,32 +565,28 @@ def load_keye_vl1_5(question: str, image_urls: list[str]) -> ModelRequestData:
565
*placeholders,
566
{"type": "text", "text": question},
567
],
564
- },
568
+ }
569
]
570
567
- processor = AutoProcessor.from_pretrained(model_name, trust_remote_code=True)
571
+ processor = AutoProcessor.from_pretrained(model_name)
572
573
prompt = processor.apply_chat_template(
574
messages, tokenize=False, add_generation_prompt=True
575
)
576
573
- image_data = [fetch_image(url) for url in image_urls]
574
-
577
return ModelRequestData(
578
engine_args=engine_args,
579
prompt=prompt,
578
- image_data=image_data,
580
+ image_data=[fetch_image(url) for url in image_urls],
581
)
582
583
582
-def load_kimi_vl(question: str, image_urls: list[str]) -> ModelRequestData:
583
- model_name = "moonshotai/Kimi-VL-A3B-Instruct"
584
-
584
+def load_llava_onevision(question: str, image_urls: list[str]) -> ModelRequestData:
585
+ model_name = "llava-hf/llava-onevision-qwen2-7b-ov-hf"
586
engine_args = EngineArgs(
587
model=model_name,
587
- trust_remote_code=True,
588
- max_model_len=4096,
589
- max_num_seqs=4,
588
+ max_model_len=16384,
589
+ max_num_seqs=16,
590
limit_mm_per_prompt={"image": len(image_urls)},
591
)
592
@@ -601,7 +601,7 @@ def load_kimi_vl(question: str, image_urls: list[str]) -> ModelRequestData:
601
}
602
]
603
604
- processor = AutoProcessor.from_pretrained(model_name, trust_remote_code=True)
604
+ processor = AutoProcessor.from_pretrained(model_name)
605
606
prompt = processor.apply_chat_template(
607
messages, tokenize=False, add_generation_prompt=True
vllm/model_executor/models/llava.py
+5
-15
@@ -57,7 +57,6 @@ from .siglip import SiglipVisionModel
57
from .utils import (
58
AutoWeightsLoader,
59
WeightsMapper,
60
- flatten_bn,
60
init_vllm_registered_model,
61
maybe_prefix,
62
)
@@ -507,6 +506,8 @@ def init_vision_tower_for_llava(
506
dummy_inputs=LlavaDummyInputsBuilder,
507
)
508
class LlavaForConditionalGeneration(nn.Module, SupportsMultiModal, SupportsPP):
509
+ merge_by_field_config = True
510
+
511
packed_modules_mapping = {
512
"qkv_proj": ["q_proj", "k_proj", "v_proj"],
513
"gate_up_proj": ["gate_proj", "up_proj"],
@@ -592,37 +593,26 @@ class LlavaForConditionalGeneration(nn.Module, SupportsMultiModal, SupportsPP):
593
return None
594
595
if pixel_values is not None:
595
- if not isinstance(pixel_values, (torch.Tensor, list)):
596
- raise ValueError(
597
- f"Incorrect type of pixel values. Got type: {type(pixel_values)}"
598
- )
599
-
596
if self.config.vision_config.model_type == "pixtral":
597
return PixtralHFImagePixelInputs(
598
type="pixel_values_pixtral",
603
- pixel_values=flatten_bn(pixel_values),
599
+ pixel_values=pixel_values,
600
)
601
602
expected_h = expected_w = self.config.vision_config.image_size
603
return LlavaImagePixelInputs(
604
type="pixel_values",
609
- pixel_values=flatten_bn(pixel_values, concat=True),
605
+ pixel_values=pixel_values,
606
resolve_bindings={"h": expected_h, "w": expected_w},
607
)
608
609
if image_embeds is not None:
614
- if not isinstance(image_embeds, (torch.Tensor, list)):
615
- raise ValueError(
616
- "Incorrect type of image embeddings. "
617
- f"Got type: {type(image_embeds)}"
618
- )
619
-
610
if self.config.vision_config.model_type == "pixtral":
611
raise ValueError("Pixtral-HF does not support image_embeds.")
612
613
return LlavaImageEmbeddingInputs(
614
type="image_embeds",
625
- data=flatten_bn(image_embeds, concat=True),
615
+ data=image_embeds,
616
)
617
618
raise AssertionError("This line should be unreachable.")
vllm/model_executor/models/llava_next.py
+5
-19
@@ -34,7 +34,6 @@ from .siglip import SiglipVisionModel
34
from .utils import (
35
AutoWeightsLoader,
36
WeightsMapper,
37
- flatten_bn,
37
init_vllm_registered_model,
38
maybe_prefix,
39
)
@@ -222,6 +221,8 @@ class LlavaNextMultiModalProcessor(
221
dummy_inputs=LlavaDummyInputsBuilder,
222
)
223
class LlavaNextForConditionalGeneration(nn.Module, SupportsMultiModal, SupportsPP):
224
+ merge_by_field_config = True
225
+
226
hf_to_vllm_mapper = WeightsMapper(
227
orig_to_new_prefix={
228
# mapping for new names in checkpoint saved after transformers v4.52
@@ -302,21 +303,11 @@ class LlavaNextForConditionalGeneration(nn.Module, SupportsMultiModal, SupportsP
303
return None
304
305
if pixel_values is not None:
305
- if not isinstance(pixel_values, (torch.Tensor, list)):
306
- raise ValueError(
307
- f"Incorrect type of pixel values. Got type: {type(pixel_values)}"
308
- )
309
-
310
- if not isinstance(image_sizes, (torch.Tensor, list)):
311
- raise ValueError(
312
- f"Incorrect type of image sizes. Got type: {type(image_sizes)}"
313
- )
314
-
306
expected_h = expected_w = self.config.vision_config.image_size
307
return LlavaNextImagePixelInputs(
308
type="pixel_values",
318
- pixel_values=flatten_bn(pixel_values),
319
- image_sizes=flatten_bn(image_sizes, concat=True),
309
+ pixel_values=pixel_values,
310
+ image_sizes=image_sizes,
311
resolve_bindings={
312
"h": expected_h,
313
"w": expected_w,
@@ -324,14 +315,9 @@ class LlavaNextForConditionalGeneration(nn.Module, SupportsMultiModal, SupportsP
315
)
316
317
if image_embeds is not None:
327
- if not isinstance(image_embeds, torch.Tensor):
328
- raise ValueError(
329
- f"Incorrect type of image embeds. Got type: {type(image_embeds)}"
330
- )
331
-
318
return LlavaNextImageEmbeddingInputs(
319
type="image_embeds",
334
- data=flatten_bn(image_embeds),
320
+ data=image_embeds,
321
)
322
323
raise AssertionError("This line should be unreachable.")
vllm/model_executor/models/llava_next_video.py
+13
-16
@@ -51,14 +51,13 @@ from .vision import get_vision_encoder_info
51
class LlavaNextVideoPixelInputs(TensorSchema):
52
"""
53
Dimensions:
54
- - bs: Batch size
55
- - nv: Number of videos
56
- - nf: Number of frames
57
- - nc: Number of channels (3)
54
+ - bn: Batch size * number of videos
55
+ - f: Number of frames
56
+ - c: Number of channels (3)
57
- h: Height of each frame
58
- w: Width of each frame
59
61
- Note that `num_frames` may be different for each batch, in which case
60
+ Note that `f` may be different for each batch, in which case
61
the data is passed as a list instead of a batched tensor.
62
63
Note that it only supports one video input for one batch.
@@ -66,9 +65,9 @@ class LlavaNextVideoPixelInputs(TensorSchema):
65
66
type: Literal["pixel_values_videos"] = "pixel_values_videos"
67
69
- data: Annotated[
68
+ pixel_values_videos: Annotated[
69
Union[torch.Tensor, list[torch.Tensor]],
71
- TensorShape("bs", "nv", "nf", 3, "h", "w"),
70
+ TensorShape("bn", "f", 3, "h", "w", dynamic_dims={"f"}),
71
]
72
73
@@ -300,6 +299,8 @@ class LlavaNextMultiModalProjector(nn.Module):
299
dummy_inputs=LlavaNextVideoDummyInputsBuilder,
300
)
301
class LlavaNextVideoForConditionalGeneration(nn.Module, SupportsMultiModal, SupportsPP):
302
+ merge_by_field_config = True
303
+
304
hf_to_vllm_mapper = WeightsMapper(
305
orig_to_new_prefix={
306
# mapping for new names in checkpoint saved after transformers v4.52
@@ -371,7 +372,7 @@ class LlavaNextVideoForConditionalGeneration(nn.Module, SupportsMultiModal, Supp
372
expected_h = expected_w = self.config.vision_config.image_size
373
return LlavaNextVideoPixelInputs(
374
type="pixel_values_videos",
374
- data=pixel_values_videos,
375
+ pixel_values_videos=pixel_values_videos,
376
resolve_bindings={
377
"h": expected_h,
378
"w": expected_w,
@@ -396,19 +397,15 @@ class LlavaNextVideoForConditionalGeneration(nn.Module, SupportsMultiModal, Supp
397
def _process_video_pixels(self, inputs: LlavaNextVideoPixelInputs):
398
assert self.vision_tower is not None
399
399
- video_pixels = inputs["data"]
400
+ video_pixels = inputs["pixel_values_videos"]
401
402
if isinstance(video_pixels, torch.Tensor):
402
- # TODO: support multiple videos per input
403
- b, num_videos, num_frames, c, h, w = video_pixels.shape
404
- assert num_videos == 1
405
- stacked_pixels = video_pixels.view(b * num_videos * num_frames, c, h, w)
403
+ bn, f, c, h, w = video_pixels.shape
404
+ stacked_pixels = video_pixels.view(bn * f, c, h, w)
405
stacked_embeddings = self._video_pixels_to_features(
406
self.vision_tower, stacked_pixels
407
)
409
- embeds = stacked_embeddings.view(
410
- b, num_frames, *stacked_embeddings.shape[1:]
411
- )
408
+ embeds = stacked_embeddings.view(bn, f, *stacked_embeddings.shape[1:])
409
410
elif is_list_of(video_pixels, torch.Tensor):
411
frames_per_videos = [v.shape[0] for v in video_pixels]
vllm/model_executor/models/llava_onevision.py
+7
-27
@@ -44,7 +44,6 @@ from .siglip import SiglipVisionModel
44
from .utils import (
45
AutoWeightsLoader,
46
WeightsMapper,
47
- flatten_bn,
47
init_vllm_registered_model,
48
maybe_prefix,
49
)
@@ -62,7 +61,7 @@ class LlavaOnevisionVideoPixelInputs(TensorSchema):
61
- h: Height
62
- w: Width
63
65
- Note that `num_videos` may be different for each batch, and 'num_frames'
64
+ Note that `f` may be different for each batch, and 'num_frames'
65
may be different for each video, in which case the data is passed as a
66
list instead of a batched tensor.
67
"""
@@ -480,6 +479,8 @@ class LlavaOnevisionMultiModalProjector(nn.Module):
479
dummy_inputs=LlavaOnevisionDummyInputsBuilder,
480
)
481
class LlavaOnevisionForConditionalGeneration(nn.Module, SupportsMultiModal, SupportsPP):
482
+ merge_by_field_config = True
483
+
484
hf_to_vllm_mapper = WeightsMapper(
485
orig_to_new_prefix={
486
# mapping for new names in checkpoint saved after transformers v4.52
@@ -539,20 +540,10 @@ class LlavaOnevisionForConditionalGeneration(nn.Module, SupportsMultiModal, Supp
540
return None
541
542
if pixel_values is not None:
542
- if not isinstance(pixel_values, (torch.Tensor, list)):
543
- raise ValueError(
544
- f"Incorrect type of pixel values. Got type: {type(pixel_values)}"
545
- )
546
-
547
- if not isinstance(image_sizes, (torch.Tensor, list)):
548
- raise ValueError(
549
- f"Incorrect type of image sizes. Got type: {type(image_sizes)}"
550
- )
551
-
543
return LlavaOnevisionImagePixelInputs(
544
type="pixel_values",
554
- pixel_values=flatten_bn(pixel_values),
555
- image_sizes=flatten_bn(image_sizes, concat=True),
545
+ pixel_values=pixel_values,
546
+ image_sizes=image_sizes,
547
resolve_bindings={
548
"h": self.config.vision_config.image_size,
549
"w": self.config.vision_config.image_size,
@@ -560,14 +551,9 @@ class LlavaOnevisionForConditionalGeneration(nn.Module, SupportsMultiModal, Supp
551
)
552
553
if image_embeds is not None:
563
- if not isinstance(image_embeds, torch.Tensor):
564
- raise ValueError(
565
- f"Incorrect type of image embeds. Got type: {type(image_embeds)}"
566
- )
567
-
554
return LlavaOnevisionImageEmbeddingInputs(
555
type="image_embeds",
570
- data=flatten_bn(image_embeds),
556
+ data=image_embeds,
557
)
558
559
raise AssertionError("This line should be unreachable.")
@@ -586,15 +572,9 @@ class LlavaOnevisionForConditionalGeneration(nn.Module, SupportsMultiModal, Supp
572
if pixel_values_videos is None:
573
return None
574
589
- if not isinstance(pixel_values_videos, (torch.Tensor, list)):
590
- raise ValueError(
591
- "Incorrect type of pixel_values_videos. "
592
- f"Got type: {type(pixel_values_videos)}"
593
- )
594
-
575
return LlavaOnevisionVideoPixelInputs(
576
type="pixel_values_videos",
597
- pixel_values_videos=flatten_bn(pixel_values_videos),
577
+ pixel_values_videos=pixel_values_videos,
578
resolve_bindings={
579
"h": self.config.vision_config.image_size,
580
"w": self.config.vision_config.image_size,
vllm/model_executor/models/minimax_vl_01.py
+5
-20
@@ -32,7 +32,6 @@ from .pixtral import PixtralHFVisionModel
32
from .siglip import SiglipVisionModel
33
from .utils import (
34
AutoWeightsLoader,
35
- flatten_bn,
35
init_vllm_registered_model,
36
maybe_prefix,
37
)
@@ -180,6 +179,8 @@ class MiniMaxVL01MultiModalProcessor(
179
dummy_inputs=MiniMaxVL01DummyInputsBuilder,
180
)
181
class MiniMaxVL01ForConditionalGeneration(nn.Module, SupportsMultiModal, SupportsPP):
182
+ merge_by_field_config = True
183
+
184
packed_modules_mapping = {
185
"qkv_proj": ["q_proj", "k_proj", "v_proj"],
186
"gate_up_proj": ["gate_proj", "up_proj"],
@@ -338,32 +339,16 @@ class MiniMaxVL01ForConditionalGeneration(nn.Module, SupportsMultiModal, Support
339
return None
340
341
if pixel_values is not None and image_sizes is not None:
341
- if not isinstance(pixel_values, (torch.Tensor, list)):
342
- raise ValueError(
343
- f"Incorrect type of pixel values. Got type: {type(pixel_values)}"
344
- )
345
-
346
- if not isinstance(image_sizes, (torch.Tensor, list)):
347
- raise ValueError(
348
- f"Incorrect type of image sizes. Got type: {type(image_sizes)}"
349
- )
350
-
342
return MiniMaxVL01ImagePixelInputs(
343
type="pixel_values",
353
- pixel_values=flatten_bn(pixel_values),
354
- image_sizes=flatten_bn(image_sizes, concat=True),
344
+ pixel_values=pixel_values,
345
+ image_sizes=image_sizes,
346
)
347
348
if image_embeds is not None:
358
- if not isinstance(image_embeds, (torch.Tensor, list)):
359
- raise ValueError(
360
- "Incorrect type of image embeddings. "
361
- f"Got type: {type(image_embeds)}"
362
- )
363
-
349
return MiniMaxVL01ImageEmbeddingInputs(
350
type="image_embeds",
366
- data=flatten_bn(image_embeds, concat=True),
351
+ data=image_embeds,
352
)
353
354
raise AssertionError("This line should be unreachable.")
vllm/model_executor/models/mistral3.py
+3
-8
@@ -52,7 +52,6 @@ from .pixtral import PixtralHFEncoderInfo, PixtralHFVisionModel
52
from .utils import (
53
AutoWeightsLoader,
54
WeightsMapper,
55
- flatten_bn,
55
init_vllm_registered_model,
56
maybe_prefix,
57
)
@@ -424,6 +423,8 @@ def init_vision_tower_for_llava(
423
class Mistral3ForConditionalGeneration(
424
nn.Module, SupportsLoRA, SupportsMultiModal, SupportsPP
425
):
426
+ merge_by_field_config = True
427
+
428
packed_modules_mapping = {
429
"qkv_proj": ["q_proj", "k_proj", "v_proj"],
430
"gate_up_proj": ["gate_proj", "up_proj"],
@@ -510,15 +511,9 @@ class Mistral3ForConditionalGeneration(
511
if pixel_values is None and image_embeds is None:
512
return None
513
513
- assert pixel_values is not None
514
- if not isinstance(pixel_values, (torch.Tensor, list)):
515
- raise ValueError(
516
- f"Incorrect type of pixel values. Got type: {type(pixel_values)}"
517
- )
518
-
514
return Mistral3ImagePixelInputs(
515
type="pixel_values_pixtral",
521
- pixel_values=flatten_bn(pixel_values),
516
+ pixel_values=pixel_values,
517
)
518
519
def _process_image_input(
vllm/model_executor/models/pixtral.py
+4
-2
@@ -64,7 +64,7 @@ from vllm.transformers_utils.tokenizer import (
64
from vllm.utils.tensor_schema import TensorSchema, TensorShape
65
66
from .interfaces import MultiModalEmbeddings, SupportsMultiModal, SupportsPP
67
-from .utils import flatten_bn, init_vllm_registered_model, maybe_prefix
67
+from .utils import init_vllm_registered_model, maybe_prefix
68
from .vision import (
69
VisionEncoderInfo,
70
VisionFeatureSelectStrategy,
@@ -365,6 +365,8 @@ class PixtralMultiModalProcessor(BaseMultiModalProcessor[PixtralProcessingInfo])
365
dummy_inputs=PixtralDummyInputsBuilder,
366
)
367
class PixtralForConditionalGeneration(nn.Module, SupportsMultiModal, SupportsPP):
368
+ merge_by_field_config = True
369
+
370
@classmethod
371
def get_placeholder_str(cls, modality: str, i: int) -> Optional[str]:
372
if modality.startswith("image"):
@@ -424,7 +426,7 @@ class PixtralForConditionalGeneration(nn.Module, SupportsMultiModal, SupportsPP)
426
427
return PixtralImagePixelInputs(
428
type="pixel_values",
427
- images=flatten_bn(images),
429
+ images=images,
430
)
431
432
def _process_image_input(
vllm/model_executor/models/tarsier.py
+4
-13
@@ -49,7 +49,6 @@ from .interfaces import MultiModalEmbeddings, SupportsMultiModal, SupportsPP
49
from .siglip import SiglipVisionModel
50
from .utils import (
51
AutoWeightsLoader,
52
- flatten_bn,
52
init_vllm_registered_model,
53
maybe_prefix,
54
)
@@ -404,6 +403,8 @@ def init_vision_tower_for_tarsier(
403
dummy_inputs=TarsierDummyInputsBuilder,
404
)
405
class TarsierForConditionalGeneration(nn.Module, SupportsMultiModal, SupportsPP):
406
+ merge_by_field_config = True
407
+
408
packed_modules_mapping = {
409
"qkv_proj": ["q_proj", "k_proj", "v_proj"],
410
"gate_up_proj": ["gate_proj", "up_proj"],
@@ -467,25 +468,15 @@ class TarsierForConditionalGeneration(nn.Module, SupportsMultiModal, SupportsPP)
468
return None
469
470
if pixel_values is not None:
470
- if not isinstance(pixel_values, (torch.Tensor, list)):
471
- raise ValueError(
472
- f"Incorrect type of pixel values. Got type: {type(pixel_values)}"
473
- )
474
-
471
return TarsierImagePixelInputs(
472
type="pixel_values",
477
- pixel_values=flatten_bn(pixel_values, concat=True),
473
+ pixel_values=pixel_values,
474
)
475
476
if image_embeds is not None:
481
- if not isinstance(image_embeds, (torch.Tensor, list)):
482
- raise ValueError(
483
- "Incorrect type of image embeddings. "
484
- f"Got type: {type(image_embeds)}"
485
- )
477
return TarsierImageEmbeddingInputs(
478
type="image_embeds",
488
- data=flatten_bn(image_embeds, concat=True),
479
+ data=image_embeds,
480
)
481
482
raise AssertionError("This line should be unreachable.")