A high-throughput and memory-efficient inference and serving engine for LLMs
Mirrored from https://github.com/tduncan-lum/Nvidia-Optimization-Stack---VLLM.git
Synced Nov 14, 05:30 UTC
[CI/Build] Update VLM common tests (#22841)
Signed-off-by: DarkLight1337 <tlleungac@connect.ust.hk>
Cyrus Leung committed
Aug 14, 2025 at 01:03 UTC
c9232d41f433abd1d6f0960bcec020660078d718
2 files changed
+2
-33
tests/models/multimodal/generation/test_common.py
+1
-15
@@ -561,7 +561,7 @@ VLM_TEST_SETTINGS = {
561
get_stop_token_ids=lambda tok: tok.convert_tokens_to_ids(['<|im_end|>', '<|endoftext|>']), # noqa: E501
562
hf_output_post_proc=model_utils.minicpmv_trunc_hf_output,
563
patch_hf_runner=model_utils.minicpmo_26_patch_hf_runner,
564
- # FIXME: https://huggingface.co/openbmb/MiniCPM-V-2_6/discussions/55
564
+ # FIXME: https://huggingface.co/openbmb/MiniCPM-o-2_6/discussions/49
565
marks=[pytest.mark.skip("HF import fails")],
566
),
567
"minicpmv_26": VLMTestInfo(
@@ -574,8 +574,6 @@ VLM_TEST_SETTINGS = {
574
get_stop_token_ids=lambda tok: tok.convert_tokens_to_ids(['<|im_end|>', '<|endoftext|>']), # noqa: E501
575
hf_output_post_proc=model_utils.minicpmv_trunc_hf_output,
576
patch_hf_runner=model_utils.minicpmv_26_patch_hf_runner,
577
- # FIXME: https://huggingface.co/openbmb/MiniCPM-V-2_6/discussions/55
578
- marks=[pytest.mark.skip("HF import fails")],
577
),
578
"minimax_vl_01": VLMTestInfo(
579
models=["MiniMaxAI/MiniMax-VL-01"],
@@ -611,18 +609,6 @@ VLM_TEST_SETTINGS = {
609
patch_hf_runner=model_utils.ovis_patch_hf_runner,
610
marks=[large_gpu_mark(min_gb=32)],
611
),
614
- "ovis1_6": VLMTestInfo(
615
- models=["AIDC-AI/Ovis1.6-Llama3.2-3B"],
616
- test_type=(VLMTestType.IMAGE, VLMTestType.MULTI_IMAGE),
617
- prompt_formatter=lambda img_prompt: f"<|begin_of_text|><|start_header_id|>system<|end_header_id|>\n\nYou are a helpful and honest multimodal assistant.<|eot_id|><|start_header_id|>user<|end_header_id|>\n\n{img_prompt}<|eot_id|><|start_header_id|>assistant<|end_header_id|>\n\n", # noqa: E501
618
- img_idx_to_prompt=lambda idx: "<image>\n", # noqa: E501
619
- max_model_len=4096,
620
- max_num_seqs=2,
621
- dtype="half",
622
- # use sdpa mode for hf runner since ovis2 didn't work with flash_attn
623
- hf_model_kwargs={"llm_attn_implementation": "sdpa"},
624
- patch_hf_runner=model_utils.ovis_patch_hf_runner,
625
- ),
612
"ovis2": VLMTestInfo(
613
models=["AIDC-AI/Ovis2-1B"],
614
test_type=(VLMTestType.IMAGE, VLMTestType.MULTI_IMAGE),
vllm/model_executor/models/minicpmv.py
+1
-18
@@ -85,30 +85,13 @@ class MiniCPMVImagePixelInputs(TensorSchema):
85
- w: Width
86
"""
87
88
- def _validate_nested_tensors(
89
- self,
90
- value: Union[list[torch.Tensor], tuple[torch.Tensor, ...]],
91
- field_name: str,
92
- expected_shape: tuple[Union[int, str], ...],
93
- dynamic_dims: set[str],
94
- ) -> tuple[int, ...]:
95
- # value[0] is the scaled image,
96
- # and value[1:] is a collection of image slices.
97
- # It is ensured that all slices in the collection
98
- # have the same shape.
99
- if field_name == "pixel_values":
100
- value = value[1:] if len(value) > 1 else value
101
-
102
- return super()._validate_nested_tensors(value, field_name,
103
- expected_shape, dynamic_dims)
104
-
88
type: Literal["pixel_values"] = "pixel_values"
89
90
# Note that the image size may vary, so we pass it as a list instead of a
91
# batched tensor.
92
pixel_values: Annotated[
93
list[torch.Tensor],
111
- TensorShape("bns", "c", "h", "w"),
94
+ TensorShape("bns", "c", "h", "w", dynamic_dims={"h", "w"}),
95
]
96
tgt_sizes: Annotated[
97
torch.Tensor,