A high-throughput and memory-efficient inference and serving engine for LLMs
Mirrored from https://github.com/tduncan-lum/Nvidia-Optimization-Stack---VLLM.git
Synced Nov 2, 08:30 UTC
[ROCm][CI] Fix failure in the Phi3V pooling test (#44028)
Signed-off-by: Andreas Karatzas <akaratza@amd.com>
Andreas Karatzas committed
May 29, 2026 at 23:14 UTC
ef8840adc73bfbe3108811cebcd8af7252f9b6f0
1 file changed
+59
-13
tests/models/multimodal/pooling/test_phi3v.py
+59
-13
@@ -8,6 +8,8 @@ from PIL import Image
8
9
from vllm.assets.base import get_vllm_public_assets
10
from vllm.assets.image import VLM_IMAGES_DIR
11
+from vllm.config import ModelConfig
12
+from vllm.multimodal import MULTIMODAL_REGISTRY
13
14
from ....conftest import IMAGE_ASSETS, HfRunner, PromptImageInput, VllmRunner
15
from ....utils import large_gpu_test
@@ -37,6 +39,18 @@ HF_IMAGE_PROMPTS = IMAGE_ASSETS.prompts(
39
40
MODELS = ["TIGER-Lab/VLM2Vec-Full"]
41
42
+SPECIAL_TOKEN_IMAGE_PROMPT = (
43
+ "\n<s><|user|>\n <|image_1|>\n\t <s>"
44
+ "Represent the given image for classification<|end|>"
45
+ "\n<|assistant|>\n"
46
+)
47
+
48
+
49
+def _get_cherry_blossom_image() -> Image.Image:
50
+ return Image.open(
51
+ get_vllm_public_assets(filename="cherry_blossom.jpg", s3_prefix=VLM_IMAGES_DIR)
52
+ )
53
+
54
55
def _run_test(
56
hf_runner: type[HfRunner],
@@ -123,19 +137,6 @@ def test_models_image(
137
input_texts_images = [
138
(text, asset.pil_image) for text, asset in zip(HF_IMAGE_PROMPTS, image_assets)
139
]
126
- # add cases for special_tokens
127
- input_texts_images.append(
128
- (
129
- "\n<s><|user|>\n <|image_1|>\n\t <s>"
130
- "Represent the given image for classification<|end|>"
131
- "\n<|assistant|>\n",
132
- Image.open(
133
- get_vllm_public_assets(
134
- filename="cherry_blossom.jpg", s3_prefix=VLM_IMAGES_DIR
135
- )
136
- ),
137
- )
138
- )
140
input_texts = [text for text, _ in input_texts_images]
141
input_images = [image for _, image in input_texts_images]
142
@@ -147,3 +148,48 @@ def test_models_image(
148
model,
149
dtype=dtype,
150
)
151
+
152
+
153
+@pytest.mark.core_model
154
+@pytest.mark.parametrize("model", MODELS)
155
+@pytest.mark.parametrize("dtype", ["half"])
156
+def test_models_image_special_tokens_processing(
157
+ model: str,
158
+ dtype: str,
159
+) -> None:
160
+ model_config = ModelConfig(
161
+ model,
162
+ runner="pooling",
163
+ trust_remote_code=True,
164
+ dtype=dtype,
165
+ max_model_len=1024,
166
+ )
167
+ processor = MULTIMODAL_REGISTRY.create_processor(model_config)
168
+ image = _get_cherry_blossom_image()
169
+
170
+ processed_inputs = processor(
171
+ SPECIAL_TOKEN_IMAGE_PROMPT,
172
+ mm_items=processor.info.parse_mm_data({"image": image}),
173
+ hf_processor_mm_kwargs={},
174
+ )
175
+
176
+ hf_processor = processor.info.get_hf_processor()
177
+ hf_inputs = hf_processor(
178
+ SPECIAL_TOKEN_IMAGE_PROMPT,
179
+ images=image,
180
+ return_tensors="pt",
181
+ )
182
+
183
+ image_token_id = hf_processor.get_special_image_token_id()
184
+ hf_prompt_token_ids = [
185
+ image_token_id if token_id < 0 else token_id
186
+ for token_id in hf_inputs["input_ids"][0].tolist()
187
+ ]
188
+
189
+ prompt_token_ids = processed_inputs["prompt_token_ids"]
190
+
191
+ assert prompt_token_ids == hf_prompt_token_ids
192
+ assert prompt_token_ids.count(image_token_id) == hf_prompt_token_ids.count(
193
+ image_token_id
194
+ )
195
+ assert prompt_token_ids.count(image_token_id) > 0