A high-throughput and memory-efficient inference and serving engine for LLMs
Mirrored from https://github.com/tduncan-lum/Nvidia-Optimization-Stack---VLLM.git
Synced Nov 15, 17:15 UTC
fix test_phi3v (#15321)
Signed-off-by: pansicheng <sicheng.pan.chn@gmail.com>
pansicheng committed
Mar 30, 2025 at 17:01 UTC
7fd8c0f85ce3cb7062dcaecfe05d087462362148
4 files changed
+110
-14
tests/entrypoints/openai/test_vision.py
+40
-6
@@ -3,6 +3,9 @@
3
import openai
4
import pytest
5
import pytest_asyncio
6
+import requests
7
+from PIL import Image
8
+from transformers import AutoProcessor
9
10
from vllm.multimodal.utils import encode_image_base64, fetch_image
11
@@ -53,11 +56,31 @@ def base64_encoded_image() -> dict[str, str]:
56
}
57
58
59
+def get_hf_prompt_tokens(model_name, content, image_url):
60
+ processor = AutoProcessor.from_pretrained(model_name,
61
+ trust_remote_code=True,
62
+ num_crops=4)
63
+
64
+ placeholder = "<|image_1|>\n"
65
+ messages = [{
66
+ "role": "user",
67
+ "content": f"{placeholder}{content}",
68
+ }]
69
+ images = [Image.open(requests.get(image_url, stream=True).raw)]
70
+
71
+ prompt = processor.tokenizer.apply_chat_template(
72
+ messages, tokenize=False, add_generation_prompt=True)
73
+ inputs = processor(prompt, images, return_tensors="pt")
74
+
75
+ return inputs.input_ids.shape[1]
76
+
77
+
78
@pytest.mark.asyncio
79
@pytest.mark.parametrize("model_name", [MODEL_NAME])
80
@pytest.mark.parametrize("image_url", TEST_IMAGE_URLS)
81
async def test_single_chat_session_image(client: openai.AsyncOpenAI,
82
model_name: str, image_url: str):
83
+ content_text = "What's in this image?"
84
messages = [{
85
"role":
86
"user",
@@ -70,16 +93,17 @@ async def test_single_chat_session_image(client: openai.AsyncOpenAI,
93
},
94
{
95
"type": "text",
73
- "text": "What's in this image?"
96
+ "text": content_text
97
},
98
],
99
}]
100
101
+ max_completion_tokens = 10
102
# test single completion
103
chat_completion = await client.chat.completions.create(
104
model=model_name,
105
messages=messages,
82
- max_completion_tokens=10,
106
+ max_completion_tokens=max_completion_tokens,
107
logprobs=True,
108
temperature=0.0,
109
top_logprobs=5)
@@ -87,8 +111,12 @@ async def test_single_chat_session_image(client: openai.AsyncOpenAI,
111
112
choice = chat_completion.choices[0]
113
assert choice.finish_reason == "length"
114
+ hf_prompt_tokens = get_hf_prompt_tokens(model_name, content_text,
115
+ image_url)
116
assert chat_completion.usage == openai.types.CompletionUsage(
91
- completion_tokens=10, prompt_tokens=774, total_tokens=784)
117
+ completion_tokens=max_completion_tokens,
118
+ prompt_tokens=hf_prompt_tokens,
119
+ total_tokens=hf_prompt_tokens + max_completion_tokens)
120
121
message = choice.message
122
message = chat_completion.choices[0].message
@@ -150,6 +178,7 @@ async def test_single_chat_session_image_base64encoded(
178
client: openai.AsyncOpenAI, model_name: str, image_url: str,
179
base64_encoded_image: dict[str, str]):
180
181
+ content_text = "What's in this image?"
182
messages = [{
183
"role":
184
"user",
@@ -163,16 +192,17 @@ async def test_single_chat_session_image_base64encoded(
192
},
193
{
194
"type": "text",
166
- "text": "What's in this image?"
195
+ "text": content_text
196
},
197
],
198
}]
199
200
+ max_completion_tokens = 10
201
# test single completion
202
chat_completion = await client.chat.completions.create(
203
model=model_name,
204
messages=messages,
175
- max_completion_tokens=10,
205
+ max_completion_tokens=max_completion_tokens,
206
logprobs=True,
207
temperature=0.0,
208
top_logprobs=5)
@@ -180,8 +210,12 @@ async def test_single_chat_session_image_base64encoded(
210
211
choice = chat_completion.choices[0]
212
assert choice.finish_reason == "length"
213
+ hf_prompt_tokens = get_hf_prompt_tokens(model_name, content_text,
214
+ image_url)
215
assert chat_completion.usage == openai.types.CompletionUsage(
184
- completion_tokens=10, prompt_tokens=774, total_tokens=784)
216
+ completion_tokens=max_completion_tokens,
217
+ prompt_tokens=hf_prompt_tokens,
218
+ total_tokens=hf_prompt_tokens + max_completion_tokens)
219
220
message = choice.message
221
message = chat_completion.choices[0].message
tests/entrypoints/openai/test_vision_embedding.py
+21
-3
@@ -2,6 +2,8 @@
2
3
import pytest
4
import requests
5
+from PIL import Image
6
+from transformers import AutoProcessor
7
8
from vllm.entrypoints.openai.protocol import EmbeddingResponse
9
from vllm.multimodal.utils import encode_image_base64, fetch_image
@@ -52,11 +54,24 @@ def base64_encoded_image() -> dict[str, str]:
54
}
55
56
57
+def get_hf_prompt_tokens(model_name, content, image_url):
58
+ processor = AutoProcessor.from_pretrained(model_name,
59
+ trust_remote_code=True,
60
+ num_crops=4)
61
+
62
+ placeholder = "<|image_1|> "
63
+ prompt = f"{placeholder}{content}"
64
+ images = [Image.open(requests.get(image_url, stream=True).raw)]
65
+ inputs = processor(prompt, images, return_tensors="pt")
66
+ return inputs.input_ids.shape[1]
67
+
68
+
69
@pytest.mark.asyncio
70
@pytest.mark.parametrize("model_name", [MODEL_NAME])
71
@pytest.mark.parametrize("image_url", TEST_IMAGE_URLS)
72
async def test_image_embedding(server: RemoteOpenAIServer, model_name: str,
73
image_url: str):
74
+ content_text = "Represent the given image."
75
messages = [{
76
"role":
77
"user",
@@ -69,7 +84,7 @@ async def test_image_embedding(server: RemoteOpenAIServer, model_name: str,
84
},
85
{
86
"type": "text",
72
- "text": "Represent the given image."
87
+ "text": content_text
88
},
89
],
90
}]
@@ -85,9 +100,12 @@ async def test_image_embedding(server: RemoteOpenAIServer, model_name: str,
100
response.raise_for_status()
101
embeddings = EmbeddingResponse.model_validate(response.json())
102
103
+ hf_prompt_tokens = get_hf_prompt_tokens(model_name, content_text,
104
+ image_url)
105
+
106
assert embeddings.id is not None
107
assert len(embeddings.data) == 1
108
assert len(embeddings.data[0].embedding) == 3072
109
assert embeddings.usage.completion_tokens == 0
92
- assert embeddings.usage.prompt_tokens == 763
93
- assert embeddings.usage.total_tokens == 763
110
+ assert embeddings.usage.prompt_tokens == hf_prompt_tokens
111
+ assert embeddings.usage.total_tokens == hf_prompt_tokens
tests/models/embedding/vision_language/test_phi3v.py
+13
@@ -2,6 +2,10 @@
2
3
import pytest
4
import torch.nn.functional as F
5
+from PIL import Image
6
+
7
+from vllm.assets.base import get_vllm_public_assets
8
+from vllm.assets.image import VLM_IMAGES_DIR
9
10
from ....conftest import IMAGE_ASSETS, HfRunner, PromptImageInput, VllmRunner
11
from ....utils import large_gpu_test
@@ -112,6 +116,15 @@ def test_models_image(
116
(text, asset.pil_image)
117
for text, asset in zip(HF_IMAGE_PROMPTS, image_assets)
118
]
119
+ # add cases for special_tokens
120
+ input_texts_images.append((
121
+ "\n<s><|user|>\n <|image_1|>\n\t <s>"
122
+ "Represent the given image for classification<|end|>"
123
+ "\n<|assistant|>\n",
124
+ Image.open(
125
+ get_vllm_public_assets(filename="cherry_blossom.jpg",
126
+ s3_prefix=VLM_IMAGES_DIR)),
127
+ ))
128
input_texts = [text for text, _ in input_texts_images]
129
input_images = [image for _, image in input_texts_images]
130
vllm/model_executor/models/phi3v.py
+36
-5
@@ -14,6 +14,7 @@
14
# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
15
# See the License for the specific language governing permissions and
16
# limitations under the License.
17
+import re
18
from collections.abc import Iterable, Mapping, Sequence
19
from functools import cached_property
20
from typing import Any, List, Literal, Optional, Set, Tuple, TypedDict, Union
@@ -428,10 +429,6 @@ class Phi3VMultiModalProcessor(BaseMultiModalProcessor[Phi3VProcessingInfo]):
429
hf_processor = self.info.get_hf_processor(**hf_processor_mm_kwargs)
430
image_tokens: list[str] = hf_processor.img_tokens # type: ignore
431
431
- tokenizer = self.info.get_tokenizer()
432
- bos_token_id = tokenizer.bos_token_id
433
- assert isinstance(bos_token_id, int)
434
-
432
def get_replacement_phi3v(item_idx: int):
433
images = mm_items.get_items(
434
"image", (ImageEmbeddingItems, ImageProcessorItems))
@@ -449,7 +446,7 @@ class Phi3VMultiModalProcessor(BaseMultiModalProcessor[Phi3VProcessingInfo]):
446
image_tokens = [_IMAGE_TOKEN_ID] * num_image_tokens
447
448
return PromptUpdateDetails(
452
- full=image_tokens + [bos_token_id],
449
+ full=image_tokens,
450
features=image_tokens,
451
)
452
@@ -469,6 +466,40 @@ class Phi3VMultiModalProcessor(BaseMultiModalProcessor[Phi3VProcessingInfo]):
466
mm_prompt_updates: Mapping[str, Sequence[BoundPromptUpdate]],
467
mm_item_counts: Mapping[str, int],
468
) -> tuple[list[int], str, Mapping[str, list[PlaceholderFeaturesInfo]]]:
469
+ # align to hf behavior when there are images
470
+ if len(mm_item_counts):
471
+ tokenizer = self.info.get_tokenizer()
472
+ # to decode token_ids to the original text, we need to
473
+ # 1. remove the first bos token
474
+ # 2. remove space after each special token
475
+ # introduced by the tokenizer
476
+ if len(token_ids) and token_ids[0] == tokenizer.bos_token_id:
477
+ token_ids = token_ids[1:]
478
+ text = tokenizer.decode(token_ids)
479
+ for special_tokens in tokenizer.special_tokens_map.values():
480
+ if isinstance(special_tokens, str):
481
+ text = text.replace(f"{special_tokens} ", special_tokens)
482
+ elif isinstance(special_tokens, list):
483
+ for special_token in special_tokens:
484
+ text = text.replace(f"{special_token} ", special_token)
485
+ # perform hf behavior
486
+ # https://huggingface.co/microsoft/Phi-3.5-vision-instruct/blob/64f88b6/processing_phi3_v.py#L407
487
+ pattern = r"<\|image_\d+\|>"
488
+ prompt_chunks = [
489
+ tokenizer(chunk).input_ids
490
+ for chunk in re.split(pattern, text)
491
+ ]
492
+ image_tags = [
493
+ tokenizer(chunk, add_special_tokens=False).input_ids
494
+ for chunk in re.findall(pattern, text)
495
+ ]
496
+ if len(prompt_chunks) > len(image_tags):
497
+ image_tags.append([])
498
+ token_ids = [
499
+ e for sublist in zip(prompt_chunks, image_tags)
500
+ for ele in sublist for e in ele
501
+ ]
502
+
503
token_ids, text, placeholders = super()._apply_prompt_updates(
504
token_ids=token_ids,
505
mm_prompt_updates=mm_prompt_updates,