A high-throughput and memory-efficient inference and serving engine for LLMs
Mirrored from https://github.com/tduncan-lum/Nvidia-Optimization-Stack---VLLM.git
Synced Nov 15, 21:00 UTC
[Bugfix] Fix broken vision language example (#14292)
Signed-off-by: Isotr0py <2037008807@qq.com>
Isotr0py committed
Mar 5, 2025 at 23:57 UTC
f71b00a19e25abebdff13a5ae9acc93749125e11
1 file changed
+18
-22
examples/offline_inference/vision_language.py
+18
-22
@@ -152,15 +152,13 @@ def run_h2ovl(questions: list[str], modality: str):
152
153
tokenizer = AutoTokenizer.from_pretrained(model_name,
154
trust_remote_code=True)
155
- prompts = [
156
- tokenizer.apply_chat_template([{
157
- 'role': 'user',
158
- 'content': f"<image>\n{question}"
159
- }],
160
- tokenize=False,
161
- add_generation_prompt=True)
162
- for question in questions
163
- ]
155
+ messages = [[{
156
+ 'role': 'user',
157
+ 'content': f"<image>\n{question}"
158
+ }] for question in questions]
159
+ prompts = tokenizer.apply_chat_template(messages,
160
+ tokenize=False,
161
+ add_generation_prompt=True)
162
163
# Stop tokens for H2OVL-Mississippi
164
# https://huggingface.co/h2oai/h2ovl-mississippi-800m
@@ -209,15 +207,13 @@ def run_internvl(questions: list[str], modality: str):
207
208
tokenizer = AutoTokenizer.from_pretrained(model_name,
209
trust_remote_code=True)
212
- prompts = [
213
- tokenizer.apply_chat_template([{
214
- 'role': 'user',
215
- 'content': f"<image>\n{question}"
216
- }],
217
- tokenize=False,
218
- add_generation_prompt=True)
219
- for question in questions
220
- ]
210
+ messages = [[{
211
+ 'role': 'user',
212
+ 'content': f"<image>\n{question}"
213
+ }] for question in questions]
214
+ prompts = tokenizer.apply_chat_template(messages,
215
+ tokenize=False,
216
+ add_generation_prompt=True)
217
218
# Stop tokens for InternVL
219
# models variants may have different stop tokens
@@ -399,7 +395,7 @@ def run_mllama(questions: list[str], modality: str):
395
)
396
397
tokenizer = AutoTokenizer.from_pretrained(model_name)
402
- messages = [{
398
+ messages = [[{
399
"role":
400
"user",
401
"content": [{
@@ -408,7 +404,7 @@ def run_mllama(questions: list[str], modality: str):
404
"type": "text",
405
"text": f"{question}"
406
}]
411
- } for question in questions]
407
+ }] for question in questions]
408
prompts = tokenizer.apply_chat_template(messages,
409
add_generation_prompt=True,
410
tokenize=False)
@@ -454,10 +450,10 @@ def run_nvlm_d(questions: list[str], modality: str):
450
451
tokenizer = AutoTokenizer.from_pretrained(model_name,
452
trust_remote_code=True)
457
- messages = [{
453
+ messages = [[{
454
'role': 'user',
455
'content': f"<image>\n{question}"
460
- } for question in questions]
456
+ }] for question in questions]
457
prompts = tokenizer.apply_chat_template(messages,
458
tokenize=False,
459
add_generation_prompt=True)