A high-throughput and memory-efficient inference and serving engine for LLMs
Mirrored from https://github.com/tduncan-lum/Nvidia-Optimization-Stack---VLLM.git
Synced Nov 16, 04:30 UTC
[Doc] Reorder vision language examples in alphabet order (#11228)
Signed-off-by: Isotr0py <2037008807@qq.com>
Isotr0py committed
Dec 16, 2024 at 19:23 UTC
2ca830dbaa1a7c30b8ff4d7c860c63f87dc18be3
2 files changed
+387
-387
examples/offline_inference_vision_language.py
+243
-243
@@ -19,6 +19,159 @@ from vllm.utils import FlexibleArgumentParser
19
# Unless specified, these settings have been tested to work on a single L4.
20
21
22
+# Aria
23
+def run_aria(question: str, modality: str):
24
+ assert modality == "image"
25
+ model_name = "rhymes-ai/Aria"
26
+
27
+ llm = LLM(model=model_name,
28
+ tokenizer_mode="slow",
29
+ trust_remote_code=True,
30
+ dtype="bfloat16",
31
+ mm_cache_preprocessor=args.mm_cache_preprocessor)
32
+
33
+ prompt = (f"<|im_start|>user\n<fim_prefix><|img|><fim_suffix>\n{question}"
34
+ "<|im_end|>\n<|im_start|>assistant\n")
35
+
36
+ stop_token_ids = [93532, 93653, 944, 93421, 1019, 93653, 93519]
37
+ return llm, prompt, stop_token_ids
38
+
39
+
40
+# BLIP-2
41
+def run_blip2(question: str, modality: str):
42
+ assert modality == "image"
43
+
44
+ # BLIP-2 prompt format is inaccurate on HuggingFace model repository.
45
+ # See https://huggingface.co/Salesforce/blip2-opt-2.7b/discussions/15#64ff02f3f8cf9e4f5b038262 #noqa
46
+ prompt = f"Question: {question} Answer:"
47
+ llm = LLM(model="Salesforce/blip2-opt-2.7b",
48
+ mm_cache_preprocessor=args.mm_cache_preprocessor)
49
+ stop_token_ids = None
50
+ return llm, prompt, stop_token_ids
51
+
52
+
53
+# Chameleon
54
+def run_chameleon(question: str, modality: str):
55
+ assert modality == "image"
56
+
57
+ prompt = f"{question}<image>"
58
+ llm = LLM(model="facebook/chameleon-7b",
59
+ max_model_len=4096,
60
+ mm_cache_preprocessor=args.mm_cache_preprocessor)
61
+ stop_token_ids = None
62
+ return llm, prompt, stop_token_ids
63
+
64
+
65
+# Fuyu
66
+def run_fuyu(question: str, modality: str):
67
+ assert modality == "image"
68
+
69
+ prompt = f"{question}\n"
70
+ llm = LLM(model="adept/fuyu-8b",
71
+ max_model_len=2048,
72
+ max_num_seqs=2,
73
+ mm_cache_preprocessor=args.mm_cache_preprocessor)
74
+ stop_token_ids = None
75
+ return llm, prompt, stop_token_ids
76
+
77
+
78
+# GLM-4v
79
+def run_glm4v(question: str, modality: str):
80
+ assert modality == "image"
81
+ model_name = "THUDM/glm-4v-9b"
82
+
83
+ llm = LLM(model=model_name,
84
+ max_model_len=2048,
85
+ max_num_seqs=2,
86
+ trust_remote_code=True,
87
+ enforce_eager=True,
88
+ mm_cache_preprocessor=args.mm_cache_preprocessor)
89
+ prompt = question
90
+ stop_token_ids = [151329, 151336, 151338]
91
+ return llm, prompt, stop_token_ids
92
+
93
+
94
+# H2OVL-Mississippi
95
+def run_h2ovl(question: str, modality: str):
96
+ assert modality == "image"
97
+
98
+ model_name = "h2oai/h2ovl-mississippi-2b"
99
+
100
+ llm = LLM(
101
+ model=model_name,
102
+ trust_remote_code=True,
103
+ max_model_len=8192,
104
+ mm_cache_preprocessor=args.mm_cache_preprocessor,
105
+ )
106
+
107
+ tokenizer = AutoTokenizer.from_pretrained(model_name,
108
+ trust_remote_code=True)
109
+ messages = [{'role': 'user', 'content': f"<image>\n{question}"}]
110
+ prompt = tokenizer.apply_chat_template(messages,
111
+ tokenize=False,
112
+ add_generation_prompt=True)
113
+
114
+ # Stop tokens for H2OVL-Mississippi
115
+ # https://huggingface.co/h2oai/h2ovl-mississippi-2b
116
+ stop_token_ids = [tokenizer.eos_token_id]
117
+ return llm, prompt, stop_token_ids
118
+
119
+
120
+# Idefics3-8B-Llama3
121
+def run_idefics3(question: str, modality: str):
122
+ assert modality == "image"
123
+ model_name = "HuggingFaceM4/Idefics3-8B-Llama3"
124
+
125
+ llm = LLM(
126
+ model=model_name,
127
+ max_model_len=8192,
128
+ max_num_seqs=2,
129
+ enforce_eager=True,
130
+ # if you are running out of memory, you can reduce the "longest_edge".
131
+ # see: https://huggingface.co/HuggingFaceM4/Idefics3-8B-Llama3#model-optimizations
132
+ mm_processor_kwargs={
133
+ "size": {
134
+ "longest_edge": 3 * 364
135
+ },
136
+ },
137
+ mm_cache_preprocessor=args.mm_cache_preprocessor,
138
+ )
139
+ prompt = (
140
+ f"<|begin_of_text|>User:<image>{question}<end_of_utterance>\nAssistant:"
141
+ )
142
+ stop_token_ids = None
143
+ return llm, prompt, stop_token_ids
144
+
145
+
146
+# InternVL
147
+def run_internvl(question: str, modality: str):
148
+ assert modality == "image"
149
+
150
+ model_name = "OpenGVLab/InternVL2-2B"
151
+
152
+ llm = LLM(
153
+ model=model_name,
154
+ trust_remote_code=True,
155
+ max_model_len=4096,
156
+ mm_cache_preprocessor=args.mm_cache_preprocessor,
157
+ )
158
+
159
+ tokenizer = AutoTokenizer.from_pretrained(model_name,
160
+ trust_remote_code=True)
161
+ messages = [{'role': 'user', 'content': f"<image>\n{question}"}]
162
+ prompt = tokenizer.apply_chat_template(messages,
163
+ tokenize=False,
164
+ add_generation_prompt=True)
165
+
166
+ # Stop tokens for InternVL
167
+ # models variants may have different stop tokens
168
+ # please refer to the model card for the correct "stop words":
169
+ # https://huggingface.co/OpenGVLab/InternVL2-2B/blob/main/conversation.py
170
+ stop_tokens = ["<|endoftext|>", "<|im_start|>", "<|im_end|>", "<|end|>"]
171
+ stop_token_ids = [tokenizer.convert_tokens_to_ids(i) for i in stop_tokens]
172
+ return llm, prompt, stop_token_ids
173
+
174
+
175
# LLaVA-1.5
176
def run_llava(question: str, modality: str):
177
assert modality == "image"
@@ -75,83 +228,20 @@ def run_llava_onevision(question: str, modality: str):
228
return llm, prompt, stop_token_ids
229
230
78
-# Fuyu
79
-def run_fuyu(question: str, modality: str):
80
- assert modality == "image"
81
-
82
- prompt = f"{question}\n"
83
- llm = LLM(model="adept/fuyu-8b",
84
- max_model_len=2048,
85
- max_num_seqs=2,
86
- mm_cache_preprocessor=args.mm_cache_preprocessor)
87
- stop_token_ids = None
88
- return llm, prompt, stop_token_ids
89
-
90
-
91
-# Phi-3-Vision
92
-def run_phi3v(question: str, modality: str):
231
+# Mantis
232
+def run_mantis(question: str, modality: str):
233
assert modality == "image"
234
95
- prompt = f"<|user|>\n<|image_1|>\n{question}<|end|>\n<|assistant|>\n"
235
+ llama3_template = '<|start_header_id|>user<|end_header_id|>\n\n{}<|eot_id|><|start_header_id|>assistant<|end_header_id|>\n\n' # noqa: E501
236
+ prompt = llama3_template.format(f"{question}\n<image>")
237
97
- # num_crops is an override kwarg to the multimodal image processor;
98
- # For some models, e.g., Phi-3.5-vision-instruct, it is recommended
99
- # to use 16 for single frame scenarios, and 4 for multi-frame.
100
- #
101
- # Generally speaking, a larger value for num_crops results in more
102
- # tokens per image instance, because it may scale the image more in
103
- # the image preprocessing. Some references in the model docs and the
104
- # formula for image tokens after the preprocessing
105
- # transform can be found below.
106
- #
107
- # https://huggingface.co/microsoft/Phi-3.5-vision-instruct#loading-the-model-locally
108
- # https://huggingface.co/microsoft/Phi-3.5-vision-instruct/blob/main/processing_phi3_v.py#L194
238
llm = LLM(
110
- model="microsoft/Phi-3.5-vision-instruct",
111
- trust_remote_code=True,
239
+ model="TIGER-Lab/Mantis-8B-siglip-llama3",
240
max_model_len=4096,
113
- max_num_seqs=2,
114
- # Note - mm_processor_kwargs can also be passed to generate/chat calls
115
- mm_processor_kwargs={"num_crops": 16},
241
+ hf_overrides={"architectures": ["MantisForConditionalGeneration"]},
242
mm_cache_preprocessor=args.mm_cache_preprocessor,
243
)
118
- stop_token_ids = None
119
- return llm, prompt, stop_token_ids
120
-
121
-
122
-# PaliGemma
123
-def run_paligemma(question: str, modality: str):
124
- assert modality == "image"
125
-
126
- # PaliGemma has special prompt format for VQA
127
- prompt = "caption en"
128
- llm = LLM(model="google/paligemma-3b-mix-224",
129
- mm_cache_preprocessor=args.mm_cache_preprocessor)
130
- stop_token_ids = None
131
- return llm, prompt, stop_token_ids
132
-
133
-
134
-# PaliGemma 2
135
-def run_paligemma2(question: str, modality: str):
136
- assert modality == "image"
137
-
138
- # PaliGemma 2 has special prompt format for VQA
139
- prompt = "caption en"
140
- llm = LLM(model="google/paligemma2-3b-ft-docci-448",
141
- mm_cache_preprocessor=args.mm_cache_preprocessor)
142
- stop_token_ids = None
143
- return llm, prompt, stop_token_ids
144
-
145
-
146
-# Chameleon
147
-def run_chameleon(question: str, modality: str):
148
- assert modality == "image"
149
-
150
- prompt = f"{question}<image>"
151
- llm = LLM(model="facebook/chameleon-7b",
152
- max_model_len=4096,
153
- mm_cache_preprocessor=args.mm_cache_preprocessor)
154
- stop_token_ids = None
244
+ stop_token_ids = [128009]
245
return llm, prompt, stop_token_ids
246
247
@@ -199,58 +289,45 @@ def run_minicpmv(question: str, modality: str):
289
return llm, prompt, stop_token_ids
290
291
202
-# H2OVL-Mississippi
203
-def run_h2ovl(question: str, modality: str):
292
+# LLama 3.2
293
+def run_mllama(question: str, modality: str):
294
assert modality == "image"
295
206
- model_name = "h2oai/h2ovl-mississippi-2b"
296
+ model_name = "meta-llama/Llama-3.2-11B-Vision-Instruct"
297
298
+ # Note: The default setting of max_num_seqs (256) and
299
+ # max_model_len (131072) for this model may cause OOM.
300
+ # You may lower either to run this example on lower-end GPUs.
301
+
302
+ # The configuration below has been confirmed to launch on a single L40 GPU.
303
llm = LLM(
304
model=model_name,
210
- trust_remote_code=True,
211
- max_model_len=8192,
305
+ max_model_len=4096,
306
+ max_num_seqs=16,
307
+ enforce_eager=True,
308
mm_cache_preprocessor=args.mm_cache_preprocessor,
309
)
310
215
- tokenizer = AutoTokenizer.from_pretrained(model_name,
216
- trust_remote_code=True)
217
- messages = [{'role': 'user', 'content': f"<image>\n{question}"}]
218
- prompt = tokenizer.apply_chat_template(messages,
219
- tokenize=False,
220
- add_generation_prompt=True)
221
-
222
- # Stop tokens for H2OVL-Mississippi
223
- # https://huggingface.co/h2oai/h2ovl-mississippi-2b
224
- stop_token_ids = [tokenizer.eos_token_id]
311
+ prompt = f"<|image|><|begin_of_text|>{question}"
312
+ stop_token_ids = None
313
return llm, prompt, stop_token_ids
314
315
228
-# InternVL
229
-def run_internvl(question: str, modality: str):
316
+# Molmo
317
+def run_molmo(question, modality):
318
assert modality == "image"
319
232
- model_name = "OpenGVLab/InternVL2-2B"
320
+ model_name = "allenai/Molmo-7B-D-0924"
321
322
llm = LLM(
323
model=model_name,
324
trust_remote_code=True,
237
- max_model_len=4096,
325
+ dtype="bfloat16",
326
mm_cache_preprocessor=args.mm_cache_preprocessor,
327
)
328
241
- tokenizer = AutoTokenizer.from_pretrained(model_name,
242
- trust_remote_code=True)
243
- messages = [{'role': 'user', 'content': f"<image>\n{question}"}]
244
- prompt = tokenizer.apply_chat_template(messages,
245
- tokenize=False,
246
- add_generation_prompt=True)
247
-
248
- # Stop tokens for InternVL
249
- # models variants may have different stop tokens
250
- # please refer to the model card for the correct "stop words":
251
- # https://huggingface.co/OpenGVLab/InternVL2-2B/blob/main/conversation.py
252
- stop_tokens = ["<|endoftext|>", "<|im_start|>", "<|im_end|>", "<|end|>"]
253
- stop_token_ids = [tokenizer.convert_tokens_to_ids(i) for i in stop_tokens]
329
+ prompt = question
330
+ stop_token_ids = None
331
return llm, prompt, stop_token_ids
332
333
@@ -279,58 +356,57 @@ def run_nvlm_d(question: str, modality: str):
356
return llm, prompt, stop_token_ids
357
358
282
-# BLIP-2
283
-def run_blip2(question: str, modality: str):
359
+# PaliGemma
360
+def run_paligemma(question: str, modality: str):
361
assert modality == "image"
362
286
- # BLIP-2 prompt format is inaccurate on HuggingFace model repository.
287
- # See https://huggingface.co/Salesforce/blip2-opt-2.7b/discussions/15#64ff02f3f8cf9e4f5b038262 #noqa
288
- prompt = f"Question: {question} Answer:"
289
- llm = LLM(model="Salesforce/blip2-opt-2.7b",
363
+ # PaliGemma has special prompt format for VQA
364
+ prompt = "caption en"
365
+ llm = LLM(model="google/paligemma-3b-mix-224",
366
mm_cache_preprocessor=args.mm_cache_preprocessor)
367
stop_token_ids = None
368
return llm, prompt, stop_token_ids
369
370
295
-# Qwen
296
-def run_qwen_vl(question: str, modality: str):
371
+# PaliGemma 2
372
+def run_paligemma2(question: str, modality: str):
373
assert modality == "image"
374
299
- llm = LLM(
300
- model="Qwen/Qwen-VL",
301
- trust_remote_code=True,
302
- max_model_len=1024,
303
- max_num_seqs=2,
304
- mm_cache_preprocessor=args.mm_cache_preprocessor,
305
- )
306
-
307
- prompt = f"{question}Picture 1: <img></img>\n"
375
+ # PaliGemma 2 has special prompt format for VQA
376
+ prompt = "caption en"
377
+ llm = LLM(model="google/paligemma2-3b-ft-docci-448",
378
+ mm_cache_preprocessor=args.mm_cache_preprocessor)
379
stop_token_ids = None
380
return llm, prompt, stop_token_ids
381
382
312
-# Qwen2-VL
313
-def run_qwen2_vl(question: str, modality: str):
383
+# Phi-3-Vision
384
+def run_phi3v(question: str, modality: str):
385
assert modality == "image"
386
316
- model_name = "Qwen/Qwen2-VL-7B-Instruct"
387
+ prompt = f"<|user|>\n<|image_1|>\n{question}<|end|>\n<|assistant|>\n"
388
389
+ # num_crops is an override kwarg to the multimodal image processor;
390
+ # For some models, e.g., Phi-3.5-vision-instruct, it is recommended
391
+ # to use 16 for single frame scenarios, and 4 for multi-frame.
392
+ #
393
+ # Generally speaking, a larger value for num_crops results in more
394
+ # tokens per image instance, because it may scale the image more in
395
+ # the image preprocessing. Some references in the model docs and the
396
+ # formula for image tokens after the preprocessing
397
+ # transform can be found below.
398
+ #
399
+ # https://huggingface.co/microsoft/Phi-3.5-vision-instruct#loading-the-model-locally
400
+ # https://huggingface.co/microsoft/Phi-3.5-vision-instruct/blob/main/processing_phi3_v.py#L194
401
llm = LLM(
319
- model=model_name,
402
+ model="microsoft/Phi-3.5-vision-instruct",
403
+ trust_remote_code=True,
404
max_model_len=4096,
321
- max_num_seqs=5,
405
+ max_num_seqs=2,
406
# Note - mm_processor_kwargs can also be passed to generate/chat calls
323
- mm_processor_kwargs={
324
- "min_pixels": 28 * 28,
325
- "max_pixels": 1280 * 28 * 28,
326
- },
407
+ mm_processor_kwargs={"num_crops": 16},
408
mm_cache_preprocessor=args.mm_cache_preprocessor,
409
)
329
-
330
- prompt = ("<|im_start|>system\nYou are a helpful assistant.<|im_end|>\n"
331
- "<|im_start|>user\n<|vision_start|><|image_pad|><|vision_end|>"
332
- f"{question}<|im_end|>\n"
333
- "<|im_start|>assistant\n")
410
stop_token_ids = None
411
return llm, prompt, stop_token_ids
412
@@ -352,149 +428,73 @@ def run_pixtral_hf(question: str, modality: str):
428
return llm, prompt, stop_token_ids
429
430
355
-# LLama 3.2
356
-def run_mllama(question: str, modality: str):
357
- assert modality == "image"
358
-
359
- model_name = "meta-llama/Llama-3.2-11B-Vision-Instruct"
360
-
361
- # Note: The default setting of max_num_seqs (256) and
362
- # max_model_len (131072) for this model may cause OOM.
363
- # You may lower either to run this example on lower-end GPUs.
364
-
365
- # The configuration below has been confirmed to launch on a single L40 GPU.
366
- llm = LLM(
367
- model=model_name,
368
- max_model_len=4096,
369
- max_num_seqs=16,
370
- enforce_eager=True,
371
- mm_cache_preprocessor=args.mm_cache_preprocessor,
372
- )
373
-
374
- prompt = f"<|image|><|begin_of_text|>{question}"
375
- stop_token_ids = None
376
- return llm, prompt, stop_token_ids
377
-
378
-
379
-# Molmo
380
-def run_molmo(question, modality):
431
+# Qwen
432
+def run_qwen_vl(question: str, modality: str):
433
assert modality == "image"
434
383
- model_name = "allenai/Molmo-7B-D-0924"
384
-
435
llm = LLM(
386
- model=model_name,
436
+ model="Qwen/Qwen-VL",
437
trust_remote_code=True,
388
- dtype="bfloat16",
438
+ max_model_len=1024,
439
+ max_num_seqs=2,
440
mm_cache_preprocessor=args.mm_cache_preprocessor,
441
)
442
392
- prompt = question
443
+ prompt = f"{question}Picture 1: <img></img>\n"
444
stop_token_ids = None
445
return llm, prompt, stop_token_ids
446
447
397
-# GLM-4v
398
-def run_glm4v(question: str, modality: str):
448
+# Qwen2-VL
449
+def run_qwen2_vl(question: str, modality: str):
450
assert modality == "image"
400
- model_name = "THUDM/glm-4v-9b"
451
402
- llm = LLM(model=model_name,
403
- max_model_len=2048,
404
- max_num_seqs=2,
405
- trust_remote_code=True,
406
- enforce_eager=True,
407
- mm_cache_preprocessor=args.mm_cache_preprocessor)
408
- prompt = question
409
- stop_token_ids = [151329, 151336, 151338]
410
- return llm, prompt, stop_token_ids
411
-
412
-
413
-# Idefics3-8B-Llama3
414
-def run_idefics3(question: str, modality: str):
415
- assert modality == "image"
416
- model_name = "HuggingFaceM4/Idefics3-8B-Llama3"
452
+ model_name = "Qwen/Qwen2-VL-7B-Instruct"
453
454
llm = LLM(
455
model=model_name,
420
- max_model_len=8192,
421
- max_num_seqs=2,
422
- enforce_eager=True,
423
- # if you are running out of memory, you can reduce the "longest_edge".
424
- # see: https://huggingface.co/HuggingFaceM4/Idefics3-8B-Llama3#model-optimizations
456
+ max_model_len=4096,
457
+ max_num_seqs=5,
458
+ # Note - mm_processor_kwargs can also be passed to generate/chat calls
459
mm_processor_kwargs={
426
- "size": {
427
- "longest_edge": 3 * 364
428
- },
460
+ "min_pixels": 28 * 28,
461
+ "max_pixels": 1280 * 28 * 28,
462
},
463
mm_cache_preprocessor=args.mm_cache_preprocessor,
464
)
432
- prompt = (
433
- f"<|begin_of_text|>User:<image>{question}<end_of_utterance>\nAssistant:"
434
- )
435
- stop_token_ids = None
436
- return llm, prompt, stop_token_ids
437
-
465
439
-# Aria
440
-def run_aria(question: str, modality: str):
441
- assert modality == "image"
442
- model_name = "rhymes-ai/Aria"
443
-
444
- llm = LLM(model=model_name,
445
- tokenizer_mode="slow",
446
- trust_remote_code=True,
447
- dtype="bfloat16",
448
- mm_cache_preprocessor=args.mm_cache_preprocessor)
449
-
450
- prompt = (f"<|im_start|>user\n<fim_prefix><|img|><fim_suffix>\n{question}"
451
- "<|im_end|>\n<|im_start|>assistant\n")
452
-
453
- stop_token_ids = [93532, 93653, 944, 93421, 1019, 93653, 93519]
454
- return llm, prompt, stop_token_ids
455
-
456
-
457
-# Mantis
458
-def run_mantis(question: str, modality: str):
459
- assert modality == "image"
460
-
461
- llama3_template = '<|start_header_id|>user<|end_header_id|>\n\n{}<|eot_id|><|start_header_id|>assistant<|end_header_id|>\n\n' # noqa: E501
462
- prompt = llama3_template.format(f"{question}\n<image>")
463
-
464
- llm = LLM(
465
- model="TIGER-Lab/Mantis-8B-siglip-llama3",
466
- max_model_len=4096,
467
- hf_overrides={"architectures": ["MantisForConditionalGeneration"]},
468
- mm_cache_preprocessor=args.mm_cache_preprocessor,
469
- )
470
- stop_token_ids = [128009]
466
+ prompt = ("<|im_start|>system\nYou are a helpful assistant.<|im_end|>\n"
467
+ "<|im_start|>user\n<|vision_start|><|image_pad|><|vision_end|>"
468
+ f"{question}<|im_end|>\n"
469
+ "<|im_start|>assistant\n")
470
+ stop_token_ids = None
471
return llm, prompt, stop_token_ids
472
473
474
model_example_map = {
475
+ "aria": run_aria,
476
+ "blip-2": run_blip2,
477
+ "chameleon": run_chameleon,
478
+ "fuyu": run_fuyu,
479
+ "glm4v": run_glm4v,
480
+ "h2ovl_chat": run_h2ovl,
481
+ "idefics3": run_idefics3,
482
+ "internvl_chat": run_internvl,
483
"llava": run_llava,
484
"llava-next": run_llava_next,
485
"llava-next-video": run_llava_next_video,
486
"llava-onevision": run_llava_onevision,
479
- "fuyu": run_fuyu,
480
- "phi3_v": run_phi3v,
481
- "paligemma": run_paligemma,
482
- "paligemma2": run_paligemma2,
483
- "chameleon": run_chameleon,
487
+ "mantis": run_mantis,
488
"minicpmv": run_minicpmv,
485
- "blip-2": run_blip2,
486
- "h2ovl_chat": run_h2ovl,
487
- "internvl_chat": run_internvl,
489
+ "mllama": run_mllama,
490
+ "molmo": run_molmo,
491
"NVLM_D": run_nvlm_d,
492
+ "paligemma": run_paligemma,
493
+ "paligemma2": run_paligemma2,
494
+ "phi3_v": run_phi3v,
495
+ "pixtral_hf": run_pixtral_hf,
496
"qwen_vl": run_qwen_vl,
497
"qwen2_vl": run_qwen2_vl,
491
- "pixtral_hf": run_pixtral_hf,
492
- "mllama": run_mllama,
493
- "molmo": run_molmo,
494
- "glm4v": run_glm4v,
495
- "idefics3": run_idefics3,
496
- "aria": run_aria,
497
- "mantis": run_mantis,
498
}
499
500
examples/offline_inference_vision_language_multi_image.py
+144
-144
@@ -33,78 +33,23 @@ class ModelRequestData(NamedTuple):
33
# Unless specified, these settings have been tested to work on a single L4.
34
35
36
-def load_qwenvl_chat(question: str, image_urls: List[str]) -> ModelRequestData:
37
- model_name = "Qwen/Qwen-VL-Chat"
38
- llm = LLM(
39
- model=model_name,
40
- trust_remote_code=True,
41
- max_model_len=1024,
42
- max_num_seqs=2,
43
- limit_mm_per_prompt={"image": len(image_urls)},
44
- )
45
- placeholders = "".join(f"Picture {i}: <img></img>\n"
46
- for i, _ in enumerate(image_urls, start=1))
47
-
48
- # This model does not have a chat_template attribute on its tokenizer,
49
- # so we need to explicitly pass it. We use ChatML since it's used in the
50
- # generation utils of the model:
51
- # https://huggingface.co/Qwen/Qwen-VL-Chat/blob/main/qwen_generation_utils.py#L265
52
- tokenizer = AutoTokenizer.from_pretrained(model_name,
53
- trust_remote_code=True)
54
-
55
- # Copied from: https://huggingface.co/docs/transformers/main/en/chat_templating
56
- chat_template = "{% if not add_generation_prompt is defined %}{% set add_generation_prompt = false %}{% endif %}{% for message in messages %}{{'<|im_start|>' + message['role'] + '\n' + message['content'] + '<|im_end|>' + '\n'}}{% endfor %}{% if add_generation_prompt %}{{ '<|im_start|>assistant\n' }}{% endif %}" # noqa: E501
57
-
58
- messages = [{'role': 'user', 'content': f"{placeholders}\n{question}"}]
59
- prompt = tokenizer.apply_chat_template(messages,
60
- tokenize=False,
61
- add_generation_prompt=True,
62
- chat_template=chat_template)
63
-
64
- stop_tokens = ["<|endoftext|>", "<|im_start|>", "<|im_end|>"]
65
- stop_token_ids = [tokenizer.convert_tokens_to_ids(i) for i in stop_tokens]
66
- return ModelRequestData(
67
- llm=llm,
68
- prompt=prompt,
69
- stop_token_ids=stop_token_ids,
70
- image_data=[fetch_image(url) for url in image_urls],
71
- chat_template=chat_template,
72
- )
73
-
74
-
75
-def load_phi3v(question: str, image_urls: List[str]) -> ModelRequestData:
76
- # num_crops is an override kwarg to the multimodal image processor;
77
- # For some models, e.g., Phi-3.5-vision-instruct, it is recommended
78
- # to use 16 for single frame scenarios, and 4 for multi-frame.
79
- #
80
- # Generally speaking, a larger value for num_crops results in more
81
- # tokens per image instance, because it may scale the image more in
82
- # the image preprocessing. Some references in the model docs and the
83
- # formula for image tokens after the preprocessing
84
- # transform can be found below.
85
- #
86
- # https://huggingface.co/microsoft/Phi-3.5-vision-instruct#loading-the-model-locally
87
- # https://huggingface.co/microsoft/Phi-3.5-vision-instruct/blob/main/processing_phi3_v.py#L194
88
- llm = LLM(
89
- model="microsoft/Phi-3.5-vision-instruct",
90
- trust_remote_code=True,
91
- max_model_len=4096,
92
- max_num_seqs=2,
93
- limit_mm_per_prompt={"image": len(image_urls)},
94
- mm_processor_kwargs={"num_crops": 4},
95
- )
96
- placeholders = "\n".join(f"<|image_{i}|>"
97
- for i, _ in enumerate(image_urls, start=1))
98
- prompt = f"<|user|>\n{placeholders}\n{question}<|end|>\n<|assistant|>\n"
99
- stop_token_ids = None
100
-
36
+def load_aria(question, image_urls: List[str]) -> ModelRequestData:
37
+ model_name = "rhymes-ai/Aria"
38
+ llm = LLM(model=model_name,
39
+ tokenizer_mode="slow",
40
+ trust_remote_code=True,
41
+ dtype="bfloat16",
42
+ limit_mm_per_prompt={"image": len(image_urls)})
43
+ placeholders = "<fim_prefix><|img|><fim_suffix>\n" * len(image_urls)
44
+ prompt = (f"<|im_start|>user\n{placeholders}{question}<|im_end|>\n"
45
+ "<|im_start|>assistant\n")
46
+ stop_token_ids = [93532, 93653, 944, 93421, 1019, 93653, 93519]
47
return ModelRequestData(
48
llm=llm,
49
prompt=prompt,
50
stop_token_ids=stop_token_ids,
51
image_data=[fetch_image(url) for url in image_urls],
106
- chat_template=None,
107
- )
52
+ chat_template=None)
53
54
55
def load_h2onvl(question: str, image_urls: List[str]) -> ModelRequestData:
@@ -141,6 +86,37 @@ def load_h2onvl(question: str, image_urls: List[str]) -> ModelRequestData:
86
)
87
88
89
+def load_idefics3(question, image_urls: List[str]) -> ModelRequestData:
90
+ model_name = "HuggingFaceM4/Idefics3-8B-Llama3"
91
+
92
+ # The configuration below has been confirmed to launch on a single L40 GPU.
93
+ llm = LLM(
94
+ model=model_name,
95
+ max_model_len=8192,
96
+ max_num_seqs=16,
97
+ enforce_eager=True,
98
+ limit_mm_per_prompt={"image": len(image_urls)},
99
+ # if you are running out of memory, you can reduce the "longest_edge".
100
+ # see: https://huggingface.co/HuggingFaceM4/Idefics3-8B-Llama3#model-optimizations
101
+ mm_processor_kwargs={
102
+ "size": {
103
+ "longest_edge": 2 * 364
104
+ },
105
+ },
106
+ )
107
+
108
+ placeholders = "\n".join(f"Image-{i}: <image>\n"
109
+ for i, _ in enumerate(image_urls, start=1))
110
+ prompt = f"<|begin_of_text|>User:{placeholders}\n{question}<end_of_utterance>\nAssistant:" # noqa: E501
111
+ return ModelRequestData(
112
+ llm=llm,
113
+ prompt=prompt,
114
+ stop_token_ids=None,
115
+ image_data=[fetch_image(url) for url in image_urls],
116
+ chat_template=None,
117
+ )
118
+
119
+
120
def load_internvl(question: str, image_urls: List[str]) -> ModelRequestData:
121
model_name = "OpenGVLab/InternVL2-2B"
122
@@ -178,6 +154,28 @@ def load_internvl(question: str, image_urls: List[str]) -> ModelRequestData:
154
)
155
156
157
+def load_mllama(question, image_urls: List[str]) -> ModelRequestData:
158
+ model_name = "meta-llama/Llama-3.2-11B-Vision-Instruct"
159
+
160
+ # The configuration below has been confirmed to launch on a single L40 GPU.
161
+ llm = LLM(
162
+ model=model_name,
163
+ max_model_len=4096,
164
+ max_num_seqs=16,
165
+ enforce_eager=True,
166
+ limit_mm_per_prompt={"image": len(image_urls)},
167
+ )
168
+
169
+ prompt = f"<|image|><|image|><|begin_of_text|>{question}"
170
+ return ModelRequestData(
171
+ llm=llm,
172
+ prompt=prompt,
173
+ stop_token_ids=None,
174
+ image_data=[fetch_image(url) for url in image_urls],
175
+ chat_template=None,
176
+ )
177
+
178
+
179
def load_nvlm_d(question: str, image_urls: List[str]):
180
model_name = "nvidia/NVLM-D-72B"
181
@@ -211,6 +209,80 @@ def load_nvlm_d(question: str, image_urls: List[str]):
209
)
210
211
212
+def load_phi3v(question: str, image_urls: List[str]) -> ModelRequestData:
213
+ # num_crops is an override kwarg to the multimodal image processor;
214
+ # For some models, e.g., Phi-3.5-vision-instruct, it is recommended
215
+ # to use 16 for single frame scenarios, and 4 for multi-frame.
216
+ #
217
+ # Generally speaking, a larger value for num_crops results in more
218
+ # tokens per image instance, because it may scale the image more in
219
+ # the image preprocessing. Some references in the model docs and the
220
+ # formula for image tokens after the preprocessing
221
+ # transform can be found below.
222
+ #
223
+ # https://huggingface.co/microsoft/Phi-3.5-vision-instruct#loading-the-model-locally
224
+ # https://huggingface.co/microsoft/Phi-3.5-vision-instruct/blob/main/processing_phi3_v.py#L194
225
+ llm = LLM(
226
+ model="microsoft/Phi-3.5-vision-instruct",
227
+ trust_remote_code=True,
228
+ max_model_len=4096,
229
+ max_num_seqs=2,
230
+ limit_mm_per_prompt={"image": len(image_urls)},
231
+ mm_processor_kwargs={"num_crops": 4},
232
+ )
233
+ placeholders = "\n".join(f"<|image_{i}|>"
234
+ for i, _ in enumerate(image_urls, start=1))
235
+ prompt = f"<|user|>\n{placeholders}\n{question}<|end|>\n<|assistant|>\n"
236
+ stop_token_ids = None
237
+
238
+ return ModelRequestData(
239
+ llm=llm,
240
+ prompt=prompt,
241
+ stop_token_ids=stop_token_ids,
242
+ image_data=[fetch_image(url) for url in image_urls],
243
+ chat_template=None,
244
+ )
245
+
246
+
247
+def load_qwenvl_chat(question: str, image_urls: List[str]) -> ModelRequestData:
248
+ model_name = "Qwen/Qwen-VL-Chat"
249
+ llm = LLM(
250
+ model=model_name,
251
+ trust_remote_code=True,
252
+ max_model_len=1024,
253
+ max_num_seqs=2,
254
+ limit_mm_per_prompt={"image": len(image_urls)},
255
+ )
256
+ placeholders = "".join(f"Picture {i}: <img></img>\n"
257
+ for i, _ in enumerate(image_urls, start=1))
258
+
259
+ # This model does not have a chat_template attribute on its tokenizer,
260
+ # so we need to explicitly pass it. We use ChatML since it's used in the
261
+ # generation utils of the model:
262
+ # https://huggingface.co/Qwen/Qwen-VL-Chat/blob/main/qwen_generation_utils.py#L265
263
+ tokenizer = AutoTokenizer.from_pretrained(model_name,
264
+ trust_remote_code=True)
265
+
266
+ # Copied from: https://huggingface.co/docs/transformers/main/en/chat_templating
267
+ chat_template = "{% if not add_generation_prompt is defined %}{% set add_generation_prompt = false %}{% endif %}{% for message in messages %}{{'<|im_start|>' + message['role'] + '\n' + message['content'] + '<|im_end|>' + '\n'}}{% endfor %}{% if add_generation_prompt %}{{ '<|im_start|>assistant\n' }}{% endif %}" # noqa: E501
268
+
269
+ messages = [{'role': 'user', 'content': f"{placeholders}\n{question}"}]
270
+ prompt = tokenizer.apply_chat_template(messages,
271
+ tokenize=False,
272
+ add_generation_prompt=True,
273
+ chat_template=chat_template)
274
+
275
+ stop_tokens = ["<|endoftext|>", "<|im_start|>", "<|im_end|>"]
276
+ stop_token_ids = [tokenizer.convert_tokens_to_ids(i) for i in stop_tokens]
277
+ return ModelRequestData(
278
+ llm=llm,
279
+ prompt=prompt,
280
+ stop_token_ids=stop_token_ids,
281
+ image_data=[fetch_image(url) for url in image_urls],
282
+ chat_template=chat_template,
283
+ )
284
+
285
+
286
def load_qwen2_vl(question, image_urls: List[str]) -> ModelRequestData:
287
try:
288
from qwen_vl_utils import process_vision_info
@@ -268,88 +340,16 @@ def load_qwen2_vl(question, image_urls: List[str]) -> ModelRequestData:
340
)
341
342
271
-def load_mllama(question, image_urls: List[str]) -> ModelRequestData:
272
- model_name = "meta-llama/Llama-3.2-11B-Vision-Instruct"
273
-
274
- # The configuration below has been confirmed to launch on a single L40 GPU.
275
- llm = LLM(
276
- model=model_name,
277
- max_model_len=4096,
278
- max_num_seqs=16,
279
- enforce_eager=True,
280
- limit_mm_per_prompt={"image": len(image_urls)},
281
- )
282
-
283
- prompt = f"<|image|><|image|><|begin_of_text|>{question}"
284
- return ModelRequestData(
285
- llm=llm,
286
- prompt=prompt,
287
- stop_token_ids=None,
288
- image_data=[fetch_image(url) for url in image_urls],
289
- chat_template=None,
290
- )
291
-
292
-
293
-def load_idefics3(question, image_urls: List[str]) -> ModelRequestData:
294
- model_name = "HuggingFaceM4/Idefics3-8B-Llama3"
295
-
296
- # The configuration below has been confirmed to launch on a single L40 GPU.
297
- llm = LLM(
298
- model=model_name,
299
- max_model_len=8192,
300
- max_num_seqs=16,
301
- enforce_eager=True,
302
- limit_mm_per_prompt={"image": len(image_urls)},
303
- # if you are running out of memory, you can reduce the "longest_edge".
304
- # see: https://huggingface.co/HuggingFaceM4/Idefics3-8B-Llama3#model-optimizations
305
- mm_processor_kwargs={
306
- "size": {
307
- "longest_edge": 2 * 364
308
- },
309
- },
310
- )
311
-
312
- placeholders = "\n".join(f"Image-{i}: <image>\n"
313
- for i, _ in enumerate(image_urls, start=1))
314
- prompt = f"<|begin_of_text|>User:{placeholders}\n{question}<end_of_utterance>\nAssistant:" # noqa: E501
315
- return ModelRequestData(
316
- llm=llm,
317
- prompt=prompt,
318
- stop_token_ids=None,
319
- image_data=[fetch_image(url) for url in image_urls],
320
- chat_template=None,
321
- )
322
-
323
-
324
-def load_aria(question, image_urls: List[str]) -> ModelRequestData:
325
- model_name = "rhymes-ai/Aria"
326
- llm = LLM(model=model_name,
327
- tokenizer_mode="slow",
328
- trust_remote_code=True,
329
- dtype="bfloat16",
330
- limit_mm_per_prompt={"image": len(image_urls)})
331
- placeholders = "<fim_prefix><|img|><fim_suffix>\n" * len(image_urls)
332
- prompt = (f"<|im_start|>user\n{placeholders}{question}<|im_end|>\n"
333
- "<|im_start|>assistant\n")
334
- stop_token_ids = [93532, 93653, 944, 93421, 1019, 93653, 93519]
335
- return ModelRequestData(
336
- llm=llm,
337
- prompt=prompt,
338
- stop_token_ids=stop_token_ids,
339
- image_data=[fetch_image(url) for url in image_urls],
340
- chat_template=None)
341
-
342
-
343
model_example_map = {
344
- "phi3_v": load_phi3v,
344
+ "aria": load_aria,
345
"h2ovl_chat": load_h2onvl,
346
+ "idefics3": load_idefics3,
347
"internvl_chat": load_internvl,
348
+ "mllama": load_mllama,
349
"NVLM_D": load_nvlm_d,
348
- "qwen2_vl": load_qwen2_vl,
350
+ "phi3_v": load_phi3v,
351
"qwen_vl_chat": load_qwenvl_chat,
350
- "mllama": load_mllama,
351
- "idefics3": load_idefics3,
352
- "aria": load_aria,
352
+ "qwen2_vl": load_qwen2_vl,
353
}
354
355