@tduncan / Nvidia-Optimization-Stack-VLLM Mirror · read-only / commits / 2ca830dbaa
Fork 0

A high-throughput and memory-efficient inference and serving engine for LLMs

Mirrored from https://github.com/tduncan-lum/Nvidia-Optimization-Stack---VLLM.git

Synced Nov 16, 04:30 UTC

[Doc] Reorder vision language examples in alphabet order (#11228)

Signed-off-by: Isotr0py <2037008807@qq.com>

Isotr0py committed Dec 16, 2024 at 19:23 UTC 2ca830dbaa1a7c30b8ff4d7c860c63f87dc18be3
2 files changed +387 -387
examples/offline_inference_vision_language.py
+243 -243
@@ -19,6 +19,159 @@ from vllm.utils import FlexibleArgumentParser
19 # Unless specified, these settings have been tested to work on a single L4.
20
21
22 +# Aria
23 +def run_aria(question: str, modality: str):
24 + assert modality == "image"
25 + model_name = "rhymes-ai/Aria"
26 +
27 + llm = LLM(model=model_name,
28 + tokenizer_mode="slow",
29 + trust_remote_code=True,
30 + dtype="bfloat16",
31 + mm_cache_preprocessor=args.mm_cache_preprocessor)
32 +
33 + prompt = (f"<|im_start|>user\n<fim_prefix><|img|><fim_suffix>\n{question}"
34 + "<|im_end|>\n<|im_start|>assistant\n")
35 +
36 + stop_token_ids = [93532, 93653, 944, 93421, 1019, 93653, 93519]
37 + return llm, prompt, stop_token_ids
38 +
39 +
40 +# BLIP-2
41 +def run_blip2(question: str, modality: str):
42 + assert modality == "image"
43 +
44 + # BLIP-2 prompt format is inaccurate on HuggingFace model repository.
45 + # See https://huggingface.co/Salesforce/blip2-opt-2.7b/discussions/15#64ff02f3f8cf9e4f5b038262 #noqa
46 + prompt = f"Question: {question} Answer:"
47 + llm = LLM(model="Salesforce/blip2-opt-2.7b",
48 + mm_cache_preprocessor=args.mm_cache_preprocessor)
49 + stop_token_ids = None
50 + return llm, prompt, stop_token_ids
51 +
52 +
53 +# Chameleon
54 +def run_chameleon(question: str, modality: str):
55 + assert modality == "image"
56 +
57 + prompt = f"{question}<image>"
58 + llm = LLM(model="facebook/chameleon-7b",
59 + max_model_len=4096,
60 + mm_cache_preprocessor=args.mm_cache_preprocessor)
61 + stop_token_ids = None
62 + return llm, prompt, stop_token_ids
63 +
64 +
65 +# Fuyu
66 +def run_fuyu(question: str, modality: str):
67 + assert modality == "image"
68 +
69 + prompt = f"{question}\n"
70 + llm = LLM(model="adept/fuyu-8b",
71 + max_model_len=2048,
72 + max_num_seqs=2,
73 + mm_cache_preprocessor=args.mm_cache_preprocessor)
74 + stop_token_ids = None
75 + return llm, prompt, stop_token_ids
76 +
77 +
78 +# GLM-4v
79 +def run_glm4v(question: str, modality: str):
80 + assert modality == "image"
81 + model_name = "THUDM/glm-4v-9b"
82 +
83 + llm = LLM(model=model_name,
84 + max_model_len=2048,
85 + max_num_seqs=2,
86 + trust_remote_code=True,
87 + enforce_eager=True,
88 + mm_cache_preprocessor=args.mm_cache_preprocessor)
89 + prompt = question
90 + stop_token_ids = [151329, 151336, 151338]
91 + return llm, prompt, stop_token_ids
92 +
93 +
94 +# H2OVL-Mississippi
95 +def run_h2ovl(question: str, modality: str):
96 + assert modality == "image"
97 +
98 + model_name = "h2oai/h2ovl-mississippi-2b"
99 +
100 + llm = LLM(
101 + model=model_name,
102 + trust_remote_code=True,
103 + max_model_len=8192,
104 + mm_cache_preprocessor=args.mm_cache_preprocessor,
105 + )
106 +
107 + tokenizer = AutoTokenizer.from_pretrained(model_name,
108 + trust_remote_code=True)
109 + messages = [{'role': 'user', 'content': f"<image>\n{question}"}]
110 + prompt = tokenizer.apply_chat_template(messages,
111 + tokenize=False,
112 + add_generation_prompt=True)
113 +
114 + # Stop tokens for H2OVL-Mississippi
115 + # https://huggingface.co/h2oai/h2ovl-mississippi-2b
116 + stop_token_ids = [tokenizer.eos_token_id]
117 + return llm, prompt, stop_token_ids
118 +
119 +
120 +# Idefics3-8B-Llama3
121 +def run_idefics3(question: str, modality: str):
122 + assert modality == "image"
123 + model_name = "HuggingFaceM4/Idefics3-8B-Llama3"
124 +
125 + llm = LLM(
126 + model=model_name,
127 + max_model_len=8192,
128 + max_num_seqs=2,
129 + enforce_eager=True,
130 + # if you are running out of memory, you can reduce the "longest_edge".
131 + # see: https://huggingface.co/HuggingFaceM4/Idefics3-8B-Llama3#model-optimizations
132 + mm_processor_kwargs={
133 + "size": {
134 + "longest_edge": 3 * 364
135 + },
136 + },
137 + mm_cache_preprocessor=args.mm_cache_preprocessor,
138 + )
139 + prompt = (
140 + f"<|begin_of_text|>User:<image>{question}<end_of_utterance>\nAssistant:"
141 + )
142 + stop_token_ids = None
143 + return llm, prompt, stop_token_ids
144 +
145 +
146 +# InternVL
147 +def run_internvl(question: str, modality: str):
148 + assert modality == "image"
149 +
150 + model_name = "OpenGVLab/InternVL2-2B"
151 +
152 + llm = LLM(
153 + model=model_name,
154 + trust_remote_code=True,
155 + max_model_len=4096,
156 + mm_cache_preprocessor=args.mm_cache_preprocessor,
157 + )
158 +
159 + tokenizer = AutoTokenizer.from_pretrained(model_name,
160 + trust_remote_code=True)
161 + messages = [{'role': 'user', 'content': f"<image>\n{question}"}]
162 + prompt = tokenizer.apply_chat_template(messages,
163 + tokenize=False,
164 + add_generation_prompt=True)
165 +
166 + # Stop tokens for InternVL
167 + # models variants may have different stop tokens
168 + # please refer to the model card for the correct "stop words":
169 + # https://huggingface.co/OpenGVLab/InternVL2-2B/blob/main/conversation.py
170 + stop_tokens = ["<|endoftext|>", "<|im_start|>", "<|im_end|>", "<|end|>"]
171 + stop_token_ids = [tokenizer.convert_tokens_to_ids(i) for i in stop_tokens]
172 + return llm, prompt, stop_token_ids
173 +
174 +
175 # LLaVA-1.5
176 def run_llava(question: str, modality: str):
177 assert modality == "image"
@@ -75,83 +228,20 @@ def run_llava_onevision(question: str, modality: str):
228 return llm, prompt, stop_token_ids
229
230
78 -# Fuyu
79 -def run_fuyu(question: str, modality: str):
80 - assert modality == "image"
81 -
82 - prompt = f"{question}\n"
83 - llm = LLM(model="adept/fuyu-8b",
84 - max_model_len=2048,
85 - max_num_seqs=2,
86 - mm_cache_preprocessor=args.mm_cache_preprocessor)
87 - stop_token_ids = None
88 - return llm, prompt, stop_token_ids
89 -
90 -
91 -# Phi-3-Vision
92 -def run_phi3v(question: str, modality: str):
231 +# Mantis
232 +def run_mantis(question: str, modality: str):
233 assert modality == "image"
234
95 - prompt = f"<|user|>\n<|image_1|>\n{question}<|end|>\n<|assistant|>\n"
235 + llama3_template = '<|start_header_id|>user<|end_header_id|>\n\n{}<|eot_id|><|start_header_id|>assistant<|end_header_id|>\n\n' # noqa: E501
236 + prompt = llama3_template.format(f"{question}\n<image>")
237
97 - # num_crops is an override kwarg to the multimodal image processor;
98 - # For some models, e.g., Phi-3.5-vision-instruct, it is recommended
99 - # to use 16 for single frame scenarios, and 4 for multi-frame.
100 - #
101 - # Generally speaking, a larger value for num_crops results in more
102 - # tokens per image instance, because it may scale the image more in
103 - # the image preprocessing. Some references in the model docs and the
104 - # formula for image tokens after the preprocessing
105 - # transform can be found below.
106 - #
107 - # https://huggingface.co/microsoft/Phi-3.5-vision-instruct#loading-the-model-locally
108 - # https://huggingface.co/microsoft/Phi-3.5-vision-instruct/blob/main/processing_phi3_v.py#L194
238 llm = LLM(
110 - model="microsoft/Phi-3.5-vision-instruct",
111 - trust_remote_code=True,
239 + model="TIGER-Lab/Mantis-8B-siglip-llama3",
240 max_model_len=4096,
113 - max_num_seqs=2,
114 - # Note - mm_processor_kwargs can also be passed to generate/chat calls
115 - mm_processor_kwargs={"num_crops": 16},
241 + hf_overrides={"architectures": ["MantisForConditionalGeneration"]},
242 mm_cache_preprocessor=args.mm_cache_preprocessor,
243 )
118 - stop_token_ids = None
119 - return llm, prompt, stop_token_ids
120 -
121 -
122 -# PaliGemma
123 -def run_paligemma(question: str, modality: str):
124 - assert modality == "image"
125 -
126 - # PaliGemma has special prompt format for VQA
127 - prompt = "caption en"
128 - llm = LLM(model="google/paligemma-3b-mix-224",
129 - mm_cache_preprocessor=args.mm_cache_preprocessor)
130 - stop_token_ids = None
131 - return llm, prompt, stop_token_ids
132 -
133 -
134 -# PaliGemma 2
135 -def run_paligemma2(question: str, modality: str):
136 - assert modality == "image"
137 -
138 - # PaliGemma 2 has special prompt format for VQA
139 - prompt = "caption en"
140 - llm = LLM(model="google/paligemma2-3b-ft-docci-448",
141 - mm_cache_preprocessor=args.mm_cache_preprocessor)
142 - stop_token_ids = None
143 - return llm, prompt, stop_token_ids
144 -
145 -
146 -# Chameleon
147 -def run_chameleon(question: str, modality: str):
148 - assert modality == "image"
149 -
150 - prompt = f"{question}<image>"
151 - llm = LLM(model="facebook/chameleon-7b",
152 - max_model_len=4096,
153 - mm_cache_preprocessor=args.mm_cache_preprocessor)
154 - stop_token_ids = None
244 + stop_token_ids = [128009]
245 return llm, prompt, stop_token_ids
246
247
@@ -199,58 +289,45 @@ def run_minicpmv(question: str, modality: str):
289 return llm, prompt, stop_token_ids
290
291
202 -# H2OVL-Mississippi
203 -def run_h2ovl(question: str, modality: str):
292 +# LLama 3.2
293 +def run_mllama(question: str, modality: str):
294 assert modality == "image"
295
206 - model_name = "h2oai/h2ovl-mississippi-2b"
296 + model_name = "meta-llama/Llama-3.2-11B-Vision-Instruct"
297
298 + # Note: The default setting of max_num_seqs (256) and
299 + # max_model_len (131072) for this model may cause OOM.
300 + # You may lower either to run this example on lower-end GPUs.
301 +
302 + # The configuration below has been confirmed to launch on a single L40 GPU.
303 llm = LLM(
304 model=model_name,
210 - trust_remote_code=True,
211 - max_model_len=8192,
305 + max_model_len=4096,
306 + max_num_seqs=16,
307 + enforce_eager=True,
308 mm_cache_preprocessor=args.mm_cache_preprocessor,
309 )
310
215 - tokenizer = AutoTokenizer.from_pretrained(model_name,
216 - trust_remote_code=True)
217 - messages = [{'role': 'user', 'content': f"<image>\n{question}"}]
218 - prompt = tokenizer.apply_chat_template(messages,
219 - tokenize=False,
220 - add_generation_prompt=True)
221 -
222 - # Stop tokens for H2OVL-Mississippi
223 - # https://huggingface.co/h2oai/h2ovl-mississippi-2b
224 - stop_token_ids = [tokenizer.eos_token_id]
311 + prompt = f"<|image|><|begin_of_text|>{question}"
312 + stop_token_ids = None
313 return llm, prompt, stop_token_ids
314
315
228 -# InternVL
229 -def run_internvl(question: str, modality: str):
316 +# Molmo
317 +def run_molmo(question, modality):
318 assert modality == "image"
319
232 - model_name = "OpenGVLab/InternVL2-2B"
320 + model_name = "allenai/Molmo-7B-D-0924"
321
322 llm = LLM(
323 model=model_name,
324 trust_remote_code=True,
237 - max_model_len=4096,
325 + dtype="bfloat16",
326 mm_cache_preprocessor=args.mm_cache_preprocessor,
327 )
328
241 - tokenizer = AutoTokenizer.from_pretrained(model_name,
242 - trust_remote_code=True)
243 - messages = [{'role': 'user', 'content': f"<image>\n{question}"}]
244 - prompt = tokenizer.apply_chat_template(messages,
245 - tokenize=False,
246 - add_generation_prompt=True)
247 -
248 - # Stop tokens for InternVL
249 - # models variants may have different stop tokens
250 - # please refer to the model card for the correct "stop words":
251 - # https://huggingface.co/OpenGVLab/InternVL2-2B/blob/main/conversation.py
252 - stop_tokens = ["<|endoftext|>", "<|im_start|>", "<|im_end|>", "<|end|>"]
253 - stop_token_ids = [tokenizer.convert_tokens_to_ids(i) for i in stop_tokens]
329 + prompt = question
330 + stop_token_ids = None
331 return llm, prompt, stop_token_ids
332
333
@@ -279,58 +356,57 @@ def run_nvlm_d(question: str, modality: str):
356 return llm, prompt, stop_token_ids
357
358
282 -# BLIP-2
283 -def run_blip2(question: str, modality: str):
359 +# PaliGemma
360 +def run_paligemma(question: str, modality: str):
361 assert modality == "image"
362
286 - # BLIP-2 prompt format is inaccurate on HuggingFace model repository.
287 - # See https://huggingface.co/Salesforce/blip2-opt-2.7b/discussions/15#64ff02f3f8cf9e4f5b038262 #noqa
288 - prompt = f"Question: {question} Answer:"
289 - llm = LLM(model="Salesforce/blip2-opt-2.7b",
363 + # PaliGemma has special prompt format for VQA
364 + prompt = "caption en"
365 + llm = LLM(model="google/paligemma-3b-mix-224",
366 mm_cache_preprocessor=args.mm_cache_preprocessor)
367 stop_token_ids = None
368 return llm, prompt, stop_token_ids
369
370
295 -# Qwen
296 -def run_qwen_vl(question: str, modality: str):
371 +# PaliGemma 2
372 +def run_paligemma2(question: str, modality: str):
373 assert modality == "image"
374
299 - llm = LLM(
300 - model="Qwen/Qwen-VL",
301 - trust_remote_code=True,
302 - max_model_len=1024,
303 - max_num_seqs=2,
304 - mm_cache_preprocessor=args.mm_cache_preprocessor,
305 - )
306 -
307 - prompt = f"{question}Picture 1: <img></img>\n"
375 + # PaliGemma 2 has special prompt format for VQA
376 + prompt = "caption en"
377 + llm = LLM(model="google/paligemma2-3b-ft-docci-448",
378 + mm_cache_preprocessor=args.mm_cache_preprocessor)
379 stop_token_ids = None
380 return llm, prompt, stop_token_ids
381
382
312 -# Qwen2-VL
313 -def run_qwen2_vl(question: str, modality: str):
383 +# Phi-3-Vision
384 +def run_phi3v(question: str, modality: str):
385 assert modality == "image"
386
316 - model_name = "Qwen/Qwen2-VL-7B-Instruct"
387 + prompt = f"<|user|>\n<|image_1|>\n{question}<|end|>\n<|assistant|>\n"
388
389 + # num_crops is an override kwarg to the multimodal image processor;
390 + # For some models, e.g., Phi-3.5-vision-instruct, it is recommended
391 + # to use 16 for single frame scenarios, and 4 for multi-frame.
392 + #
393 + # Generally speaking, a larger value for num_crops results in more
394 + # tokens per image instance, because it may scale the image more in
395 + # the image preprocessing. Some references in the model docs and the
396 + # formula for image tokens after the preprocessing
397 + # transform can be found below.
398 + #
399 + # https://huggingface.co/microsoft/Phi-3.5-vision-instruct#loading-the-model-locally
400 + # https://huggingface.co/microsoft/Phi-3.5-vision-instruct/blob/main/processing_phi3_v.py#L194
401 llm = LLM(
319 - model=model_name,
402 + model="microsoft/Phi-3.5-vision-instruct",
403 + trust_remote_code=True,
404 max_model_len=4096,
321 - max_num_seqs=5,
405 + max_num_seqs=2,
406 # Note - mm_processor_kwargs can also be passed to generate/chat calls
323 - mm_processor_kwargs={
324 - "min_pixels": 28 * 28,
325 - "max_pixels": 1280 * 28 * 28,
326 - },
407 + mm_processor_kwargs={"num_crops": 16},
408 mm_cache_preprocessor=args.mm_cache_preprocessor,
409 )
329 -
330 - prompt = ("<|im_start|>system\nYou are a helpful assistant.<|im_end|>\n"
331 - "<|im_start|>user\n<|vision_start|><|image_pad|><|vision_end|>"
332 - f"{question}<|im_end|>\n"
333 - "<|im_start|>assistant\n")
410 stop_token_ids = None
411 return llm, prompt, stop_token_ids
412
@@ -352,149 +428,73 @@ def run_pixtral_hf(question: str, modality: str):
428 return llm, prompt, stop_token_ids
429
430
355 -# LLama 3.2
356 -def run_mllama(question: str, modality: str):
357 - assert modality == "image"
358 -
359 - model_name = "meta-llama/Llama-3.2-11B-Vision-Instruct"
360 -
361 - # Note: The default setting of max_num_seqs (256) and
362 - # max_model_len (131072) for this model may cause OOM.
363 - # You may lower either to run this example on lower-end GPUs.
364 -
365 - # The configuration below has been confirmed to launch on a single L40 GPU.
366 - llm = LLM(
367 - model=model_name,
368 - max_model_len=4096,
369 - max_num_seqs=16,
370 - enforce_eager=True,
371 - mm_cache_preprocessor=args.mm_cache_preprocessor,
372 - )
373 -
374 - prompt = f"<|image|><|begin_of_text|>{question}"
375 - stop_token_ids = None
376 - return llm, prompt, stop_token_ids
377 -
378 -
379 -# Molmo
380 -def run_molmo(question, modality):
431 +# Qwen
432 +def run_qwen_vl(question: str, modality: str):
433 assert modality == "image"
434
383 - model_name = "allenai/Molmo-7B-D-0924"
384 -
435 llm = LLM(
386 - model=model_name,
436 + model="Qwen/Qwen-VL",
437 trust_remote_code=True,
388 - dtype="bfloat16",
438 + max_model_len=1024,
439 + max_num_seqs=2,
440 mm_cache_preprocessor=args.mm_cache_preprocessor,
441 )
442
392 - prompt = question
443 + prompt = f"{question}Picture 1: <img></img>\n"
444 stop_token_ids = None
445 return llm, prompt, stop_token_ids
446
447
397 -# GLM-4v
398 -def run_glm4v(question: str, modality: str):
448 +# Qwen2-VL
449 +def run_qwen2_vl(question: str, modality: str):
450 assert modality == "image"
400 - model_name = "THUDM/glm-4v-9b"
451
402 - llm = LLM(model=model_name,
403 - max_model_len=2048,
404 - max_num_seqs=2,
405 - trust_remote_code=True,
406 - enforce_eager=True,
407 - mm_cache_preprocessor=args.mm_cache_preprocessor)
408 - prompt = question
409 - stop_token_ids = [151329, 151336, 151338]
410 - return llm, prompt, stop_token_ids
411 -
412 -
413 -# Idefics3-8B-Llama3
414 -def run_idefics3(question: str, modality: str):
415 - assert modality == "image"
416 - model_name = "HuggingFaceM4/Idefics3-8B-Llama3"
452 + model_name = "Qwen/Qwen2-VL-7B-Instruct"
453
454 llm = LLM(
455 model=model_name,
420 - max_model_len=8192,
421 - max_num_seqs=2,
422 - enforce_eager=True,
423 - # if you are running out of memory, you can reduce the "longest_edge".
424 - # see: https://huggingface.co/HuggingFaceM4/Idefics3-8B-Llama3#model-optimizations
456 + max_model_len=4096,
457 + max_num_seqs=5,
458 + # Note - mm_processor_kwargs can also be passed to generate/chat calls
459 mm_processor_kwargs={
426 - "size": {
427 - "longest_edge": 3 * 364
428 - },
460 + "min_pixels": 28 * 28,
461 + "max_pixels": 1280 * 28 * 28,
462 },
463 mm_cache_preprocessor=args.mm_cache_preprocessor,
464 )
432 - prompt = (
433 - f"<|begin_of_text|>User:<image>{question}<end_of_utterance>\nAssistant:"
434 - )
435 - stop_token_ids = None
436 - return llm, prompt, stop_token_ids
437 -
465
439 -# Aria
440 -def run_aria(question: str, modality: str):
441 - assert modality == "image"
442 - model_name = "rhymes-ai/Aria"
443 -
444 - llm = LLM(model=model_name,
445 - tokenizer_mode="slow",
446 - trust_remote_code=True,
447 - dtype="bfloat16",
448 - mm_cache_preprocessor=args.mm_cache_preprocessor)
449 -
450 - prompt = (f"<|im_start|>user\n<fim_prefix><|img|><fim_suffix>\n{question}"
451 - "<|im_end|>\n<|im_start|>assistant\n")
452 -
453 - stop_token_ids = [93532, 93653, 944, 93421, 1019, 93653, 93519]
454 - return llm, prompt, stop_token_ids
455 -
456 -
457 -# Mantis
458 -def run_mantis(question: str, modality: str):
459 - assert modality == "image"
460 -
461 - llama3_template = '<|start_header_id|>user<|end_header_id|>\n\n{}<|eot_id|><|start_header_id|>assistant<|end_header_id|>\n\n' # noqa: E501
462 - prompt = llama3_template.format(f"{question}\n<image>")
463 -
464 - llm = LLM(
465 - model="TIGER-Lab/Mantis-8B-siglip-llama3",
466 - max_model_len=4096,
467 - hf_overrides={"architectures": ["MantisForConditionalGeneration"]},
468 - mm_cache_preprocessor=args.mm_cache_preprocessor,
469 - )
470 - stop_token_ids = [128009]
466 + prompt = ("<|im_start|>system\nYou are a helpful assistant.<|im_end|>\n"
467 + "<|im_start|>user\n<|vision_start|><|image_pad|><|vision_end|>"
468 + f"{question}<|im_end|>\n"
469 + "<|im_start|>assistant\n")
470 + stop_token_ids = None
471 return llm, prompt, stop_token_ids
472
473
474 model_example_map = {
475 + "aria": run_aria,
476 + "blip-2": run_blip2,
477 + "chameleon": run_chameleon,
478 + "fuyu": run_fuyu,
479 + "glm4v": run_glm4v,
480 + "h2ovl_chat": run_h2ovl,
481 + "idefics3": run_idefics3,
482 + "internvl_chat": run_internvl,
483 "llava": run_llava,
484 "llava-next": run_llava_next,
485 "llava-next-video": run_llava_next_video,
486 "llava-onevision": run_llava_onevision,
479 - "fuyu": run_fuyu,
480 - "phi3_v": run_phi3v,
481 - "paligemma": run_paligemma,
482 - "paligemma2": run_paligemma2,
483 - "chameleon": run_chameleon,
487 + "mantis": run_mantis,
488 "minicpmv": run_minicpmv,
485 - "blip-2": run_blip2,
486 - "h2ovl_chat": run_h2ovl,
487 - "internvl_chat": run_internvl,
489 + "mllama": run_mllama,
490 + "molmo": run_molmo,
491 "NVLM_D": run_nvlm_d,
492 + "paligemma": run_paligemma,
493 + "paligemma2": run_paligemma2,
494 + "phi3_v": run_phi3v,
495 + "pixtral_hf": run_pixtral_hf,
496 "qwen_vl": run_qwen_vl,
497 "qwen2_vl": run_qwen2_vl,
491 - "pixtral_hf": run_pixtral_hf,
492 - "mllama": run_mllama,
493 - "molmo": run_molmo,
494 - "glm4v": run_glm4v,
495 - "idefics3": run_idefics3,
496 - "aria": run_aria,
497 - "mantis": run_mantis,
498 }
499
500
examples/offline_inference_vision_language_multi_image.py
+144 -144
@@ -33,78 +33,23 @@ class ModelRequestData(NamedTuple):
33 # Unless specified, these settings have been tested to work on a single L4.
34
35
36 -def load_qwenvl_chat(question: str, image_urls: List[str]) -> ModelRequestData:
37 - model_name = "Qwen/Qwen-VL-Chat"
38 - llm = LLM(
39 - model=model_name,
40 - trust_remote_code=True,
41 - max_model_len=1024,
42 - max_num_seqs=2,
43 - limit_mm_per_prompt={"image": len(image_urls)},
44 - )
45 - placeholders = "".join(f"Picture {i}: <img></img>\n"
46 - for i, _ in enumerate(image_urls, start=1))
47 -
48 - # This model does not have a chat_template attribute on its tokenizer,
49 - # so we need to explicitly pass it. We use ChatML since it's used in the
50 - # generation utils of the model:
51 - # https://huggingface.co/Qwen/Qwen-VL-Chat/blob/main/qwen_generation_utils.py#L265
52 - tokenizer = AutoTokenizer.from_pretrained(model_name,
53 - trust_remote_code=True)
54 -
55 - # Copied from: https://huggingface.co/docs/transformers/main/en/chat_templating
56 - chat_template = "{% if not add_generation_prompt is defined %}{% set add_generation_prompt = false %}{% endif %}{% for message in messages %}{{'<|im_start|>' + message['role'] + '\n' + message['content'] + '<|im_end|>' + '\n'}}{% endfor %}{% if add_generation_prompt %}{{ '<|im_start|>assistant\n' }}{% endif %}" # noqa: E501
57 -
58 - messages = [{'role': 'user', 'content': f"{placeholders}\n{question}"}]
59 - prompt = tokenizer.apply_chat_template(messages,
60 - tokenize=False,
61 - add_generation_prompt=True,
62 - chat_template=chat_template)
63 -
64 - stop_tokens = ["<|endoftext|>", "<|im_start|>", "<|im_end|>"]
65 - stop_token_ids = [tokenizer.convert_tokens_to_ids(i) for i in stop_tokens]
66 - return ModelRequestData(
67 - llm=llm,
68 - prompt=prompt,
69 - stop_token_ids=stop_token_ids,
70 - image_data=[fetch_image(url) for url in image_urls],
71 - chat_template=chat_template,
72 - )
73 -
74 -
75 -def load_phi3v(question: str, image_urls: List[str]) -> ModelRequestData:
76 - # num_crops is an override kwarg to the multimodal image processor;
77 - # For some models, e.g., Phi-3.5-vision-instruct, it is recommended
78 - # to use 16 for single frame scenarios, and 4 for multi-frame.
79 - #
80 - # Generally speaking, a larger value for num_crops results in more
81 - # tokens per image instance, because it may scale the image more in
82 - # the image preprocessing. Some references in the model docs and the
83 - # formula for image tokens after the preprocessing
84 - # transform can be found below.
85 - #
86 - # https://huggingface.co/microsoft/Phi-3.5-vision-instruct#loading-the-model-locally
87 - # https://huggingface.co/microsoft/Phi-3.5-vision-instruct/blob/main/processing_phi3_v.py#L194
88 - llm = LLM(
89 - model="microsoft/Phi-3.5-vision-instruct",
90 - trust_remote_code=True,
91 - max_model_len=4096,
92 - max_num_seqs=2,
93 - limit_mm_per_prompt={"image": len(image_urls)},
94 - mm_processor_kwargs={"num_crops": 4},
95 - )
96 - placeholders = "\n".join(f"<|image_{i}|>"
97 - for i, _ in enumerate(image_urls, start=1))
98 - prompt = f"<|user|>\n{placeholders}\n{question}<|end|>\n<|assistant|>\n"
99 - stop_token_ids = None
100 -
36 +def load_aria(question, image_urls: List[str]) -> ModelRequestData:
37 + model_name = "rhymes-ai/Aria"
38 + llm = LLM(model=model_name,
39 + tokenizer_mode="slow",
40 + trust_remote_code=True,
41 + dtype="bfloat16",
42 + limit_mm_per_prompt={"image": len(image_urls)})
43 + placeholders = "<fim_prefix><|img|><fim_suffix>\n" * len(image_urls)
44 + prompt = (f"<|im_start|>user\n{placeholders}{question}<|im_end|>\n"
45 + "<|im_start|>assistant\n")
46 + stop_token_ids = [93532, 93653, 944, 93421, 1019, 93653, 93519]
47 return ModelRequestData(
48 llm=llm,
49 prompt=prompt,
50 stop_token_ids=stop_token_ids,
51 image_data=[fetch_image(url) for url in image_urls],
106 - chat_template=None,
107 - )
52 + chat_template=None)
53
54
55 def load_h2onvl(question: str, image_urls: List[str]) -> ModelRequestData:
@@ -141,6 +86,37 @@ def load_h2onvl(question: str, image_urls: List[str]) -> ModelRequestData:
86 )
87
88
89 +def load_idefics3(question, image_urls: List[str]) -> ModelRequestData:
90 + model_name = "HuggingFaceM4/Idefics3-8B-Llama3"
91 +
92 + # The configuration below has been confirmed to launch on a single L40 GPU.
93 + llm = LLM(
94 + model=model_name,
95 + max_model_len=8192,
96 + max_num_seqs=16,
97 + enforce_eager=True,
98 + limit_mm_per_prompt={"image": len(image_urls)},
99 + # if you are running out of memory, you can reduce the "longest_edge".
100 + # see: https://huggingface.co/HuggingFaceM4/Idefics3-8B-Llama3#model-optimizations
101 + mm_processor_kwargs={
102 + "size": {
103 + "longest_edge": 2 * 364
104 + },
105 + },
106 + )
107 +
108 + placeholders = "\n".join(f"Image-{i}: <image>\n"
109 + for i, _ in enumerate(image_urls, start=1))
110 + prompt = f"<|begin_of_text|>User:{placeholders}\n{question}<end_of_utterance>\nAssistant:" # noqa: E501
111 + return ModelRequestData(
112 + llm=llm,
113 + prompt=prompt,
114 + stop_token_ids=None,
115 + image_data=[fetch_image(url) for url in image_urls],
116 + chat_template=None,
117 + )
118 +
119 +
120 def load_internvl(question: str, image_urls: List[str]) -> ModelRequestData:
121 model_name = "OpenGVLab/InternVL2-2B"
122
@@ -178,6 +154,28 @@ def load_internvl(question: str, image_urls: List[str]) -> ModelRequestData:
154 )
155
156
157 +def load_mllama(question, image_urls: List[str]) -> ModelRequestData:
158 + model_name = "meta-llama/Llama-3.2-11B-Vision-Instruct"
159 +
160 + # The configuration below has been confirmed to launch on a single L40 GPU.
161 + llm = LLM(
162 + model=model_name,
163 + max_model_len=4096,
164 + max_num_seqs=16,
165 + enforce_eager=True,
166 + limit_mm_per_prompt={"image": len(image_urls)},
167 + )
168 +
169 + prompt = f"<|image|><|image|><|begin_of_text|>{question}"
170 + return ModelRequestData(
171 + llm=llm,
172 + prompt=prompt,
173 + stop_token_ids=None,
174 + image_data=[fetch_image(url) for url in image_urls],
175 + chat_template=None,
176 + )
177 +
178 +
179 def load_nvlm_d(question: str, image_urls: List[str]):
180 model_name = "nvidia/NVLM-D-72B"
181
@@ -211,6 +209,80 @@ def load_nvlm_d(question: str, image_urls: List[str]):
209 )
210
211
212 +def load_phi3v(question: str, image_urls: List[str]) -> ModelRequestData:
213 + # num_crops is an override kwarg to the multimodal image processor;
214 + # For some models, e.g., Phi-3.5-vision-instruct, it is recommended
215 + # to use 16 for single frame scenarios, and 4 for multi-frame.
216 + #
217 + # Generally speaking, a larger value for num_crops results in more
218 + # tokens per image instance, because it may scale the image more in
219 + # the image preprocessing. Some references in the model docs and the
220 + # formula for image tokens after the preprocessing
221 + # transform can be found below.
222 + #
223 + # https://huggingface.co/microsoft/Phi-3.5-vision-instruct#loading-the-model-locally
224 + # https://huggingface.co/microsoft/Phi-3.5-vision-instruct/blob/main/processing_phi3_v.py#L194
225 + llm = LLM(
226 + model="microsoft/Phi-3.5-vision-instruct",
227 + trust_remote_code=True,
228 + max_model_len=4096,
229 + max_num_seqs=2,
230 + limit_mm_per_prompt={"image": len(image_urls)},
231 + mm_processor_kwargs={"num_crops": 4},
232 + )
233 + placeholders = "\n".join(f"<|image_{i}|>"
234 + for i, _ in enumerate(image_urls, start=1))
235 + prompt = f"<|user|>\n{placeholders}\n{question}<|end|>\n<|assistant|>\n"
236 + stop_token_ids = None
237 +
238 + return ModelRequestData(
239 + llm=llm,
240 + prompt=prompt,
241 + stop_token_ids=stop_token_ids,
242 + image_data=[fetch_image(url) for url in image_urls],
243 + chat_template=None,
244 + )
245 +
246 +
247 +def load_qwenvl_chat(question: str, image_urls: List[str]) -> ModelRequestData:
248 + model_name = "Qwen/Qwen-VL-Chat"
249 + llm = LLM(
250 + model=model_name,
251 + trust_remote_code=True,
252 + max_model_len=1024,
253 + max_num_seqs=2,
254 + limit_mm_per_prompt={"image": len(image_urls)},
255 + )
256 + placeholders = "".join(f"Picture {i}: <img></img>\n"
257 + for i, _ in enumerate(image_urls, start=1))
258 +
259 + # This model does not have a chat_template attribute on its tokenizer,
260 + # so we need to explicitly pass it. We use ChatML since it's used in the
261 + # generation utils of the model:
262 + # https://huggingface.co/Qwen/Qwen-VL-Chat/blob/main/qwen_generation_utils.py#L265
263 + tokenizer = AutoTokenizer.from_pretrained(model_name,
264 + trust_remote_code=True)
265 +
266 + # Copied from: https://huggingface.co/docs/transformers/main/en/chat_templating
267 + chat_template = "{% if not add_generation_prompt is defined %}{% set add_generation_prompt = false %}{% endif %}{% for message in messages %}{{'<|im_start|>' + message['role'] + '\n' + message['content'] + '<|im_end|>' + '\n'}}{% endfor %}{% if add_generation_prompt %}{{ '<|im_start|>assistant\n' }}{% endif %}" # noqa: E501
268 +
269 + messages = [{'role': 'user', 'content': f"{placeholders}\n{question}"}]
270 + prompt = tokenizer.apply_chat_template(messages,
271 + tokenize=False,
272 + add_generation_prompt=True,
273 + chat_template=chat_template)
274 +
275 + stop_tokens = ["<|endoftext|>", "<|im_start|>", "<|im_end|>"]
276 + stop_token_ids = [tokenizer.convert_tokens_to_ids(i) for i in stop_tokens]
277 + return ModelRequestData(
278 + llm=llm,
279 + prompt=prompt,
280 + stop_token_ids=stop_token_ids,
281 + image_data=[fetch_image(url) for url in image_urls],
282 + chat_template=chat_template,
283 + )
284 +
285 +
286 def load_qwen2_vl(question, image_urls: List[str]) -> ModelRequestData:
287 try:
288 from qwen_vl_utils import process_vision_info
@@ -268,88 +340,16 @@ def load_qwen2_vl(question, image_urls: List[str]) -> ModelRequestData:
340 )
341
342
271 -def load_mllama(question, image_urls: List[str]) -> ModelRequestData:
272 - model_name = "meta-llama/Llama-3.2-11B-Vision-Instruct"
273 -
274 - # The configuration below has been confirmed to launch on a single L40 GPU.
275 - llm = LLM(
276 - model=model_name,
277 - max_model_len=4096,
278 - max_num_seqs=16,
279 - enforce_eager=True,
280 - limit_mm_per_prompt={"image": len(image_urls)},
281 - )
282 -
283 - prompt = f"<|image|><|image|><|begin_of_text|>{question}"
284 - return ModelRequestData(
285 - llm=llm,
286 - prompt=prompt,
287 - stop_token_ids=None,
288 - image_data=[fetch_image(url) for url in image_urls],
289 - chat_template=None,
290 - )
291 -
292 -
293 -def load_idefics3(question, image_urls: List[str]) -> ModelRequestData:
294 - model_name = "HuggingFaceM4/Idefics3-8B-Llama3"
295 -
296 - # The configuration below has been confirmed to launch on a single L40 GPU.
297 - llm = LLM(
298 - model=model_name,
299 - max_model_len=8192,
300 - max_num_seqs=16,
301 - enforce_eager=True,
302 - limit_mm_per_prompt={"image": len(image_urls)},
303 - # if you are running out of memory, you can reduce the "longest_edge".
304 - # see: https://huggingface.co/HuggingFaceM4/Idefics3-8B-Llama3#model-optimizations
305 - mm_processor_kwargs={
306 - "size": {
307 - "longest_edge": 2 * 364
308 - },
309 - },
310 - )
311 -
312 - placeholders = "\n".join(f"Image-{i}: <image>\n"
313 - for i, _ in enumerate(image_urls, start=1))
314 - prompt = f"<|begin_of_text|>User:{placeholders}\n{question}<end_of_utterance>\nAssistant:" # noqa: E501
315 - return ModelRequestData(
316 - llm=llm,
317 - prompt=prompt,
318 - stop_token_ids=None,
319 - image_data=[fetch_image(url) for url in image_urls],
320 - chat_template=None,
321 - )
322 -
323 -
324 -def load_aria(question, image_urls: List[str]) -> ModelRequestData:
325 - model_name = "rhymes-ai/Aria"
326 - llm = LLM(model=model_name,
327 - tokenizer_mode="slow",
328 - trust_remote_code=True,
329 - dtype="bfloat16",
330 - limit_mm_per_prompt={"image": len(image_urls)})
331 - placeholders = "<fim_prefix><|img|><fim_suffix>\n" * len(image_urls)
332 - prompt = (f"<|im_start|>user\n{placeholders}{question}<|im_end|>\n"
333 - "<|im_start|>assistant\n")
334 - stop_token_ids = [93532, 93653, 944, 93421, 1019, 93653, 93519]
335 - return ModelRequestData(
336 - llm=llm,
337 - prompt=prompt,
338 - stop_token_ids=stop_token_ids,
339 - image_data=[fetch_image(url) for url in image_urls],
340 - chat_template=None)
341 -
342 -
343 model_example_map = {
344 - "phi3_v": load_phi3v,
344 + "aria": load_aria,
345 "h2ovl_chat": load_h2onvl,
346 + "idefics3": load_idefics3,
347 "internvl_chat": load_internvl,
348 + "mllama": load_mllama,
349 "NVLM_D": load_nvlm_d,
348 - "qwen2_vl": load_qwen2_vl,
350 + "phi3_v": load_phi3v,
351 "qwen_vl_chat": load_qwenvl_chat,
350 - "mllama": load_mllama,
351 - "idefics3": load_idefics3,
352 - "aria": load_aria,
352 + "qwen2_vl": load_qwen2_vl,
353 }
354
355