@tduncan / Nvidia-Optimization-Stack-VLLM Mirror · read-only / commits / 6eaf1e5c52
Fork 0

A high-throughput and memory-efficient inference and serving engine for LLMs

Mirrored from https://github.com/tduncan-lum/Nvidia-Optimization-Stack---VLLM.git

Synced Nov 15, 18:30 UTC

[Misc] Add `--seed` option to offline multi-modal examples (#14934)

Signed-off-by: DarkLight1337 <tlleungac@connect.ust.hk>

Cyrus Leung committed Mar 17, 2025 at 18:00 UTC 6eaf1e5c52d5e72a577ad03d378a28b39f0e849e
6 files changed +537 -315
.buildkite/test-pipeline.yaml
+5 -2
@@ -226,10 +226,13 @@ steps:
226 - python3 offline_inference/basic/chat.py
227 - python3 offline_inference/prefix_caching.py
228 - python3 offline_inference/llm_engine_example.py
229 - - python3 offline_inference/vision_language.py
230 - - python3 offline_inference/vision_language_multi_image.py
229 + - python3 offline_inference/audio_language.py --seed 0
230 + - python3 offline_inference/vision_language.py --seed 0
231 + - python3 offline_inference/vision_language_embedding.py --seed 0
232 + - python3 offline_inference/vision_language_multi_image.py --seed 0
233 - VLLM_USE_V1=0 python3 other/tensorize_vllm_model.py --model facebook/opt-125m serialize --serialized-directory /tmp/ --suffix v1 && python3 other/tensorize_vllm_model.py --model facebook/opt-125m deserialize --path-to-tensors /tmp/vllm/facebook/opt-125m/v1/model.tensors
234 - python3 offline_inference/encoder_decoder.py
235 + - python3 offline_inference/encoder_decoder_multimodal.py --model-type whisper --seed 0
236 - python3 offline_inference/basic/classify.py
237 - python3 offline_inference/basic/embed.py
238 - python3 offline_inference/basic/score.py
examples/offline_inference/audio_language.py
+88 -44
@@ -7,11 +7,13 @@ For most models, the prompt format should follow corresponding examples
7 on HuggingFace model repository.
8 """
9 import os
10 +from dataclasses import asdict
11 +from typing import NamedTuple, Optional
12
13 from huggingface_hub import snapshot_download
14 from transformers import AutoTokenizer
15
14 -from vllm import LLM, SamplingParams
16 +from vllm import LLM, EngineArgs, SamplingParams
17 from vllm.assets.audio import AudioAsset
18 from vllm.lora.request import LoRARequest
19 from vllm.utils import FlexibleArgumentParser
@@ -23,21 +25,31 @@ question_per_audio_count = {
25 2: "What sport and what nursery rhyme are referenced?"
26 }
27
28 +
29 +class ModelRequestData(NamedTuple):
30 + engine_args: EngineArgs
31 + prompt: str
32 + stop_token_ids: Optional[list[int]] = None
33 + lora_requests: Optional[list[LoRARequest]] = None
34 +
35 +
36 # NOTE: The default `max_num_seqs` and `max_model_len` may result in OOM on
37 # lower-end GPUs.
38 # Unless specified, these settings have been tested to work on a single L4.
39
40
41 # MiniCPM-O
32 -def run_minicpmo(question: str, audio_count: int):
42 +def run_minicpmo(question: str, audio_count: int) -> ModelRequestData:
43 model_name = "openbmb/MiniCPM-o-2_6"
44 tokenizer = AutoTokenizer.from_pretrained(model_name,
45 trust_remote_code=True)
36 - llm = LLM(model=model_name,
37 - trust_remote_code=True,
38 - max_model_len=4096,
39 - max_num_seqs=5,
40 - limit_mm_per_prompt={"audio": audio_count})
46 + engine_args = EngineArgs(
47 + model=model_name,
48 + trust_remote_code=True,
49 + max_model_len=4096,
50 + max_num_seqs=5,
51 + limit_mm_per_prompt={"audio": audio_count},
52 + )
53
54 stop_tokens = ['<|im_end|>', '<|endoftext|>']
55 stop_token_ids = [tokenizer.convert_tokens_to_ids(i) for i in stop_tokens]
@@ -52,11 +64,16 @@ def run_minicpmo(question: str, audio_count: int):
64 tokenize=False,
65 add_generation_prompt=True,
66 chat_template=audio_chat_template)
55 - return llm, prompt, stop_token_ids
67 +
68 + return ModelRequestData(
69 + engine_args=engine_args,
70 + prompt=prompt,
71 + stop_token_ids=stop_token_ids,
72 + )
73
74
75 # Phi-4-multimodal-instruct
59 -def run_phi4mm(questions: str, audio_count: int):
76 +def run_phi4mm(question: str, audio_count: int) -> ModelRequestData:
77 """
78 Phi-4-multimodal-instruct supports both image and audio inputs. Here, we
79 show how to process audio inputs.
@@ -67,9 +84,9 @@ def run_phi4mm(questions: str, audio_count: int):
84 speech_lora_path = os.path.join(model_path, "speech-lora")
85 placeholders = "".join([f"<|audio_{i+1}|>" for i in range(audio_count)])
86
70 - prompts = f"<|user|>{placeholders}{questions}<|end|><|assistant|>"
87 + prompts = f"<|user|>{placeholders}{question}<|end|><|assistant|>"
88
72 - llm = LLM(
89 + engine_args = EngineArgs(
90 model=model_path,
91 trust_remote_code=True,
92 max_model_len=4096,
@@ -79,24 +96,24 @@ def run_phi4mm(questions: str, audio_count: int):
96 lora_extra_vocab_size=0,
97 limit_mm_per_prompt={"audio": audio_count},
98 )
82 - lora_request = LoRARequest("speech", 1, speech_lora_path)
83 - # To maintain code compatibility in this script, we add LoRA here.
84 - llm.llm_engine.add_lora(lora_request=lora_request)
85 - # You can also add LoRA using:
86 - # llm.generate(prompts, lora_request=lora_request,...)
99
88 - stop_token_ids = None
89 - return llm, prompts, stop_token_ids
100 + return ModelRequestData(
101 + engine_args=engine_args,
102 + prompt=prompts,
103 + lora_requests=[LoRARequest("speech", 1, speech_lora_path)],
104 + )
105
106
107 # Qwen2-Audio
93 -def run_qwen2_audio(question: str, audio_count: int):
108 +def run_qwen2_audio(question: str, audio_count: int) -> ModelRequestData:
109 model_name = "Qwen/Qwen2-Audio-7B-Instruct"
110
96 - llm = LLM(model=model_name,
97 - max_model_len=4096,
98 - max_num_seqs=5,
99 - limit_mm_per_prompt={"audio": audio_count})
111 + engine_args = EngineArgs(
112 + model=model_name,
113 + max_model_len=4096,
114 + max_num_seqs=5,
115 + limit_mm_per_prompt={"audio": audio_count},
116 + )
117
118 audio_in_prompt = "".join([
119 f"Audio {idx+1}: "
@@ -107,12 +124,15 @@ def run_qwen2_audio(question: str, audio_count: int):
124 "<|im_start|>user\n"
125 f"{audio_in_prompt}{question}<|im_end|>\n"
126 "<|im_start|>assistant\n")
110 - stop_token_ids = None
111 - return llm, prompt, stop_token_ids
127 +
128 + return ModelRequestData(
129 + engine_args=engine_args,
130 + prompt=prompt,
131 + )
132
133
134 # Ultravox 0.5-1B
115 -def run_ultravox(question: str, audio_count: int):
135 +def run_ultravox(question: str, audio_count: int) -> ModelRequestData:
136 model_name = "fixie-ai/ultravox-v0_5-llama-3_2-1b"
137
138 tokenizer = AutoTokenizer.from_pretrained(model_name)
@@ -124,29 +144,39 @@ def run_ultravox(question: str, audio_count: int):
144 tokenize=False,
145 add_generation_prompt=True)
146
127 - llm = LLM(model=model_name,
128 - max_model_len=4096,
129 - max_num_seqs=5,
130 - trust_remote_code=True,
131 - limit_mm_per_prompt={"audio": audio_count})
132 - stop_token_ids = None
133 - return llm, prompt, stop_token_ids
147 + engine_args = EngineArgs(
148 + model=model_name,
149 + max_model_len=4096,
150 + max_num_seqs=5,
151 + trust_remote_code=True,
152 + limit_mm_per_prompt={"audio": audio_count},
153 + )
154 +
155 + return ModelRequestData(
156 + engine_args=engine_args,
157 + prompt=prompt,
158 + )
159
160
161 # Whisper
137 -def run_whisper(question: str, audio_count: int):
162 +def run_whisper(question: str, audio_count: int) -> ModelRequestData:
163 assert audio_count == 1, (
164 "Whisper only support single audio input per prompt")
165 model_name = "openai/whisper-large-v3-turbo"
166
167 prompt = "<|startoftranscript|>"
168
144 - llm = LLM(model=model_name,
145 - max_model_len=448,
146 - max_num_seqs=5,
147 - limit_mm_per_prompt={"audio": audio_count})
148 - stop_token_ids = None
149 - return llm, prompt, stop_token_ids
169 + engine_args = EngineArgs(
170 + model=model_name,
171 + max_model_len=448,
172 + max_num_seqs=5,
173 + limit_mm_per_prompt={"audio": audio_count},
174 + )
175 +
176 + return ModelRequestData(
177 + engine_args=engine_args,
178 + prompt=prompt,
179 + )
180
181
182 model_example_map = {
@@ -164,14 +194,24 @@ def main(args):
194 raise ValueError(f"Model type {model} is not supported.")
195
196 audio_count = args.num_audios
167 - llm, prompt, stop_token_ids = model_example_map[model](
168 - question_per_audio_count[audio_count], audio_count)
197 + req_data = model_example_map[model](question_per_audio_count[audio_count],
198 + audio_count)
199 +
200 + engine_args = asdict(req_data.engine_args) | {"seed": args.seed}
201 + llm = LLM(**engine_args)
202 +
203 + # To maintain code compatibility in this script, we add LoRA here.
204 + # You can also add LoRA using:
205 + # llm.generate(prompts, lora_request=lora_request,...)
206 + if req_data.lora_requests:
207 + for lora_request in req_data.lora_requests:
208 + llm.llm_engine.add_lora(lora_request=lora_request)
209
210 # We set temperature to 0.2 so that outputs can be different
211 # even when all prompts are identical when running batch inference.
212 sampling_params = SamplingParams(temperature=0.2,
213 max_tokens=64,
174 - stop_token_ids=stop_token_ids)
214 + stop_token_ids=req_data.stop_token_ids)
215
216 mm_data = {}
217 if audio_count > 0:
@@ -183,7 +223,7 @@ def main(args):
223 }
224
225 assert args.num_prompts > 0
186 - inputs = {"prompt": prompt, "multi_modal_data": mm_data}
226 + inputs = {"prompt": req_data.prompt, "multi_modal_data": mm_data}
227 if args.num_prompts > 1:
228 # Batch inference
229 inputs = [inputs] * args.num_prompts
@@ -214,6 +254,10 @@ if __name__ == "__main__":
254 default=1,
255 choices=[0, 1, 2],
256 help="Number of audio items per prompt.")
257 + parser.add_argument("--seed",
258 + type=int,
259 + default=None,
260 + help="Set the seed when initializing `vllm.LLM`.")
261
262 args = parser.parse_args()
263 main(args)
examples/offline_inference/encoder_decoder_multimodal.py
+37 -11
@@ -4,16 +4,23 @@ This example shows how to use vLLM for running offline inference with
4 the explicit/implicit prompt format on enc-dec LMMs for text generation.
5 """
6 import time
7 +from collections.abc import Sequence
8 +from dataclasses import asdict
9 +from typing import NamedTuple
10
8 -from vllm import LLM, SamplingParams
11 +from vllm import LLM, EngineArgs, PromptType, SamplingParams
12 from vllm.assets.audio import AudioAsset
13 from vllm.assets.image import ImageAsset
14 from vllm.utils import FlexibleArgumentParser
15
16
17 +class ModelRequestData(NamedTuple):
18 + engine_args: EngineArgs
19 + prompts: Sequence[PromptType]
20 +
21 +
22 def run_florence2():
15 - # Create a Florence-2 encoder/decoder model instance
16 - llm = LLM(
23 + engine_args = EngineArgs(
24 model="microsoft/Florence-2-large",
25 tokenizer="facebook/bart-large",
26 max_num_seqs=8,
@@ -39,12 +46,15 @@ def run_florence2():
46 "decoder_prompt": "",
47 },
48 ]
42 - return llm, prompts
49 +
50 + return ModelRequestData(
51 + engine_args=engine_args,
52 + prompts=prompts,
53 + )
54
55
56 def run_mllama():
46 - # Create a Mllama encoder/decoder model instance
47 - llm = LLM(
57 + engine_args = EngineArgs(
58 model="meta-llama/Llama-3.2-11B-Vision-Instruct",
59 max_model_len=4096,
60 max_num_seqs=2,
@@ -69,12 +79,15 @@ def run_mllama():
79 "decoder_prompt": "<|image|><|begin_of_text|>Please describe the image.", # noqa: E501
80 },
81 ]
72 - return llm, prompts
82 +
83 + return ModelRequestData(
84 + engine_args=engine_args,
85 + prompts=prompts,
86 + )
87
88
89 def run_whisper():
76 - # Create a Whisper encoder/decoder model instance
77 - llm = LLM(
90 + engine_args = EngineArgs(
91 model="openai/whisper-large-v3-turbo",
92 max_model_len=448,
93 max_num_seqs=16,
@@ -99,7 +112,11 @@ def run_whisper():
112 "decoder_prompt": "<|startoftranscript|>",
113 }
114 ]
102 - return llm, prompts
115 +
116 + return ModelRequestData(
117 + engine_args=engine_args,
118 + prompts=prompts,
119 + )
120
121
122 model_example_map = {
@@ -114,7 +131,12 @@ def main(args):
131 if model not in model_example_map:
132 raise ValueError(f"Model type {model} is not supported.")
133
117 - llm, prompts = model_example_map[model]()
134 + req_data = model_example_map[model]()
135 +
136 + engine_args = asdict(req_data.engine_args) | {"seed": args.seed}
137 + llm = LLM(**engine_args)
138 +
139 + prompts = req_data.prompts
140
141 # Create a sampling params object.
142 sampling_params = SamplingParams(
@@ -153,6 +175,10 @@ if __name__ == "__main__":
175 default="mllama",
176 choices=model_example_map.keys(),
177 help='Huggingface "model_type".')
178 + parser.add_argument("--seed",
179 + type=int,
180 + default=None,
181 + help="Set the seed when initializing `vllm.LLM`.")
182
183 args = parser.parse_args()
184 main(args)
examples/offline_inference/vision_language.py
+298 -157
@@ -8,122 +8,164 @@ on HuggingFace model repository.
8 """
9 import os
10 import random
11 +from dataclasses import asdict
12 +from typing import NamedTuple, Optional
13
14 from huggingface_hub import snapshot_download
15 from transformers import AutoTokenizer
16
15 -from vllm import LLM, SamplingParams
17 +from vllm import LLM, EngineArgs, SamplingParams
18 from vllm.assets.image import ImageAsset
19 from vllm.assets.video import VideoAsset
20 from vllm.lora.request import LoRARequest
21 from vllm.utils import FlexibleArgumentParser
22
23 +
24 +class ModelRequestData(NamedTuple):
25 + engine_args: EngineArgs
26 + prompts: list[str]
27 + stop_token_ids: Optional[list[int]] = None
28 + lora_requests: Optional[list[LoRARequest]] = None
29 +
30 +
31 # NOTE: The default `max_num_seqs` and `max_model_len` may result in OOM on
32 # lower-end GPUs.
33 # Unless specified, these settings have been tested to work on a single L4.
34
35
36 # Aria
27 -def run_aria(questions: list[str], modality: str):
37 +def run_aria(questions: list[str], modality: str) -> ModelRequestData:
38 assert modality == "image"
39 model_name = "rhymes-ai/Aria"
40
41 # NOTE: Need L40 (or equivalent) to avoid OOM
32 - llm = LLM(model=model_name,
33 - max_model_len=4096,
34 - max_num_seqs=2,
35 - dtype="bfloat16",
36 - disable_mm_preprocessor_cache=args.disable_mm_preprocessor_cache)
42 + engine_args = EngineArgs(
43 + model=model_name,
44 + max_model_len=4096,
45 + max_num_seqs=2,
46 + dtype="bfloat16",
47 + disable_mm_preprocessor_cache=args.disable_mm_preprocessor_cache,
48 + )
49
50 prompts = [(f"<|im_start|>user\n<fim_prefix><|img|><fim_suffix>{question}"
51 "<|im_end|>\n<|im_start|>assistant\n")
52 for question in questions]
53
54 stop_token_ids = [93532, 93653, 944, 93421, 1019, 93653, 93519]
43 - return llm, prompts, stop_token_ids
55 +
56 + return ModelRequestData(
57 + engine_args=engine_args,
58 + prompts=prompts,
59 + stop_token_ids=stop_token_ids,
60 + )
61
62
63 # BLIP-2
47 -def run_blip2(questions: list[str], modality: str):
64 +def run_blip2(questions: list[str], modality: str) -> ModelRequestData:
65 assert modality == "image"
66
67 # BLIP-2 prompt format is inaccurate on HuggingFace model repository.
68 # See https://huggingface.co/Salesforce/blip2-opt-2.7b/discussions/15#64ff02f3f8cf9e4f5b038262 #noqa
69 prompts = [f"Question: {question} Answer:" for question in questions]
53 - llm = LLM(model="Salesforce/blip2-opt-2.7b",
54 - disable_mm_preprocessor_cache=args.disable_mm_preprocessor_cache)
55 - stop_token_ids = None
56 - return llm, prompts, stop_token_ids
70 + engine_args = EngineArgs(
71 + model="Salesforce/blip2-opt-2.7b",
72 + disable_mm_preprocessor_cache=args.disable_mm_preprocessor_cache,
73 + )
74 +
75 + return ModelRequestData(
76 + engine_args=engine_args,
77 + prompts=prompts,
78 + )
79
80
81 # Chameleon
60 -def run_chameleon(questions: list[str], modality: str):
82 +def run_chameleon(questions: list[str], modality: str) -> ModelRequestData:
83 assert modality == "image"
84
85 prompts = [f"{question}<image>" for question in questions]
64 - llm = LLM(model="facebook/chameleon-7b",
65 - max_model_len=4096,
66 - max_num_seqs=2,
67 - disable_mm_preprocessor_cache=args.disable_mm_preprocessor_cache)
68 - stop_token_ids = None
69 - return llm, prompts, stop_token_ids
86 + engine_args = EngineArgs(
87 + model="facebook/chameleon-7b",
88 + max_model_len=4096,
89 + max_num_seqs=2,
90 + disable_mm_preprocessor_cache=args.disable_mm_preprocessor_cache,
91 + )
92 +
93 + return ModelRequestData(
94 + engine_args=engine_args,
95 + prompts=prompts,
96 + )
97
98
99 # Deepseek-VL2
73 -def run_deepseek_vl2(questions: list[str], modality: str):
100 +def run_deepseek_vl2(questions: list[str], modality: str) -> ModelRequestData:
101 assert modality == "image"
102
103 model_name = "deepseek-ai/deepseek-vl2-tiny"
104
78 - llm = LLM(model=model_name,
79 - max_model_len=4096,
80 - max_num_seqs=2,
81 - disable_mm_preprocessor_cache=args.disable_mm_preprocessor_cache,
82 - hf_overrides={"architectures": ["DeepseekVLV2ForCausalLM"]})
105 + engine_args = EngineArgs(
106 + model=model_name,
107 + max_model_len=4096,
108 + max_num_seqs=2,
109 + disable_mm_preprocessor_cache=args.disable_mm_preprocessor_cache,
110 + hf_overrides={"architectures": ["DeepseekVLV2ForCausalLM"]},
111 + )
112
113 prompts = [
114 f"<|User|>: <image>\n{question}\n\n<|Assistant|>:"
115 for question in questions
116 ]
88 - stop_token_ids = None
89 - return llm, prompts, stop_token_ids
117 +
118 + return ModelRequestData(
119 + engine_args=engine_args,
120 + prompts=prompts,
121 + )
122
123
124 # Florence2
93 -def run_florence2(question: str, modality: str):
125 +def run_florence2(questions: list[str], modality: str) -> ModelRequestData:
126 assert modality == "image"
127
96 - llm = LLM(model="microsoft/Florence-2-large",
97 - tokenizer="facebook/bart-large",
98 - max_num_seqs=8,
99 - trust_remote_code=True,
100 - dtype="bfloat16",
101 - disable_mm_preprocessor_cache=args.disable_mm_preprocessor_cache)
128 + engine_args = EngineArgs(
129 + model="microsoft/Florence-2-large",
130 + tokenizer="facebook/bart-large",
131 + max_num_seqs=8,
132 + trust_remote_code=True,
133 + dtype="bfloat16",
134 + disable_mm_preprocessor_cache=args.disable_mm_preprocessor_cache,
135 + )
136
103 - prompt = "<MORE_DETAILED_CAPTION>"
104 - stop_token_ids = None
105 - return llm, prompt, stop_token_ids
137 + prompts = ["<MORE_DETAILED_CAPTION>" for _ in questions]
138 +
139 + return ModelRequestData(
140 + engine_args=engine_args,
141 + prompts=prompts,
142 + )
143
144
145 # Fuyu
109 -def run_fuyu(questions: list[str], modality: str):
146 +def run_fuyu(questions: list[str], modality: str) -> ModelRequestData:
147 assert modality == "image"
148
149 prompts = [f"{question}\n" for question in questions]
113 - llm = LLM(model="adept/fuyu-8b",
114 - max_model_len=2048,
115 - max_num_seqs=2,
116 - disable_mm_preprocessor_cache=args.disable_mm_preprocessor_cache)
117 - stop_token_ids = None
118 - return llm, prompts, stop_token_ids
150 + engine_args = EngineArgs(
151 + model="adept/fuyu-8b",
152 + max_model_len=2048,
153 + max_num_seqs=2,
154 + disable_mm_preprocessor_cache=args.disable_mm_preprocessor_cache,
155 + )
156 +
157 + return ModelRequestData(
158 + engine_args=engine_args,
159 + prompts=prompts,
160 + )
161
162
163 # Gemma 3
122 -def run_gemma3(questions: list[str], modality: str):
164 +def run_gemma3(questions: list[str], modality: str) -> ModelRequestData:
165 assert modality == "image"
166 model_name = "google/gemma-3-4b-it"
167
126 - llm = LLM(
168 + engine_args = EngineArgs(
169 model=model_name,
170 max_model_len=2048,
171 max_num_seqs=2,
@@ -135,22 +177,27 @@ def run_gemma3(questions: list[str], modality: str):
177 prompts = [("<bos><start_of_turn>user\n"
178 f"<start_of_image>{question}<end_of_turn>\n"
179 "<start_of_turn>model\n") for question in questions]
138 - stop_token_ids = None
139 - return llm, prompts, stop_token_ids
180 +
181 + return ModelRequestData(
182 + engine_args=engine_args,
183 + prompts=prompts,
184 + )
185
186
187 # GLM-4v
143 -def run_glm4v(questions: list[str], modality: str):
188 +def run_glm4v(questions: list[str], modality: str) -> ModelRequestData:
189 assert modality == "image"
190 model_name = "THUDM/glm-4v-9b"
191
147 - llm = LLM(model=model_name,
148 - max_model_len=2048,
149 - max_num_seqs=2,
150 - trust_remote_code=True,
151 - enforce_eager=True,
152 - hf_overrides={"architectures": ["GLM4VForCausalLM"]},
153 - disable_mm_preprocessor_cache=args.disable_mm_preprocessor_cache)
192 + engine_args = EngineArgs(
193 + model=model_name,
194 + max_model_len=2048,
195 + max_num_seqs=2,
196 + trust_remote_code=True,
197 + enforce_eager=True,
198 + hf_overrides={"architectures": ["GLM4VForCausalLM"]},
199 + disable_mm_preprocessor_cache=args.disable_mm_preprocessor_cache,
200 + )
201
202 prompts = [
203 f"<|user|>\n<|begin_of_image|><|endoftext|><|end_of_image|>\
@@ -158,16 +205,21 @@ def run_glm4v(questions: list[str], modality: str):
205 ]
206
207 stop_token_ids = [151329, 151336, 151338]
161 - return llm, prompts, stop_token_ids
208 +
209 + return ModelRequestData(
210 + engine_args=engine_args,
211 + prompts=prompts,
212 + stop_token_ids=stop_token_ids,
213 + )
214
215
216 # H2OVL-Mississippi
165 -def run_h2ovl(questions: list[str], modality: str):
217 +def run_h2ovl(questions: list[str], modality: str) -> ModelRequestData:
218 assert modality == "image"
219
220 model_name = "h2oai/h2ovl-mississippi-800m"
221
170 - llm = LLM(
222 + engine_args = EngineArgs(
223 model=model_name,
224 trust_remote_code=True,
225 max_model_len=8192,
@@ -187,15 +239,20 @@ def run_h2ovl(questions: list[str], modality: str):
239 # Stop tokens for H2OVL-Mississippi
240 # https://huggingface.co/h2oai/h2ovl-mississippi-800m
241 stop_token_ids = [tokenizer.eos_token_id]
190 - return llm, prompts, stop_token_ids
242 +
243 + return ModelRequestData(
244 + engine_args=engine_args,
245 + prompts=prompts,
246 + stop_token_ids=stop_token_ids,
247 + )
248
249
250 # Idefics3-8B-Llama3
194 -def run_idefics3(questions: list[str], modality: str):
251 +def run_idefics3(questions: list[str], modality: str) -> ModelRequestData:
252 assert modality == "image"
253 model_name = "HuggingFaceM4/Idefics3-8B-Llama3"
254
198 - llm = LLM(
255 + engine_args = EngineArgs(
256 model=model_name,
257 max_model_len=8192,
258 max_num_seqs=2,
@@ -212,17 +269,20 @@ def run_idefics3(questions: list[str], modality: str):
269 prompts = [(
270 f"<|begin_of_text|>User:<image>{question}<end_of_utterance>\nAssistant:"
271 ) for question in questions]
215 - stop_token_ids = None
216 - return llm, prompts, stop_token_ids
272 +
273 + return ModelRequestData(
274 + engine_args=engine_args,
275 + prompts=prompts,
276 + )
277
278
279 # InternVL
220 -def run_internvl(questions: list[str], modality: str):
280 +def run_internvl(questions: list[str], modality: str) -> ModelRequestData:
281 assert modality == "image"
282
283 model_name = "OpenGVLab/InternVL2-2B"
284
225 - llm = LLM(
285 + engine_args = EngineArgs(
286 model=model_name,
287 trust_remote_code=True,
288 max_model_len=4096,
@@ -245,53 +305,75 @@ def run_internvl(questions: list[str], modality: str):
305 # https://huggingface.co/OpenGVLab/InternVL2-2B/blob/main/conversation.py
306 stop_tokens = ["<|endoftext|>", "<|im_start|>", "<|im_end|>", "<|end|>"]
307 stop_token_ids = [tokenizer.convert_tokens_to_ids(i) for i in stop_tokens]
248 - return llm, prompts, stop_token_ids
308 +
309 + return ModelRequestData(
310 + engine_args=engine_args,
311 + prompts=prompts,
312 + stop_token_ids=stop_token_ids,
313 + )
314
315
316 # LLaVA-1.5
252 -def run_llava(questions: list[str], modality: str):
317 +def run_llava(questions: list[str], modality: str) -> ModelRequestData:
318 assert modality == "image"
319
320 prompts = [
321 f"USER: <image>\n{question}\nASSISTANT:" for question in questions
322 ]
323
259 - llm = LLM(model="llava-hf/llava-1.5-7b-hf",
260 - max_model_len=4096,
261 - disable_mm_preprocessor_cache=args.disable_mm_preprocessor_cache)
262 - stop_token_ids = None
263 - return llm, prompts, stop_token_ids
324 + engine_args = EngineArgs(
325 + model="llava-hf/llava-1.5-7b-hf",
326 + max_model_len=4096,
327 + disable_mm_preprocessor_cache=args.disable_mm_preprocessor_cache,
328 + )
329 +
330 + return ModelRequestData(
331 + engine_args=engine_args,
332 + prompts=prompts,
333 + )
334
335
336 # LLaVA-1.6/LLaVA-NeXT
267 -def run_llava_next(questions: list[str], modality: str):
337 +def run_llava_next(questions: list[str], modality: str) -> ModelRequestData:
338 assert modality == "image"
339
340 prompts = [f"[INST] <image>\n{question} [/INST]" for question in questions]
271 - llm = LLM(model="llava-hf/llava-v1.6-mistral-7b-hf",
272 - max_model_len=8192,
273 - disable_mm_preprocessor_cache=args.disable_mm_preprocessor_cache)
274 - stop_token_ids = None
275 - return llm, prompts, stop_token_ids
341 + engine_args = EngineArgs(
342 + model="llava-hf/llava-v1.6-mistral-7b-hf",
343 + max_model_len=8192,
344 + disable_mm_preprocessor_cache=args.disable_mm_preprocessor_cache,
345 + )
346 +
347 + return ModelRequestData(
348 + engine_args=engine_args,
349 + prompts=prompts,
350 + )
351
352
353 # LlaVA-NeXT-Video
354 # Currently only support for video input
280 -def run_llava_next_video(questions: list[str], modality: str):
355 +def run_llava_next_video(questions: list[str],
356 + modality: str) -> ModelRequestData:
357 assert modality == "video"
358
359 prompts = [
360 f"USER: <video>\n{question} ASSISTANT:" for question in questions
361 ]
286 - llm = LLM(model="llava-hf/LLaVA-NeXT-Video-7B-hf",
287 - max_model_len=8192,
288 - disable_mm_preprocessor_cache=args.disable_mm_preprocessor_cache)
289 - stop_token_ids = None
290 - return llm, prompts, stop_token_ids
362 + engine_args = EngineArgs(
363 + model="llava-hf/LLaVA-NeXT-Video-7B-hf",
364 + max_model_len=8192,
365 + disable_mm_preprocessor_cache=args.disable_mm_preprocessor_cache,
366 + )
367 +
368 + return ModelRequestData(
369 + engine_args=engine_args,
370 + prompts=prompts,
371 + )
372
373
374 # LLaVA-OneVision
294 -def run_llava_onevision(questions: list[str], modality: str):
375 +def run_llava_onevision(questions: list[str],
376 + modality: str) -> ModelRequestData:
377
378 if modality == "video":
379 prompts = [
@@ -305,15 +387,20 @@ def run_llava_onevision(questions: list[str], modality: str):
387 <|im_start|>assistant\n" for question in questions
388 ]
389
308 - llm = LLM(model="llava-hf/llava-onevision-qwen2-7b-ov-hf",
309 - max_model_len=16384,
310 - disable_mm_preprocessor_cache=args.disable_mm_preprocessor_cache)
311 - stop_token_ids = None
312 - return llm, prompts, stop_token_ids
390 + engine_args = EngineArgs(
391 + model="llava-hf/llava-onevision-qwen2-7b-ov-hf",
392 + max_model_len=16384,
393 + disable_mm_preprocessor_cache=args.disable_mm_preprocessor_cache,
394 + )
395 +
396 + return ModelRequestData(
397 + engine_args=engine_args,
398 + prompts=prompts,
399 + )
400
401
402 # Mantis
316 -def run_mantis(questions: list[str], modality: str):
403 +def run_mantis(questions: list[str], modality: str) -> ModelRequestData:
404 assert modality == "image"
405
406 llama3_template = '<|start_header_id|>user<|end_header_id|>\n\n{}<|eot_id|><|start_header_id|>assistant<|end_header_id|>\n\n' # noqa: E501
@@ -322,14 +409,19 @@ def run_mantis(questions: list[str], modality: str):
409 for question in questions
410 ]
411
325 - llm = LLM(
412 + engine_args = EngineArgs(
413 model="TIGER-Lab/Mantis-8B-siglip-llama3",
414 max_model_len=4096,
415 hf_overrides={"architectures": ["MantisForConditionalGeneration"]},
416 disable_mm_preprocessor_cache=args.disable_mm_preprocessor_cache,
417 )
418 stop_token_ids = [128009]
332 - return llm, prompts, stop_token_ids
419 +
420 + return ModelRequestData(
421 + engine_args=engine_args,
422 + prompts=prompts,
423 + stop_token_ids=stop_token_ids,
424 + )
425
426
427 # MiniCPM-V
@@ -357,7 +449,7 @@ def run_minicpmv_base(questions: list[str], modality: str, model_name):
449 # model_name = "openbmb/MiniCPM-o-2_6"
450 tokenizer = AutoTokenizer.from_pretrained(model_name,
451 trust_remote_code=True)
360 - llm = LLM(
452 + engine_args = EngineArgs(
453 model=model_name,
454 max_model_len=4096,
455 max_num_seqs=2,
@@ -389,19 +481,24 @@ def run_minicpmv_base(questions: list[str], modality: str, model_name):
481 tokenize=False,
482 add_generation_prompt=True) for question in questions
483 ]
392 - return llm, prompts, stop_token_ids
484 +
485 + return ModelRequestData(
486 + engine_args=engine_args,
487 + prompts=prompts,
488 + stop_token_ids=stop_token_ids,
489 + )
490
491
395 -def run_minicpmo(questions: list[str], modality: str):
492 +def run_minicpmo(questions: list[str], modality: str) -> ModelRequestData:
493 return run_minicpmv_base(questions, modality, "openbmb/MiniCPM-o-2_6")
494
495
399 -def run_minicpmv(questions: list[str], modality: str):
496 +def run_minicpmv(questions: list[str], modality: str) -> ModelRequestData:
497 return run_minicpmv_base(questions, modality, "openbmb/MiniCPM-V-2_6")
498
499
500 # LLama 3.2
404 -def run_mllama(questions: list[str], modality: str):
501 +def run_mllama(questions: list[str], modality: str) -> ModelRequestData:
502 assert modality == "image"
503
504 model_name = "meta-llama/Llama-3.2-11B-Vision-Instruct"
@@ -411,7 +508,7 @@ def run_mllama(questions: list[str], modality: str):
508 # You may lower either to run this example on lower-end GPUs.
509
510 # The configuration below has been confirmed to launch on a single L40 GPU.
414 - llm = LLM(
511 + engine_args = EngineArgs(
512 model=model_name,
513 max_model_len=4096,
514 max_num_seqs=16,
@@ -432,17 +529,20 @@ def run_mllama(questions: list[str], modality: str):
529 prompts = tokenizer.apply_chat_template(messages,
530 add_generation_prompt=True,
531 tokenize=False)
435 - stop_token_ids = None
436 - return llm, prompts, stop_token_ids
532 +
533 + return ModelRequestData(
534 + engine_args=engine_args,
535 + prompts=prompts,
536 + )
537
538
539 # Molmo
440 -def run_molmo(questions: list[str], modality: str):
540 +def run_molmo(questions: list[str], modality: str) -> ModelRequestData:
541 assert modality == "image"
542
543 model_name = "allenai/Molmo-7B-D-0924"
544
445 - llm = LLM(
545 + engine_args = EngineArgs(
546 model=model_name,
547 trust_remote_code=True,
548 dtype="bfloat16",
@@ -453,18 +553,21 @@ def run_molmo(questions: list[str], modality: str):
553 f"<|im_start|>user <image>\n{question}<|im_end|> \
554 <|im_start|>assistant\n" for question in questions
555 ]
456 - stop_token_ids = None
457 - return llm, prompts, stop_token_ids
556 +
557 + return ModelRequestData(
558 + engine_args=engine_args,
559 + prompts=prompts,
560 + )
561
562
563 # NVLM-D
461 -def run_nvlm_d(questions: list[str], modality: str):
564 +def run_nvlm_d(questions: list[str], modality: str) -> ModelRequestData:
565 assert modality == "image"
566
567 model_name = "nvidia/NVLM-D-72B"
568
569 # Adjust this as necessary to fit in GPU
467 - llm = LLM(
570 + engine_args = EngineArgs(
571 model=model_name,
572 trust_remote_code=True,
573 max_model_len=4096,
@@ -481,36 +584,47 @@ def run_nvlm_d(questions: list[str], modality: str):
584 prompts = tokenizer.apply_chat_template(messages,
585 tokenize=False,
586 add_generation_prompt=True)
484 - stop_token_ids = None
485 - return llm, prompts, stop_token_ids
587 +
588 + return ModelRequestData(
589 + engine_args=engine_args,
590 + prompts=prompts,
591 + )
592
593
594 # PaliGemma
489 -def run_paligemma(question: str, modality: str):
595 +def run_paligemma(questions: list[str], modality: str) -> ModelRequestData:
596 assert modality == "image"
597
598 # PaliGemma has special prompt format for VQA
493 - prompt = ["caption en"]
494 - llm = LLM(model="google/paligemma-3b-mix-224",
495 - disable_mm_preprocessor_cache=args.disable_mm_preprocessor_cache)
496 - stop_token_ids = None
497 - return llm, prompt, stop_token_ids
599 + prompts = ["caption en" for _ in questions]
600 + engine_args = EngineArgs(
601 + model="google/paligemma-3b-mix-224",
602 + disable_mm_preprocessor_cache=args.disable_mm_preprocessor_cache)
603 +
604 + return ModelRequestData(
605 + engine_args=engine_args,
606 + prompts=prompts,
607 + )
608
609
610 # PaliGemma 2
501 -def run_paligemma2(question: str, modality: str):
611 +def run_paligemma2(questions: list[str], modality: str) -> ModelRequestData:
612 assert modality == "image"
613
614 # PaliGemma 2 has special prompt format for VQA
505 - prompt = ["caption en"]
506 - llm = LLM(model="google/paligemma2-3b-ft-docci-448",
507 - disable_mm_preprocessor_cache=args.disable_mm_preprocessor_cache)
508 - stop_token_ids = None
509 - return llm, prompt, stop_token_ids
615 + prompts = ["caption en" for _ in questions]
616 + engine_args = EngineArgs(
617 + model="google/paligemma2-3b-ft-docci-448",
618 + disable_mm_preprocessor_cache=args.disable_mm_preprocessor_cache)
619 +
620 + return ModelRequestData(
621 + engine_args=engine_args,
622 + prompts=prompts,
623 + )
624
625
626 # Phi-3-Vision
513 -def run_phi3v(questions: list[str], modality: str):
627 +def run_phi3v(questions: list[str], modality: str) -> ModelRequestData:
628 assert modality == "image"
629
630 prompts = [
@@ -530,7 +644,7 @@ def run_phi3v(questions: list[str], modality: str):
644 #
645 # https://huggingface.co/microsoft/Phi-3.5-vision-instruct#loading-the-model-locally
646 # https://huggingface.co/microsoft/Phi-3.5-vision-instruct/blob/main/processing_phi3_v.py#L194
533 - llm = LLM(
647 + engine_args = EngineArgs(
648 model="microsoft/Phi-3.5-vision-instruct",
649 trust_remote_code=True,
650 max_model_len=4096,
@@ -539,12 +653,15 @@ def run_phi3v(questions: list[str], modality: str):
653 mm_processor_kwargs={"num_crops": 16},
654 disable_mm_preprocessor_cache=args.disable_mm_preprocessor_cache,
655 )
542 - stop_token_ids = None
543 - return llm, prompts, stop_token_ids
656 +
657 + return ModelRequestData(
658 + engine_args=engine_args,
659 + prompts=prompts,
660 + )
661
662
663 # Phi-4-multimodal-instruct
547 -def run_phi4mm(questions: list[str], modality: str):
664 +def run_phi4mm(questions: list[str], modality: str) -> ModelRequestData:
665 """
666 Phi-4-multimodal-instruct supports both image and audio inputs. Here, we
667 show how to process image inputs.
@@ -558,7 +675,7 @@ def run_phi4mm(questions: list[str], modality: str):
675 f"<|user|><|image_1|>{question}<|end|><|assistant|>"
676 for question in questions
677 ]
561 - llm = LLM(
678 + engine_args = EngineArgs(
679 model=model_path,
680 trust_remote_code=True,
681 max_model_len=4096,
@@ -567,24 +684,22 @@ def run_phi4mm(questions: list[str], modality: str):
684 max_lora_rank=320,
685 lora_extra_vocab_size=0,
686 )
570 - lora_request = LoRARequest("vision", 1, vision_lora_path)
571 - # To maintain code compatibility in this script, we add LoRA here.
572 - llm.llm_engine.add_lora(lora_request=lora_request)
573 - # You can also add LoRA using:
574 - # llm.generate(prompts, lora_request=lora_request,...)
687
576 - stop_token_ids = None
577 - return llm, prompts, stop_token_ids
688 + return ModelRequestData(
689 + engine_args=engine_args,
690 + prompts=prompts,
691 + lora_requests=[LoRARequest("vision", 1, vision_lora_path)],
692 + )
693
694
695 # Pixtral HF-format
581 -def run_pixtral_hf(questions: list[str], modality: str):
696 +def run_pixtral_hf(questions: list[str], modality: str) -> ModelRequestData:
697 assert modality == "image"
698
699 model_name = "mistral-community/pixtral-12b"
700
701 # NOTE: Need L40 (or equivalent) to avoid OOM
587 - llm = LLM(
702 + engine_args = EngineArgs(
703 model=model_name,
704 max_model_len=8192,
705 max_num_seqs=2,
@@ -592,15 +707,18 @@ def run_pixtral_hf(questions: list[str], modality: str):
707 )
708
709 prompts = [f"<s>[INST]{question}\n[IMG][/INST]" for question in questions]
595 - stop_token_ids = None
596 - return llm, prompts, stop_token_ids
710 +
711 + return ModelRequestData(
712 + engine_args=engine_args,
713 + prompts=prompts,
714 + )
715
716
717 # Qwen
600 -def run_qwen_vl(questions: list[str], modality: str):
718 +def run_qwen_vl(questions: list[str], modality: str) -> ModelRequestData:
719 assert modality == "image"
720
603 - llm = LLM(
721 + engine_args = EngineArgs(
722 model="Qwen/Qwen-VL",
723 trust_remote_code=True,
724 max_model_len=1024,
@@ -610,16 +728,19 @@ def run_qwen_vl(questions: list[str], modality: str):
728 )
729
730 prompts = [f"{question}Picture 1: <img></img>\n" for question in questions]
613 - stop_token_ids = None
614 - return llm, prompts, stop_token_ids
731 +
732 + return ModelRequestData(
733 + engine_args=engine_args,
734 + prompts=prompts,
735 + )
736
737
738 # Qwen2-VL
618 -def run_qwen2_vl(questions: list[str], modality: str):
739 +def run_qwen2_vl(questions: list[str], modality: str) -> ModelRequestData:
740
741 model_name = "Qwen/Qwen2-VL-7B-Instruct"
742
622 - llm = LLM(
743 + engine_args = EngineArgs(
744 model=model_name,
745 max_model_len=4096,
746 max_num_seqs=5,
@@ -642,16 +763,19 @@ def run_qwen2_vl(questions: list[str], modality: str):
763 f"{question}<|im_end|>\n"
764 "<|im_start|>assistant\n") for question in questions
765 ]
645 - stop_token_ids = None
646 - return llm, prompts, stop_token_ids
766 +
767 + return ModelRequestData(
768 + engine_args=engine_args,
769 + prompts=prompts,
770 + )
771
772
773 # Qwen2.5-VL
650 -def run_qwen2_5_vl(questions: list[str], modality: str):
774 +def run_qwen2_5_vl(questions: list[str], modality: str) -> ModelRequestData:
775
776 model_name = "Qwen/Qwen2.5-VL-3B-Instruct"
777
654 - llm = LLM(
778 + engine_args = EngineArgs(
779 model=model_name,
780 max_model_len=4096,
781 max_num_seqs=5,
@@ -674,8 +798,11 @@ def run_qwen2_5_vl(questions: list[str], modality: str):
798 f"{question}<|im_end|>\n"
799 "<|im_start|>assistant\n") for question in questions
800 ]
677 - stop_token_ids = None
678 - return llm, prompts, stop_token_ids
801 +
802 + return ModelRequestData(
803 + engine_args=engine_args,
804 + prompts=prompts,
805 + )
806
807
808 model_example_map = {
@@ -789,18 +916,28 @@ def main(args):
916 data = mm_input["data"]
917 questions = mm_input["questions"]
918
792 - llm, prompts, stop_token_ids = model_example_map[model](questions,
793 - modality)
919 + req_data = model_example_map[model](questions, modality)
920 +
921 + engine_args = asdict(req_data.engine_args) | {"seed": args.seed}
922 + llm = LLM(**engine_args)
923 +
924 + # To maintain code compatibility in this script, we add LoRA here.
925 + # You can also add LoRA using:
926 + # llm.generate(prompts, lora_request=lora_request,...)
927 + if req_data.lora_requests:
928 + for lora_request in req_data.lora_requests:
929 + llm.llm_engine.add_lora(lora_request=lora_request)
930 +
931 # Don't want to check the flag multiple times, so just hijack `prompts`.
795 - prompts = prompts if args.use_different_prompt_per_request else [
796 - prompts[0]
932 + prompts = req_data.prompts if args.use_different_prompt_per_request else [
933 + req_data.prompts[0]
934 ]
935
936 # We set temperature to 0.2 so that outputs can be different
937 # even when all prompts are identical when running batch inference.
938 sampling_params = SamplingParams(temperature=0.2,
939 max_tokens=64,
803 - stop_token_ids=stop_token_ids)
940 + stop_token_ids=req_data.stop_token_ids)
941
942 assert args.num_prompts > 0
943 if args.num_prompts == 1:
@@ -865,6 +1002,10 @@ if __name__ == "__main__":
1002 type=int,
1003 default=16,
1004 help='Number of frames to extract from the video.')
1005 + parser.add_argument("--seed",
1006 + type=int,
1007 + default=None,
1008 + help="Set the seed when initializing `vllm.LLM`.")
1009
1010 parser.add_argument(
1011 '--image-repeat-prob',
examples/offline_inference/vision_language_embedding.py
+20 -11
@@ -7,11 +7,12 @@ For most models, the prompt format should follow corresponding examples
7 on HuggingFace model repository.
8 """
9 from argparse import Namespace
10 +from dataclasses import asdict
11 from typing import Literal, NamedTuple, Optional, TypedDict, Union, get_args
12
13 from PIL.Image import Image
14
14 -from vllm import LLM
15 +from vllm import LLM, EngineArgs
16 from vllm.multimodal.utils import fetch_image
17 from vllm.utils import FlexibleArgumentParser
18
@@ -37,12 +38,12 @@ Query = Union[TextQuery, ImageQuery, TextImageQuery]
38
39
40 class ModelRequestData(NamedTuple):
40 - llm: LLM
41 + engine_args: EngineArgs
42 prompt: str
43 image: Optional[Image]
44
45
45 -def run_e5_v(query: Query):
46 +def run_e5_v(query: Query) -> ModelRequestData:
47 llama3_template = '<|start_header_id|>user<|end_header_id|>\n\n{}<|eot_id|><|start_header_id|>assistant<|end_header_id|>\n\n \n' # noqa: E501
48
49 if query["modality"] == "text":
@@ -58,20 +59,20 @@ def run_e5_v(query: Query):
59 modality = query['modality']
60 raise ValueError(f"Unsupported query modality: '{modality}'")
61
61 - llm = LLM(
62 + engine_args = EngineArgs(
63 model="royokong/e5-v",
64 task="embed",
65 max_model_len=4096,
66 )
67
68 return ModelRequestData(
68 - llm=llm,
69 + engine_args=engine_args,
70 prompt=prompt,
71 image=image,
72 )
73
74
74 -def run_vlm2vec(query: Query):
75 +def run_vlm2vec(query: Query) -> ModelRequestData:
76 if query["modality"] == "text":
77 text = query["text"]
78 prompt = f"Find me an everyday image that matches the given caption: {text}" # noqa: E501
@@ -87,7 +88,7 @@ def run_vlm2vec(query: Query):
88 modality = query['modality']
89 raise ValueError(f"Unsupported query modality: '{modality}'")
90
90 - llm = LLM(
91 + engine_args = EngineArgs(
92 model="TIGER-Lab/VLM2Vec-Full",
93 task="embed",
94 trust_remote_code=True,
@@ -95,7 +96,7 @@ def run_vlm2vec(query: Query):
96 )
97
98 return ModelRequestData(
98 - llm=llm,
99 + engine_args=engine_args,
100 prompt=prompt,
101 image=image,
102 )
@@ -126,15 +127,18 @@ def get_query(modality: QueryModality):
127 raise ValueError(msg)
128
129
129 -def run_encode(model: str, modality: QueryModality):
130 +def run_encode(model: str, modality: QueryModality, seed: Optional[int]):
131 query = get_query(modality)
132 req_data = model_example_map[model](query)
133
134 + engine_args = asdict(req_data.engine_args) | {"seed": seed}
135 + llm = LLM(**engine_args)
136 +
137 mm_data = {}
138 if req_data.image is not None:
139 mm_data["image"] = req_data.image
140
137 - outputs = req_data.llm.embed({
141 + outputs = llm.embed({
142 "prompt": req_data.prompt,
143 "multi_modal_data": mm_data,
144 })
@@ -144,7 +148,7 @@ def run_encode(model: str, modality: QueryModality):
148
149
150 def main(args: Namespace):
147 - run_encode(args.model_name, args.modality)
151 + run_encode(args.model_name, args.modality, args.seed)
152
153
154 model_example_map = {
@@ -167,5 +171,10 @@ if __name__ == "__main__":
171 default="image",
172 choices=get_args(QueryModality),
173 help='Modality of the input.')
174 + parser.add_argument("--seed",
175 + type=int,
176 + default=None,
177 + help="Set the seed when initializing `vllm.LLM`.")
178 +
179 args = parser.parse_args()
180 main(args)
examples/offline_inference/vision_language_multi_image.py
+89 -90
@@ -6,13 +6,14 @@ using the chat template defined by the model.
6 """
7 import os
8 from argparse import Namespace
9 +from dataclasses import asdict
10 from typing import NamedTuple, Optional
11
12 from huggingface_hub import snapshot_download
13 from PIL.Image import Image
14 from transformers import AutoProcessor, AutoTokenizer
15
15 -from vllm import LLM, SamplingParams
16 +from vllm import LLM, EngineArgs, SamplingParams
17 from vllm.lora.request import LoRARequest
18 from vllm.multimodal.utils import fetch_image
19 from vllm.utils import FlexibleArgumentParser
@@ -25,11 +26,12 @@ IMAGE_URLS = [
26
27
28 class ModelRequestData(NamedTuple):
28 - llm: LLM
29 + engine_args: EngineArgs
30 prompt: str
30 - stop_token_ids: Optional[list[int]]
31 image_data: list[Image]
32 - chat_template: Optional[str]
32 + stop_token_ids: Optional[list[int]] = None
33 + chat_template: Optional[str] = None
34 + lora_requests: Optional[list[LoRARequest]] = None
35
36
37 # NOTE: The default `max_num_seqs` and `max_model_len` may result in OOM on
@@ -37,53 +39,55 @@ class ModelRequestData(NamedTuple):
39 # Unless specified, these settings have been tested to work on a single L4.
40
41
40 -def load_aria(question, image_urls: list[str]) -> ModelRequestData:
42 +def load_aria(question: str, image_urls: list[str]) -> ModelRequestData:
43 model_name = "rhymes-ai/Aria"
42 - llm = LLM(model=model_name,
43 - tokenizer_mode="slow",
44 - trust_remote_code=True,
45 - dtype="bfloat16",
46 - limit_mm_per_prompt={"image": len(image_urls)})
44 + engine_args = EngineArgs(
45 + model=model_name,
46 + tokenizer_mode="slow",
47 + trust_remote_code=True,
48 + dtype="bfloat16",
49 + limit_mm_per_prompt={"image": len(image_urls)},
50 + )
51 placeholders = "<fim_prefix><|img|><fim_suffix>\n" * len(image_urls)
52 prompt = (f"<|im_start|>user\n{placeholders}{question}<|im_end|>\n"
53 "<|im_start|>assistant\n")
54 stop_token_ids = [93532, 93653, 944, 93421, 1019, 93653, 93519]
55
56 return ModelRequestData(
53 - llm=llm,
57 + engine_args=engine_args,
58 prompt=prompt,
59 stop_token_ids=stop_token_ids,
60 image_data=[fetch_image(url) for url in image_urls],
57 - chat_template=None,
61 )
62
63
61 -def load_deepseek_vl2(question: str, image_urls: list[str]):
64 +def load_deepseek_vl2(question: str,
65 + image_urls: list[str]) -> ModelRequestData:
66 model_name = "deepseek-ai/deepseek-vl2-tiny"
67
64 - llm = LLM(model=model_name,
65 - max_model_len=4096,
66 - max_num_seqs=2,
67 - hf_overrides={"architectures": ["DeepseekVLV2ForCausalLM"]},
68 - limit_mm_per_prompt={"image": len(image_urls)})
68 + engine_args = EngineArgs(
69 + model=model_name,
70 + max_model_len=4096,
71 + max_num_seqs=2,
72 + hf_overrides={"architectures": ["DeepseekVLV2ForCausalLM"]},
73 + limit_mm_per_prompt={"image": len(image_urls)},
74 + )
75
76 placeholder = "".join(f"image_{i}:<image>\n"
77 for i, _ in enumerate(image_urls, start=1))
78 prompt = f"<|User|>: {placeholder}{question}\n\n<|Assistant|>:"
79
80 return ModelRequestData(
75 - llm=llm,
81 + engine_args=engine_args,
82 prompt=prompt,
77 - stop_token_ids=None,
83 image_data=[fetch_image(url) for url in image_urls],
79 - chat_template=None,
84 )
85
86
83 -def load_gemma3(question, image_urls: list[str]) -> ModelRequestData:
87 +def load_gemma3(question: str, image_urls: list[str]) -> ModelRequestData:
88 model_name = "google/gemma-3-4b-it"
89
86 - llm = LLM(
90 + engine_args = EngineArgs(
91 model=model_name,
92 max_model_len=8192,
93 max_num_seqs=2,
@@ -112,18 +116,16 @@ def load_gemma3(question, image_urls: list[str]) -> ModelRequestData:
116 add_generation_prompt=True)
117
118 return ModelRequestData(
115 - llm=llm,
119 + engine_args=engine_args,
120 prompt=prompt,
117 - stop_token_ids=None,
121 image_data=[fetch_image(url) for url in image_urls],
119 - chat_template=None,
122 )
123
124
125 def load_h2ovl(question: str, image_urls: list[str]) -> ModelRequestData:
126 model_name = "h2oai/h2ovl-mississippi-800m"
127
126 - llm = LLM(
128 + engine_args = EngineArgs(
129 model=model_name,
130 trust_remote_code=True,
131 max_model_len=8192,
@@ -146,19 +148,18 @@ def load_h2ovl(question: str, image_urls: list[str]) -> ModelRequestData:
148 stop_token_ids = [tokenizer.eos_token_id]
149
150 return ModelRequestData(
149 - llm=llm,
151 + engine_args=engine_args,
152 prompt=prompt,
153 stop_token_ids=stop_token_ids,
154 image_data=[fetch_image(url) for url in image_urls],
153 - chat_template=None,
155 )
156
157
157 -def load_idefics3(question, image_urls: list[str]) -> ModelRequestData:
158 +def load_idefics3(question: str, image_urls: list[str]) -> ModelRequestData:
159 model_name = "HuggingFaceM4/Idefics3-8B-Llama3"
160
161 # The configuration below has been confirmed to launch on a single L40 GPU.
161 - llm = LLM(
162 + engine_args = EngineArgs(
163 model=model_name,
164 max_model_len=8192,
165 max_num_seqs=16,
@@ -177,18 +178,16 @@ def load_idefics3(question, image_urls: list[str]) -> ModelRequestData:
178 for i, _ in enumerate(image_urls, start=1))
179 prompt = f"<|begin_of_text|>User:{placeholders}\n{question}<end_of_utterance>\nAssistant:" # noqa: E501
180 return ModelRequestData(
180 - llm=llm,
181 + engine_args=engine_args,
182 prompt=prompt,
182 - stop_token_ids=None,
183 image_data=[fetch_image(url) for url in image_urls],
184 - chat_template=None,
184 )
185
186
187 def load_internvl(question: str, image_urls: list[str]) -> ModelRequestData:
188 model_name = "OpenGVLab/InternVL2-2B"
189
191 - llm = LLM(
190 + engine_args = EngineArgs(
191 model=model_name,
192 trust_remote_code=True,
193 max_model_len=4096,
@@ -214,19 +213,18 @@ def load_internvl(question: str, image_urls: list[str]) -> ModelRequestData:
213 stop_token_ids = [tokenizer.convert_tokens_to_ids(i) for i in stop_tokens]
214
215 return ModelRequestData(
217 - llm=llm,
216 + engine_args=engine_args,
217 prompt=prompt,
218 stop_token_ids=stop_token_ids,
219 image_data=[fetch_image(url) for url in image_urls],
221 - chat_template=None,
220 )
221
222
225 -def load_mllama(question, image_urls: list[str]) -> ModelRequestData:
223 +def load_mllama(question: str, image_urls: list[str]) -> ModelRequestData:
224 model_name = "meta-llama/Llama-3.2-11B-Vision-Instruct"
225
226 # The configuration below has been confirmed to launch on a single L40 GPU.
229 - llm = LLM(
227 + engine_args = EngineArgs(
228 model=model_name,
229 max_model_len=4096,
230 max_num_seqs=16,
@@ -236,19 +234,17 @@ def load_mllama(question, image_urls: list[str]) -> ModelRequestData:
234 placeholders = "<|image|>" * len(image_urls)
235 prompt = f"{placeholders}<|begin_of_text|>{question}"
236 return ModelRequestData(
239 - llm=llm,
237 + engine_args=engine_args,
238 prompt=prompt,
241 - stop_token_ids=None,
239 image_data=[fetch_image(url) for url in image_urls],
243 - chat_template=None,
240 )
241
242
247 -def load_nvlm_d(question: str, image_urls: list[str]):
243 +def load_nvlm_d(question: str, image_urls: list[str]) -> ModelRequestData:
244 model_name = "nvidia/NVLM-D-72B"
245
246 # Adjust this as necessary to fit in GPU
251 - llm = LLM(
247 + engine_args = EngineArgs(
248 model=model_name,
249 trust_remote_code=True,
250 max_model_len=8192,
@@ -266,14 +262,11 @@ def load_nvlm_d(question: str, image_urls: list[str]):
262 prompt = tokenizer.apply_chat_template(messages,
263 tokenize=False,
264 add_generation_prompt=True)
269 - stop_token_ids = None
265
266 return ModelRequestData(
272 - llm=llm,
267 + engine_args=engine_args,
268 prompt=prompt,
274 - stop_token_ids=stop_token_ids,
269 image_data=[fetch_image(url) for url in image_urls],
276 - chat_template=None,
270 )
271
272
@@ -281,7 +274,7 @@ def load_pixtral_hf(question: str, image_urls: list[str]) -> ModelRequestData:
274 model_name = "mistral-community/pixtral-12b"
275
276 # Adjust this as necessary to fit in GPU
284 - llm = LLM(
277 + engine_args = EngineArgs(
278 model=model_name,
279 max_model_len=8192,
280 max_num_seqs=2,
@@ -291,14 +284,11 @@ def load_pixtral_hf(question: str, image_urls: list[str]) -> ModelRequestData:
284
285 placeholders = "[IMG]" * len(image_urls)
286 prompt = f"<s>[INST]{question}\n{placeholders}[/INST]"
294 - stop_token_ids = None
287
288 return ModelRequestData(
297 - llm=llm,
289 + engine_args=engine_args,
290 prompt=prompt,
299 - stop_token_ids=stop_token_ids,
291 image_data=[fetch_image(url) for url in image_urls],
301 - chat_template=None,
292 )
293
294
@@ -315,7 +305,7 @@ def load_phi3v(question: str, image_urls: list[str]) -> ModelRequestData:
305 #
306 # https://huggingface.co/microsoft/Phi-3.5-vision-instruct#loading-the-model-locally
307 # https://huggingface.co/microsoft/Phi-3.5-vision-instruct/blob/main/processing_phi3_v.py#L194
318 - llm = LLM(
308 + engine_args = EngineArgs(
309 model="microsoft/Phi-3.5-vision-instruct",
310 trust_remote_code=True,
311 max_model_len=4096,
@@ -326,14 +316,11 @@ def load_phi3v(question: str, image_urls: list[str]) -> ModelRequestData:
316 placeholders = "\n".join(f"<|image_{i}|>"
317 for i, _ in enumerate(image_urls, start=1))
318 prompt = f"<|user|>\n{placeholders}\n{question}<|end|>\n<|assistant|>\n"
329 - stop_token_ids = None
319
320 return ModelRequestData(
332 - llm=llm,
321 + engine_args=engine_args,
322 prompt=prompt,
334 - stop_token_ids=stop_token_ids,
323 image_data=[fetch_image(url) for url in image_urls],
336 - chat_template=None,
324 )
325
326
@@ -347,7 +334,7 @@ def load_phi4mm(question: str, image_urls: list[str]) -> ModelRequestData:
334 # Since the vision-lora and speech-lora co-exist with the base model,
335 # we have to manually specify the path of the lora weights.
336 vision_lora_path = os.path.join(model_path, "vision-lora")
350 - llm = LLM(
337 + engine_args = EngineArgs(
338 model=model_path,
339 trust_remote_code=True,
340 max_model_len=10000,
@@ -357,30 +344,23 @@ def load_phi4mm(question: str, image_urls: list[str]) -> ModelRequestData:
344 max_lora_rank=320,
345 lora_extra_vocab_size=0,
346 )
360 - lora_request = LoRARequest("vision", 1, vision_lora_path)
361 - # To maintain code compatibility in this script, we add LoRA here.
362 - llm.llm_engine.add_lora(lora_request=lora_request)
363 - # You can also add LoRA using:
364 - # llm.generate(prompts, lora_request=lora_request,...)
347
348 placeholders = "".join(f"<|image_{i}|>"
349 for i, _ in enumerate(image_urls, start=1))
350 prompt = f"<|user|>{placeholders}{question}<|end|><|assistant|>"
369 - stop_token_ids = None
351
352 return ModelRequestData(
372 - llm=llm,
353 + engine_args=engine_args,
354 prompt=prompt,
374 - stop_token_ids=stop_token_ids,
355 image_data=[fetch_image(url) for url in image_urls],
376 - chat_template=None,
356 + lora_requests=[LoRARequest("vision", 1, vision_lora_path)],
357 )
358
359
360 def load_qwen_vl_chat(question: str,
361 image_urls: list[str]) -> ModelRequestData:
362 model_name = "Qwen/Qwen-VL-Chat"
383 - llm = LLM(
363 + engine_args = EngineArgs(
364 model=model_name,
365 trust_remote_code=True,
366 max_model_len=1024,
@@ -411,7 +391,7 @@ def load_qwen_vl_chat(question: str,
391 stop_token_ids = [tokenizer.convert_tokens_to_ids(i) for i in stop_tokens]
392
393 return ModelRequestData(
414 - llm=llm,
394 + engine_args=engine_args,
395 prompt=prompt,
396 stop_token_ids=stop_token_ids,
397 image_data=[fetch_image(url) for url in image_urls],
@@ -419,7 +399,7 @@ def load_qwen_vl_chat(question: str,
399 )
400
401
422 -def load_qwen2_vl(question, image_urls: list[str]) -> ModelRequestData:
402 +def load_qwen2_vl(question: str, image_urls: list[str]) -> ModelRequestData:
403 try:
404 from qwen_vl_utils import process_vision_info
405 except ModuleNotFoundError:
@@ -431,7 +411,7 @@ def load_qwen2_vl(question, image_urls: list[str]) -> ModelRequestData:
411 model_name = "Qwen/Qwen2-VL-7B-Instruct"
412
413 # Tested on L40
434 - llm = LLM(
414 + engine_args = EngineArgs(
415 model=model_name,
416 max_model_len=32768 if process_vision_info is None else 4096,
417 max_num_seqs=5,
@@ -460,23 +440,19 @@ def load_qwen2_vl(question, image_urls: list[str]) -> ModelRequestData:
440 tokenize=False,
441 add_generation_prompt=True)
442
463 - stop_token_ids = None
464 -
443 if process_vision_info is None:
444 image_data = [fetch_image(url) for url in image_urls]
445 else:
446 image_data, _ = process_vision_info(messages)
447
448 return ModelRequestData(
471 - llm=llm,
449 + engine_args=engine_args,
450 prompt=prompt,
473 - stop_token_ids=stop_token_ids,
451 image_data=image_data,
475 - chat_template=None,
452 )
453
454
479 -def load_qwen2_5_vl(question, image_urls: list[str]) -> ModelRequestData:
455 +def load_qwen2_5_vl(question: str, image_urls: list[str]) -> ModelRequestData:
456 try:
457 from qwen_vl_utils import process_vision_info
458 except ModuleNotFoundError:
@@ -487,7 +463,7 @@ def load_qwen2_5_vl(question, image_urls: list[str]) -> ModelRequestData:
463
464 model_name = "Qwen/Qwen2.5-VL-3B-Instruct"
465
490 - llm = LLM(
466 + engine_args = EngineArgs(
467 model=model_name,
468 max_model_len=32768 if process_vision_info is None else 4096,
469 max_num_seqs=5,
@@ -516,8 +492,6 @@ def load_qwen2_5_vl(question, image_urls: list[str]) -> ModelRequestData:
492 tokenize=False,
493 add_generation_prompt=True)
494
519 - stop_token_ids = None
520 -
495 if process_vision_info is None:
496 image_data = [fetch_image(url) for url in image_urls]
497 else:
@@ -525,11 +499,9 @@ def load_qwen2_5_vl(question, image_urls: list[str]) -> ModelRequestData:
499 return_video_kwargs=False)
500
501 return ModelRequestData(
528 - llm=llm,
502 + engine_args=engine_args,
503 prompt=prompt,
530 - stop_token_ids=stop_token_ids,
504 image_data=image_data,
532 - chat_template=None,
505 )
506
507
@@ -551,14 +523,25 @@ model_example_map = {
523 }
524
525
554 -def run_generate(model, question: str, image_urls: list[str]):
526 +def run_generate(model, question: str, image_urls: list[str],
527 + seed: Optional[int]):
528 req_data = model_example_map[model](question, image_urls)
529
530 + engine_args = asdict(req_data.engine_args) | {"seed": args.seed}
531 + llm = LLM(**engine_args)
532 +
533 + # To maintain code compatibility in this script, we add LoRA here.
534 + # You can also add LoRA using:
535 + # llm.generate(prompts, lora_request=lora_request,...)
536 + if req_data.lora_requests:
537 + for lora_request in req_data.lora_requests:
538 + llm.llm_engine.add_lora(lora_request=lora_request)
539 +
540 sampling_params = SamplingParams(temperature=0.0,
541 max_tokens=128,
542 stop_token_ids=req_data.stop_token_ids)
543
561 - outputs = req_data.llm.generate(
544 + outputs = llm.generate(
545 {
546 "prompt": req_data.prompt,
547 "multi_modal_data": {
@@ -572,13 +555,24 @@ def run_generate(model, question: str, image_urls: list[str]):
555 print(generated_text)
556
557
575 -def run_chat(model: str, question: str, image_urls: list[str]):
558 +def run_chat(model: str, question: str, image_urls: list[str],
559 + seed: Optional[int]):
560 req_data = model_example_map[model](question, image_urls)
561
562 + engine_args = asdict(req_data.engine_args) | {"seed": seed}
563 + llm = LLM(**engine_args)
564 +
565 + # To maintain code compatibility in this script, we add LoRA here.
566 + # You can also add LoRA using:
567 + # llm.generate(prompts, lora_request=lora_request,...)
568 + if req_data.lora_requests:
569 + for lora_request in req_data.lora_requests:
570 + llm.llm_engine.add_lora(lora_request=lora_request)
571 +
572 sampling_params = SamplingParams(temperature=0.0,
573 max_tokens=128,
574 stop_token_ids=req_data.stop_token_ids)
581 - outputs = req_data.llm.chat(
575 + outputs = llm.chat(
576 [{
577 "role":
578 "user",
@@ -607,11 +601,12 @@ def run_chat(model: str, question: str, image_urls: list[str]):
601 def main(args: Namespace):
602 model = args.model_type
603 method = args.method
604 + seed = args.seed
605
606 if method == "generate":
612 - run_generate(model, QUESTION, IMAGE_URLS)
607 + run_generate(model, QUESTION, IMAGE_URLS, seed)
608 elif method == "chat":
614 - run_chat(model, QUESTION, IMAGE_URLS)
609 + run_chat(model, QUESTION, IMAGE_URLS, seed)
610 else:
611 raise ValueError(f"Invalid method: {method}")
612
@@ -632,6 +627,10 @@ if __name__ == "__main__":
627 default="generate",
628 choices=["generate", "chat"],
629 help="The method to run in `vllm.LLM`.")
630 + parser.add_argument("--seed",
631 + type=int,
632 + default=None,
633 + help="Set the seed when initializing `vllm.LLM`.")
634
635 args = parser.parse_args()
636 main(args)