A high-throughput and memory-efficient inference and serving engine for LLMs
Mirrored from https://github.com/tduncan-lum/Nvidia-Optimization-Stack---VLLM.git
Synced Nov 16, 15:45 UTC
[Misc] Use NamedTuple in Multi-image example (#8705)
Signed-off-by: Alex-Brooks <Alex.Brooks@ibm.com>
Alex Brooks committed
Sep 22, 2024 at 06:56 UTC
8ca5051b9afb6f8d2b3ae1b71d45d84e5d1c6f57
1 file changed
+52
-22
examples/offline_inference_vision_language_multi_image.py
+52
-22
@@ -4,8 +4,9 @@ multi-image input on vision language models, using the chat template defined
4
by the model.
5
"""
6
from argparse import Namespace
7
-from typing import List
7
+from typing import List, NamedTuple, Optional
8
9
+from PIL.Image import Image
10
from transformers import AutoProcessor, AutoTokenizer
11
12
from vllm import LLM, SamplingParams
@@ -19,7 +20,15 @@ IMAGE_URLS = [
20
]
21
22
22
-def load_qwenvl_chat(question: str, image_urls: List[str]):
23
+class ModelRequestData(NamedTuple):
24
+ llm: LLM
25
+ prompt: str
26
+ stop_token_ids: Optional[List[str]]
27
+ image_data: List[Image]
28
+ chat_template: Optional[str]
29
+
30
+
31
+def load_qwenvl_chat(question: str, image_urls: List[str]) -> ModelRequestData:
32
model_name = "Qwen/Qwen-VL-Chat"
33
llm = LLM(
34
model=model_name,
@@ -48,10 +57,16 @@ def load_qwenvl_chat(question: str, image_urls: List[str]):
57
58
stop_tokens = ["<|endoftext|>", "<|im_start|>", "<|im_end|>"]
59
stop_token_ids = [tokenizer.convert_tokens_to_ids(i) for i in stop_tokens]
51
- return llm, prompt, stop_token_ids, None, chat_template
60
+ return ModelRequestData(
61
+ llm=llm,
62
+ prompt=prompt,
63
+ stop_token_ids=stop_token_ids,
64
+ image_data=[fetch_image(url) for url in image_urls],
65
+ chat_template=chat_template,
66
+ )
67
68
54
-def load_phi3v(question: str, image_urls: List[str]):
69
+def load_phi3v(question: str, image_urls: List[str]) -> ModelRequestData:
70
llm = LLM(
71
model="microsoft/Phi-3.5-vision-instruct",
72
trust_remote_code=True,
@@ -62,10 +77,17 @@ def load_phi3v(question: str, image_urls: List[str]):
77
for i, _ in enumerate(image_urls, start=1))
78
prompt = f"<|user|>\n{placeholders}\n{question}<|end|>\n<|assistant|>\n"
79
stop_token_ids = None
65
- return llm, prompt, stop_token_ids, None, None
80
+
81
+ return ModelRequestData(
82
+ llm=llm,
83
+ prompt=prompt,
84
+ stop_token_ids=stop_token_ids,
85
+ image_data=[fetch_image(url) for url in image_urls],
86
+ chat_template=None,
87
+ )
88
89
68
-def load_internvl(question: str, image_urls: List[str]):
90
+def load_internvl(question: str, image_urls: List[str]) -> ModelRequestData:
91
model_name = "OpenGVLab/InternVL2-2B"
92
93
llm = LLM(
@@ -93,10 +115,16 @@ def load_internvl(question: str, image_urls: List[str]):
115
stop_tokens = ["<|endoftext|>", "<|im_start|>", "<|im_end|>", "<|end|>"]
116
stop_token_ids = [tokenizer.convert_tokens_to_ids(i) for i in stop_tokens]
117
96
- return llm, prompt, stop_token_ids, None, None
118
+ return ModelRequestData(
119
+ llm=llm,
120
+ prompt=prompt,
121
+ stop_token_ids=stop_token_ids,
122
+ image_data=[fetch_image(url) for url in image_urls],
123
+ chat_template=None,
124
+ )
125
126
99
-def load_qwen2_vl(question, image_urls: List[str]):
127
+def load_qwen2_vl(question, image_urls: List[str]) -> ModelRequestData:
128
try:
129
from qwen_vl_utils import process_vision_info
130
except ModuleNotFoundError:
@@ -143,7 +171,13 @@ def load_qwen2_vl(question, image_urls: List[str]):
171
else:
172
image_data, _ = process_vision_info(messages)
173
146
- return llm, prompt, stop_token_ids, image_data, None
174
+ return ModelRequestData(
175
+ llm=llm,
176
+ prompt=prompt,
177
+ stop_token_ids=stop_token_ids,
178
+ image_data=image_data,
179
+ chat_template=None,
180
+ )
181
182
183
model_example_map = {
@@ -155,20 +189,17 @@ model_example_map = {
189
190
191
def run_generate(model, question: str, image_urls: List[str]):
158
- llm, prompt, stop_token_ids, image_data, _ = model_example_map[model](
159
- question, image_urls)
160
- if image_data is None:
161
- image_data = [fetch_image(url) for url in image_urls]
192
+ req_data = model_example_map[model](question, image_urls)
193
194
sampling_params = SamplingParams(temperature=0.0,
195
max_tokens=128,
165
- stop_token_ids=stop_token_ids)
196
+ stop_token_ids=req_data.stop_token_ids)
197
167
- outputs = llm.generate(
198
+ outputs = req_data.llm.generate(
199
{
169
- "prompt": prompt,
200
+ "prompt": req_data.prompt,
201
"multi_modal_data": {
171
- "image": image_data
202
+ "image": req_data.image_data
203
},
204
},
205
sampling_params=sampling_params)
@@ -179,13 +210,12 @@ def run_generate(model, question: str, image_urls: List[str]):
210
211
212
def run_chat(model: str, question: str, image_urls: List[str]):
182
- llm, _, stop_token_ids, _, chat_template = model_example_map[model](
183
- question, image_urls)
213
+ req_data = model_example_map[model](question, image_urls)
214
215
sampling_params = SamplingParams(temperature=0.0,
216
max_tokens=128,
187
- stop_token_ids=stop_token_ids)
188
- outputs = llm.chat(
217
+ stop_token_ids=req_data.stop_token_ids)
218
+ outputs = req_data.llm.chat(
219
[{
220
"role":
221
"user",
@@ -203,7 +233,7 @@ def run_chat(model: str, question: str, image_urls: List[str]):
233
],
234
}],
235
sampling_params=sampling_params,
206
- chat_template=chat_template,
236
+ chat_template=req_data.chat_template,
237
)
238
239
for o in outputs: