@tduncan / Nvidia-Optimization-Stack-VLLM Mirror · read-only / commits / 8ca5051b9a
Fork 0

A high-throughput and memory-efficient inference and serving engine for LLMs

Mirrored from https://github.com/tduncan-lum/Nvidia-Optimization-Stack---VLLM.git

Synced Nov 16, 15:45 UTC

[Misc] Use NamedTuple in Multi-image example (#8705)

Signed-off-by: Alex-Brooks <Alex.Brooks@ibm.com>

Alex Brooks committed Sep 22, 2024 at 06:56 UTC 8ca5051b9afb6f8d2b3ae1b71d45d84e5d1c6f57
1 file changed +52 -22
examples/offline_inference_vision_language_multi_image.py
+52 -22
@@ -4,8 +4,9 @@ multi-image input on vision language models, using the chat template defined
4 by the model.
5 """
6 from argparse import Namespace
7 -from typing import List
7 +from typing import List, NamedTuple, Optional
8
9 +from PIL.Image import Image
10 from transformers import AutoProcessor, AutoTokenizer
11
12 from vllm import LLM, SamplingParams
@@ -19,7 +20,15 @@ IMAGE_URLS = [
20 ]
21
22
22 -def load_qwenvl_chat(question: str, image_urls: List[str]):
23 +class ModelRequestData(NamedTuple):
24 + llm: LLM
25 + prompt: str
26 + stop_token_ids: Optional[List[str]]
27 + image_data: List[Image]
28 + chat_template: Optional[str]
29 +
30 +
31 +def load_qwenvl_chat(question: str, image_urls: List[str]) -> ModelRequestData:
32 model_name = "Qwen/Qwen-VL-Chat"
33 llm = LLM(
34 model=model_name,
@@ -48,10 +57,16 @@ def load_qwenvl_chat(question: str, image_urls: List[str]):
57
58 stop_tokens = ["<|endoftext|>", "<|im_start|>", "<|im_end|>"]
59 stop_token_ids = [tokenizer.convert_tokens_to_ids(i) for i in stop_tokens]
51 - return llm, prompt, stop_token_ids, None, chat_template
60 + return ModelRequestData(
61 + llm=llm,
62 + prompt=prompt,
63 + stop_token_ids=stop_token_ids,
64 + image_data=[fetch_image(url) for url in image_urls],
65 + chat_template=chat_template,
66 + )
67
68
54 -def load_phi3v(question: str, image_urls: List[str]):
69 +def load_phi3v(question: str, image_urls: List[str]) -> ModelRequestData:
70 llm = LLM(
71 model="microsoft/Phi-3.5-vision-instruct",
72 trust_remote_code=True,
@@ -62,10 +77,17 @@ def load_phi3v(question: str, image_urls: List[str]):
77 for i, _ in enumerate(image_urls, start=1))
78 prompt = f"<|user|>\n{placeholders}\n{question}<|end|>\n<|assistant|>\n"
79 stop_token_ids = None
65 - return llm, prompt, stop_token_ids, None, None
80 +
81 + return ModelRequestData(
82 + llm=llm,
83 + prompt=prompt,
84 + stop_token_ids=stop_token_ids,
85 + image_data=[fetch_image(url) for url in image_urls],
86 + chat_template=None,
87 + )
88
89
68 -def load_internvl(question: str, image_urls: List[str]):
90 +def load_internvl(question: str, image_urls: List[str]) -> ModelRequestData:
91 model_name = "OpenGVLab/InternVL2-2B"
92
93 llm = LLM(
@@ -93,10 +115,16 @@ def load_internvl(question: str, image_urls: List[str]):
115 stop_tokens = ["<|endoftext|>", "<|im_start|>", "<|im_end|>", "<|end|>"]
116 stop_token_ids = [tokenizer.convert_tokens_to_ids(i) for i in stop_tokens]
117
96 - return llm, prompt, stop_token_ids, None, None
118 + return ModelRequestData(
119 + llm=llm,
120 + prompt=prompt,
121 + stop_token_ids=stop_token_ids,
122 + image_data=[fetch_image(url) for url in image_urls],
123 + chat_template=None,
124 + )
125
126
99 -def load_qwen2_vl(question, image_urls: List[str]):
127 +def load_qwen2_vl(question, image_urls: List[str]) -> ModelRequestData:
128 try:
129 from qwen_vl_utils import process_vision_info
130 except ModuleNotFoundError:
@@ -143,7 +171,13 @@ def load_qwen2_vl(question, image_urls: List[str]):
171 else:
172 image_data, _ = process_vision_info(messages)
173
146 - return llm, prompt, stop_token_ids, image_data, None
174 + return ModelRequestData(
175 + llm=llm,
176 + prompt=prompt,
177 + stop_token_ids=stop_token_ids,
178 + image_data=image_data,
179 + chat_template=None,
180 + )
181
182
183 model_example_map = {
@@ -155,20 +189,17 @@ model_example_map = {
189
190
191 def run_generate(model, question: str, image_urls: List[str]):
158 - llm, prompt, stop_token_ids, image_data, _ = model_example_map[model](
159 - question, image_urls)
160 - if image_data is None:
161 - image_data = [fetch_image(url) for url in image_urls]
192 + req_data = model_example_map[model](question, image_urls)
193
194 sampling_params = SamplingParams(temperature=0.0,
195 max_tokens=128,
165 - stop_token_ids=stop_token_ids)
196 + stop_token_ids=req_data.stop_token_ids)
197
167 - outputs = llm.generate(
198 + outputs = req_data.llm.generate(
199 {
169 - "prompt": prompt,
200 + "prompt": req_data.prompt,
201 "multi_modal_data": {
171 - "image": image_data
202 + "image": req_data.image_data
203 },
204 },
205 sampling_params=sampling_params)
@@ -179,13 +210,12 @@ def run_generate(model, question: str, image_urls: List[str]):
210
211
212 def run_chat(model: str, question: str, image_urls: List[str]):
182 - llm, _, stop_token_ids, _, chat_template = model_example_map[model](
183 - question, image_urls)
213 + req_data = model_example_map[model](question, image_urls)
214
215 sampling_params = SamplingParams(temperature=0.0,
216 max_tokens=128,
187 - stop_token_ids=stop_token_ids)
188 - outputs = llm.chat(
217 + stop_token_ids=req_data.stop_token_ids)
218 + outputs = req_data.llm.chat(
219 [{
220 "role":
221 "user",
@@ -203,7 +233,7 @@ def run_chat(model: str, question: str, image_urls: List[str]):
233 ],
234 }],
235 sampling_params=sampling_params,
206 - chat_template=chat_template,
236 + chat_template=req_data.chat_template,
237 )
238
239 for o in outputs: