A high-throughput and memory-efficient inference and serving engine for LLMs
Mirrored from https://github.com/tduncan-lum/Nvidia-Optimization-Stack---VLLM.git
Synced Nov 16, 03:15 UTC
[Bugfix] Fix image input for Pixtral-HF (#11741)
Signed-off-by: DarkLight1337 <tlleungac@connect.ust.hk>
Cyrus Leung committed
Jan 8, 2025 at 10:17 UTC
91445c7bc8000a6f6f1efed0882076d7001be968
4 files changed
+52
-6
examples/offline_inference_vision_language_multi_image.py
+36
-5
@@ -23,7 +23,7 @@ IMAGE_URLS = [
23
class ModelRequestData(NamedTuple):
24
llm: LLM
25
prompt: str
26
- stop_token_ids: Optional[List[str]]
26
+ stop_token_ids: Optional[List[int]]
27
image_data: List[Image]
28
chat_template: Optional[str]
29
@@ -44,12 +44,14 @@ def load_aria(question, image_urls: List[str]) -> ModelRequestData:
44
prompt = (f"<|im_start|>user\n{placeholders}{question}<|im_end|>\n"
45
"<|im_start|>assistant\n")
46
stop_token_ids = [93532, 93653, 944, 93421, 1019, 93653, 93519]
47
+
48
return ModelRequestData(
49
llm=llm,
50
prompt=prompt,
51
stop_token_ids=stop_token_ids,
52
image_data=[fetch_image(url) for url in image_urls],
52
- chat_template=None)
53
+ chat_template=None,
54
+ )
55
56
57
def load_h2onvl(question: str, image_urls: List[str]) -> ModelRequestData:
@@ -166,7 +168,8 @@ def load_mllama(question, image_urls: List[str]) -> ModelRequestData:
168
limit_mm_per_prompt={"image": len(image_urls)},
169
)
170
169
- prompt = f"<|image|><|image|><|begin_of_text|>{question}"
171
+ placeholders = "<|image|>" * len(image_urls)
172
+ prompt = f"{placeholders}<|begin_of_text|>{question}"
173
return ModelRequestData(
174
llm=llm,
175
prompt=prompt,
@@ -209,6 +212,31 @@ def load_nvlm_d(question: str, image_urls: List[str]):
212
)
213
214
215
+def load_pixtral_hf(question: str, image_urls: List[str]) -> ModelRequestData:
216
+ model_name = "mistral-community/pixtral-12b"
217
+
218
+ # Adjust this as necessary to fit in GPU
219
+ llm = LLM(
220
+ model=model_name,
221
+ max_model_len=8192,
222
+ max_num_seqs=2,
223
+ tensor_parallel_size=2,
224
+ limit_mm_per_prompt={"image": len(image_urls)},
225
+ )
226
+
227
+ placeholders = "[IMG]" * len(image_urls)
228
+ prompt = f"<s>[INST]{question}\n{placeholders}[/INST]"
229
+ stop_token_ids = None
230
+
231
+ return ModelRequestData(
232
+ llm=llm,
233
+ prompt=prompt,
234
+ stop_token_ids=stop_token_ids,
235
+ image_data=[fetch_image(url) for url in image_urls],
236
+ chat_template=None,
237
+ )
238
+
239
+
240
def load_phi3v(question: str, image_urls: List[str]) -> ModelRequestData:
241
# num_crops is an override kwarg to the multimodal image processor;
242
# For some models, e.g., Phi-3.5-vision-instruct, it is recommended
@@ -244,7 +272,8 @@ def load_phi3v(question: str, image_urls: List[str]) -> ModelRequestData:
272
)
273
274
247
-def load_qwenvl_chat(question: str, image_urls: List[str]) -> ModelRequestData:
275
+def load_qwen_vl_chat(question: str,
276
+ image_urls: List[str]) -> ModelRequestData:
277
model_name = "Qwen/Qwen-VL-Chat"
278
llm = LLM(
279
model=model_name,
@@ -274,6 +303,7 @@ def load_qwenvl_chat(question: str, image_urls: List[str]) -> ModelRequestData:
303
304
stop_tokens = ["<|endoftext|>", "<|im_start|>", "<|im_end|>"]
305
stop_token_ids = [tokenizer.convert_tokens_to_ids(i) for i in stop_tokens]
306
+
307
return ModelRequestData(
308
llm=llm,
309
prompt=prompt,
@@ -348,7 +378,8 @@ model_example_map = {
378
"mllama": load_mllama,
379
"NVLM_D": load_nvlm_d,
380
"phi3_v": load_phi3v,
351
- "qwen_vl_chat": load_qwenvl_chat,
381
+ "pixtral_hf": load_pixtral_hf,
382
+ "qwen_vl_chat": load_qwen_vl_chat,
383
"qwen2_vl": load_qwen2_vl,
384
}
385
vllm/model_executor/models/llava.py
+6
@@ -546,6 +546,12 @@ class LlavaForConditionalGeneration(nn.Module, SupportsMultiModal, SupportsPP):
546
raise ValueError("Incorrect type of pixel values. "
547
f"Got type: {type(pixel_values)}")
548
549
+ if self.config.vision_config.model_type == "pixtral":
550
+ return LlavaImagePixelInputs(
551
+ type="pixel_values",
552
+ data=flatten_bn(pixel_values),
553
+ )
554
+
555
return LlavaImagePixelInputs(
556
type="pixel_values",
557
data=self._validate_pixel_values(
vllm/model_executor/models/pixtral.py
+1
-1
@@ -774,7 +774,7 @@ class PixtralHFEncoderInfo(VisionEncoderInfo[PixtralVisionConfig]):
774
) -> int:
775
return get_pixtral_hf_image_feature_size(
776
image_size=self.vision_config.image_size,
777
- patch_size=self.get_image_size(),
777
+ patch_size=self.vision_config.patch_size,
778
)
779
780
def get_max_image_tokens(self) -> int:
vllm/model_executor/models/utils.py
+9
@@ -281,6 +281,15 @@ def flatten_bn(
281
...
282
283
284
+@overload
285
+def flatten_bn(
286
+ x: Union[List[torch.Tensor], torch.Tensor],
287
+ *,
288
+ concat: bool = False,
289
+) -> Union[List[torch.Tensor], torch.Tensor]:
290
+ ...
291
+
292
+
293
def flatten_bn(
294
x: Union[List[torch.Tensor], torch.Tensor],
295
*,