@tduncan / Nvidia-Optimization-Stack-VLLM Mirror · read-only / commits / 91445c7bc8
Fork 0

A high-throughput and memory-efficient inference and serving engine for LLMs

Mirrored from https://github.com/tduncan-lum/Nvidia-Optimization-Stack---VLLM.git

Synced Nov 16, 03:15 UTC

[Bugfix] Fix image input for Pixtral-HF (#11741)

Signed-off-by: DarkLight1337 <tlleungac@connect.ust.hk>

Cyrus Leung committed Jan 8, 2025 at 10:17 UTC 91445c7bc8000a6f6f1efed0882076d7001be968
4 files changed +52 -6
examples/offline_inference_vision_language_multi_image.py
+36 -5
@@ -23,7 +23,7 @@ IMAGE_URLS = [
23 class ModelRequestData(NamedTuple):
24 llm: LLM
25 prompt: str
26 - stop_token_ids: Optional[List[str]]
26 + stop_token_ids: Optional[List[int]]
27 image_data: List[Image]
28 chat_template: Optional[str]
29
@@ -44,12 +44,14 @@ def load_aria(question, image_urls: List[str]) -> ModelRequestData:
44 prompt = (f"<|im_start|>user\n{placeholders}{question}<|im_end|>\n"
45 "<|im_start|>assistant\n")
46 stop_token_ids = [93532, 93653, 944, 93421, 1019, 93653, 93519]
47 +
48 return ModelRequestData(
49 llm=llm,
50 prompt=prompt,
51 stop_token_ids=stop_token_ids,
52 image_data=[fetch_image(url) for url in image_urls],
52 - chat_template=None)
53 + chat_template=None,
54 + )
55
56
57 def load_h2onvl(question: str, image_urls: List[str]) -> ModelRequestData:
@@ -166,7 +168,8 @@ def load_mllama(question, image_urls: List[str]) -> ModelRequestData:
168 limit_mm_per_prompt={"image": len(image_urls)},
169 )
170
169 - prompt = f"<|image|><|image|><|begin_of_text|>{question}"
171 + placeholders = "<|image|>" * len(image_urls)
172 + prompt = f"{placeholders}<|begin_of_text|>{question}"
173 return ModelRequestData(
174 llm=llm,
175 prompt=prompt,
@@ -209,6 +212,31 @@ def load_nvlm_d(question: str, image_urls: List[str]):
212 )
213
214
215 +def load_pixtral_hf(question: str, image_urls: List[str]) -> ModelRequestData:
216 + model_name = "mistral-community/pixtral-12b"
217 +
218 + # Adjust this as necessary to fit in GPU
219 + llm = LLM(
220 + model=model_name,
221 + max_model_len=8192,
222 + max_num_seqs=2,
223 + tensor_parallel_size=2,
224 + limit_mm_per_prompt={"image": len(image_urls)},
225 + )
226 +
227 + placeholders = "[IMG]" * len(image_urls)
228 + prompt = f"<s>[INST]{question}\n{placeholders}[/INST]"
229 + stop_token_ids = None
230 +
231 + return ModelRequestData(
232 + llm=llm,
233 + prompt=prompt,
234 + stop_token_ids=stop_token_ids,
235 + image_data=[fetch_image(url) for url in image_urls],
236 + chat_template=None,
237 + )
238 +
239 +
240 def load_phi3v(question: str, image_urls: List[str]) -> ModelRequestData:
241 # num_crops is an override kwarg to the multimodal image processor;
242 # For some models, e.g., Phi-3.5-vision-instruct, it is recommended
@@ -244,7 +272,8 @@ def load_phi3v(question: str, image_urls: List[str]) -> ModelRequestData:
272 )
273
274
247 -def load_qwenvl_chat(question: str, image_urls: List[str]) -> ModelRequestData:
275 +def load_qwen_vl_chat(question: str,
276 + image_urls: List[str]) -> ModelRequestData:
277 model_name = "Qwen/Qwen-VL-Chat"
278 llm = LLM(
279 model=model_name,
@@ -274,6 +303,7 @@ def load_qwenvl_chat(question: str, image_urls: List[str]) -> ModelRequestData:
303
304 stop_tokens = ["<|endoftext|>", "<|im_start|>", "<|im_end|>"]
305 stop_token_ids = [tokenizer.convert_tokens_to_ids(i) for i in stop_tokens]
306 +
307 return ModelRequestData(
308 llm=llm,
309 prompt=prompt,
@@ -348,7 +378,8 @@ model_example_map = {
378 "mllama": load_mllama,
379 "NVLM_D": load_nvlm_d,
380 "phi3_v": load_phi3v,
351 - "qwen_vl_chat": load_qwenvl_chat,
381 + "pixtral_hf": load_pixtral_hf,
382 + "qwen_vl_chat": load_qwen_vl_chat,
383 "qwen2_vl": load_qwen2_vl,
384 }
385
vllm/model_executor/models/llava.py
+6
@@ -546,6 +546,12 @@ class LlavaForConditionalGeneration(nn.Module, SupportsMultiModal, SupportsPP):
546 raise ValueError("Incorrect type of pixel values. "
547 f"Got type: {type(pixel_values)}")
548
549 + if self.config.vision_config.model_type == "pixtral":
550 + return LlavaImagePixelInputs(
551 + type="pixel_values",
552 + data=flatten_bn(pixel_values),
553 + )
554 +
555 return LlavaImagePixelInputs(
556 type="pixel_values",
557 data=self._validate_pixel_values(
vllm/model_executor/models/pixtral.py
+1 -1
@@ -774,7 +774,7 @@ class PixtralHFEncoderInfo(VisionEncoderInfo[PixtralVisionConfig]):
774 ) -> int:
775 return get_pixtral_hf_image_feature_size(
776 image_size=self.vision_config.image_size,
777 - patch_size=self.get_image_size(),
777 + patch_size=self.vision_config.patch_size,
778 )
779
780 def get_max_image_tokens(self) -> int:
vllm/model_executor/models/utils.py
+9
@@ -281,6 +281,15 @@ def flatten_bn(
281 ...
282
283
284 +@overload
285 +def flatten_bn(
286 + x: Union[List[torch.Tensor], torch.Tensor],
287 + *,
288 + concat: bool = False,
289 +) -> Union[List[torch.Tensor], torch.Tensor]:
290 + ...
291 +
292 +
293 def flatten_bn(
294 x: Union[List[torch.Tensor], torch.Tensor],
295 *,