A high-throughput and memory-efficient inference and serving engine for LLMs
Mirrored from https://github.com/tduncan-lum/Nvidia-Optimization-Stack---VLLM.git
Synced Nov 16, 08:15 UTC
[Bugfix] Make image processor respect `mm_processor_kwargs` for Qwen2-VL (#10112)
Signed-off-by: Jiahao Li <liplus17@163.com>
Jiahao Li committed
Nov 7, 2024 at 18:50 UTC
999df95b4eefb920cd3539a7fa3a21b2911f3650
1 file changed
+23
-10
vllm/model_executor/models/qwen2_vl.py
+23
-10
@@ -22,8 +22,8 @@
22
# limitations under the License.
23
"""Inference-only Qwen2-VL model compatible with HuggingFace weights."""
24
from functools import partial
25
-from typing import (Any, Callable, Iterable, List, Literal, Mapping, Optional,
26
- Tuple, Type, TypedDict, Union)
25
+from typing import (Any, Callable, Dict, Iterable, List, Literal, Mapping,
26
+ Optional, Tuple, Type, TypedDict, Union)
27
28
import torch
29
import torch.nn as nn
@@ -558,6 +558,17 @@ class Qwen2VisionTransformer(nn.Module):
558
# === Vision input helpers === #
559
560
561
+def get_mm_processor_kwargs(
562
+ min_pixels: Optional[int] = None,
563
+ max_pixels: Optional[int] = None) -> Dict[str, int]:
564
+ mm_processor_kwargs = {}
565
+ if min_pixels:
566
+ mm_processor_kwargs["min_pixels"] = min_pixels
567
+ if max_pixels:
568
+ mm_processor_kwargs["max_pixels"] = max_pixels
569
+ return mm_processor_kwargs
570
+
571
+
572
def mm_input_mapper_for_qwen2_vl(
573
ctx: InputContext,
574
data: MultiModalData[object],
@@ -575,12 +586,8 @@ def mm_input_mapper_for_qwen2_vl(
586
model_config = ctx.model_config
587
# Handle mm processor kwargs; we pass these at creation time
588
# because preprocess() in transformers doesn't expose them
578
- mm_processor_kwargs = {}
579
- if min_pixels:
580
- mm_processor_kwargs["min_pixels"] = min_pixels
581
- if max_pixels:
582
- mm_processor_kwargs["max_pixels"] = max_pixels
583
-
589
+ mm_processor_kwargs = get_mm_processor_kwargs(min_pixels=min_pixels,
590
+ max_pixels=max_pixels)
591
image_processor = cached_get_image_processor(
592
model_config.model,
593
trust_remote_code=model_config.trust_remote_code,
@@ -683,7 +690,10 @@ def get_max_qwen2_vl_mm_tokens(ctx: InputContext,
690
*,
691
min_pixels=None,
692
max_pixels=None) -> int:
686
- image_processor = cached_get_image_processor(ctx.model_config.model)
693
+ mm_processor_kwargs = get_mm_processor_kwargs(min_pixels=min_pixels,
694
+ max_pixels=max_pixels)
695
+ image_processor = cached_get_image_processor(ctx.model_config.model,
696
+ **mm_processor_kwargs)
697
max_resized_height, max_resized_width, max_llm_image_tokens = \
698
_get_max_image_info(image_processor, data_type_key=data_type_key,
699
mm_count=1, min_pixels=min_pixels,
@@ -705,7 +715,10 @@ def dummy_data_for_qwen2_vl(
715
min_pixels: Optional[int] = None,
716
max_pixels: Optional[int] = None
717
) -> Tuple[SequenceData, Optional[MultiModalDataDict]]:
708
- image_processor = cached_get_image_processor(ctx.model_config.model)
718
+ mm_processor_kwargs = get_mm_processor_kwargs(min_pixels=min_pixels,
719
+ max_pixels=max_pixels)
720
+ image_processor = cached_get_image_processor(ctx.model_config.model,
721
+ **mm_processor_kwargs)
722
723
num_images = mm_counts["image"]
724
max_resized_height, max_resized_width, max_llm_image_tokens = \