@tduncan / Nvidia-Optimization-Stack-VLLM Mirror · read-only / commits / 999df95b4e
Fork 0

A high-throughput and memory-efficient inference and serving engine for LLMs

Mirrored from https://github.com/tduncan-lum/Nvidia-Optimization-Stack---VLLM.git

Synced Nov 16, 08:15 UTC

[Bugfix] Make image processor respect `mm_processor_kwargs` for Qwen2-VL (#10112)

Signed-off-by: Jiahao Li <liplus17@163.com>

Jiahao Li committed Nov 7, 2024 at 18:50 UTC 999df95b4eefb920cd3539a7fa3a21b2911f3650
1 file changed +23 -10
vllm/model_executor/models/qwen2_vl.py
+23 -10
@@ -22,8 +22,8 @@
22 # limitations under the License.
23 """Inference-only Qwen2-VL model compatible with HuggingFace weights."""
24 from functools import partial
25 -from typing import (Any, Callable, Iterable, List, Literal, Mapping, Optional,
26 - Tuple, Type, TypedDict, Union)
25 +from typing import (Any, Callable, Dict, Iterable, List, Literal, Mapping,
26 + Optional, Tuple, Type, TypedDict, Union)
27
28 import torch
29 import torch.nn as nn
@@ -558,6 +558,17 @@ class Qwen2VisionTransformer(nn.Module):
558 # === Vision input helpers === #
559
560
561 +def get_mm_processor_kwargs(
562 + min_pixels: Optional[int] = None,
563 + max_pixels: Optional[int] = None) -> Dict[str, int]:
564 + mm_processor_kwargs = {}
565 + if min_pixels:
566 + mm_processor_kwargs["min_pixels"] = min_pixels
567 + if max_pixels:
568 + mm_processor_kwargs["max_pixels"] = max_pixels
569 + return mm_processor_kwargs
570 +
571 +
572 def mm_input_mapper_for_qwen2_vl(
573 ctx: InputContext,
574 data: MultiModalData[object],
@@ -575,12 +586,8 @@ def mm_input_mapper_for_qwen2_vl(
586 model_config = ctx.model_config
587 # Handle mm processor kwargs; we pass these at creation time
588 # because preprocess() in transformers doesn't expose them
578 - mm_processor_kwargs = {}
579 - if min_pixels:
580 - mm_processor_kwargs["min_pixels"] = min_pixels
581 - if max_pixels:
582 - mm_processor_kwargs["max_pixels"] = max_pixels
583 -
589 + mm_processor_kwargs = get_mm_processor_kwargs(min_pixels=min_pixels,
590 + max_pixels=max_pixels)
591 image_processor = cached_get_image_processor(
592 model_config.model,
593 trust_remote_code=model_config.trust_remote_code,
@@ -683,7 +690,10 @@ def get_max_qwen2_vl_mm_tokens(ctx: InputContext,
690 *,
691 min_pixels=None,
692 max_pixels=None) -> int:
686 - image_processor = cached_get_image_processor(ctx.model_config.model)
693 + mm_processor_kwargs = get_mm_processor_kwargs(min_pixels=min_pixels,
694 + max_pixels=max_pixels)
695 + image_processor = cached_get_image_processor(ctx.model_config.model,
696 + **mm_processor_kwargs)
697 max_resized_height, max_resized_width, max_llm_image_tokens = \
698 _get_max_image_info(image_processor, data_type_key=data_type_key,
699 mm_count=1, min_pixels=min_pixels,
@@ -705,7 +715,10 @@ def dummy_data_for_qwen2_vl(
715 min_pixels: Optional[int] = None,
716 max_pixels: Optional[int] = None
717 ) -> Tuple[SequenceData, Optional[MultiModalDataDict]]:
708 - image_processor = cached_get_image_processor(ctx.model_config.model)
718 + mm_processor_kwargs = get_mm_processor_kwargs(min_pixels=min_pixels,
719 + max_pixels=max_pixels)
720 + image_processor = cached_get_image_processor(ctx.model_config.model,
721 + **mm_processor_kwargs)
722
723 num_images = mm_counts["image"]
724 max_resized_height, max_resized_width, max_llm_image_tokens = \