@tduncan / Nvidia-Optimization-Stack-VLLM Mirror · read-only / commits / df1a21131d
Fork 0

A high-throughput and memory-efficient inference and serving engine for LLMs

Mirrored from https://github.com/tduncan-lum/Nvidia-Optimization-Stack---VLLM.git

Synced Nov 16, 18:15 UTC

[Model] Fix Phi-3.5-vision-instruct 'num_crops' issue (#7710)

zifeitong committed Aug 21, 2024 at 18:36 UTC df1a21131d951ba8ee65363aeb9b9486f569aa4f
6 files changed +37 -13
docs/source/models/supported_models.rst
+2 -2
@@ -225,9 +225,9 @@ Multimodal Language Models
225 - :code:`google/paligemma-3b-pt-224`, :code:`google/paligemma-3b-mix-224`, etc.
226 -
227 * - :code:`Phi3VForCausalLM`
228 - - Phi-3-Vision
228 + - Phi-3-Vision, Phi-3.5-Vision
229 - Image
230 - - :code:`microsoft/Phi-3-vision-128k-instruct`, etc.
230 + - :code:`microsoft/Phi-3-vision-128k-instruct`, :code:`microsoft/Phi-3.5-vision-instruct` etc.
231 -
232 * - :code:`MiniCPMV`
233 - MiniCPM-V
tests/models/test_phi3v.py
+1 -1
@@ -21,7 +21,7 @@ HF_IMAGE_PROMPTS = IMAGE_ASSETS.prompts({
21 "<|user|>\n<|image_1|>\nWhat is the season?<|end|>\n<|assistant|>\n",
22 })
23
24 -models = ["microsoft/Phi-3-vision-128k-instruct"]
24 +models = ["microsoft/Phi-3.5-vision-instruct"]
25
26
27 def vllm_to_hf_output(vllm_output: Tuple[List[int], str,
vllm/config.py
+5 -1
@@ -13,7 +13,9 @@ from vllm.model_executor.layers.quantization import QUANTIZATION_METHODS
13 from vllm.model_executor.models import ModelRegistry
14 from vllm.platforms import current_platform
15 from vllm.tracing import is_otel_available, otel_import_error_traceback
16 -from vllm.transformers_utils.config import get_config, get_hf_text_config
16 +from vllm.transformers_utils.config import (get_config,
17 + get_hf_image_processor_config,
18 + get_hf_text_config)
19 from vllm.utils import (STR_NOT_IMPL_ENC_DEC_CUDAGRAPH, GiB_bytes,
20 cuda_device_count_stateless, get_cpu_memory, is_cpu,
21 is_hip, is_neuron, is_openvino, is_xpu,
@@ -167,6 +169,8 @@ class ModelConfig:
169 self.hf_config = get_config(self.model, trust_remote_code, revision,
170 code_revision, rope_scaling, rope_theta)
171 self.hf_text_config = get_hf_text_config(self.hf_config)
172 + self.hf_image_processor_config = get_hf_image_processor_config(
173 + self.model, revision)
174 self.dtype = _get_and_verify_dtype(self.hf_text_config, dtype)
175
176 # Choose a default enforce_eager value if the user did not specify
vllm/inputs/registry.py
+9 -2
@@ -2,8 +2,8 @@ import functools
2 from array import array
3 from collections import UserDict
4 from dataclasses import dataclass
5 -from typing import (TYPE_CHECKING, Callable, Dict, Mapping, Optional, Protocol,
6 - Tuple, Type)
5 +from typing import (TYPE_CHECKING, Any, Callable, Dict, Mapping, Optional,
6 + Protocol, Tuple, Type)
7
8 from torch import nn
9 from transformers import PretrainedConfig
@@ -55,6 +55,13 @@ class InputContext:
55
56 return hf_config
57
58 + def get_hf_image_processor_config(self) -> Dict[str, Any]:
59 + """
60 + Get the HuggingFace image processor configuration of the model.
61 + """
62 +
63 + return self.model_config.hf_image_processor_config
64 +
65
66 N = TypeVar("N", bound=Type[nn.Module])
67
vllm/model_executor/models/phi3v.py
+6 -6
@@ -15,8 +15,8 @@
15 # limitations under the License.
16 import re
17 from functools import lru_cache
18 -from typing import (Iterable, List, Literal, Mapping, Optional, Tuple,
19 - TypedDict, Union)
18 +from typing import (Any, Dict, Iterable, List, Literal, Mapping, Optional,
19 + Tuple, TypedDict, Union)
20
21 import numpy as np
22 import torch
@@ -324,12 +324,12 @@ def _calc_hd_transform_size(*, width: int, height: int, hd_num: int = 16):
324
325 # Based on https://huggingface.co/microsoft/Phi-3-vision-128k-instruct/blob/main/image_processing_phi3_v.py#L181
326 def get_phi3v_image_feature_size(
327 - hf_config: PretrainedConfig,
327 + hf_config: Dict[str, Any],
328 *,
329 input_height: int,
330 input_width: int,
331 ) -> int:
332 - num_crops = getattr(hf_config, "num_crops", 16)
332 + num_crops = hf_config.get("num_crops", 16)
333 new_width, new_height = _calc_hd_transform_size(width=input_width,
334 height=input_height,
335 hd_num=num_crops)
@@ -341,7 +341,7 @@ def get_phi3v_image_feature_size(
341 def get_max_phi3v_image_tokens(ctx: InputContext):
342
343 return get_phi3v_image_feature_size(
344 - ctx.get_hf_config(),
344 + ctx.get_hf_image_processor_config(),
345 input_height=MAX_IMAGE_FEATURE_SIZE_HEIGHT,
346 input_width=MAX_IMAGE_FEATURE_SIZE_WIDTH,
347 )
@@ -395,7 +395,7 @@ def input_processor_for_phi3v(ctx: InputContext, llm_inputs: LLMInputs):
395 return llm_inputs
396
397 model_config = ctx.model_config
398 - hf_config = ctx.get_hf_config()
398 + hf_config = ctx.get_hf_image_processor_config()
399
400 image_data = multi_modal_data["image"]
401 if isinstance(image_data, Image.Image):
vllm/transformers_utils/config.py
+14 -1
@@ -1,8 +1,10 @@
1 import contextlib
2 from pathlib import Path
3 -from typing import Dict, Optional, Type, Union
3 +from typing import Any, Dict, Optional, Type, Union
4
5 from transformers import GenerationConfig, PretrainedConfig
6 +from transformers.models.auto.image_processing_auto import (
7 + get_image_processor_config)
8 from transformers.models.auto.modeling_auto import (
9 MODEL_FOR_CAUSAL_LM_MAPPING_NAMES)
10
@@ -98,6 +100,17 @@ def get_config(
100 return config
101
102
103 +def get_hf_image_processor_config(
104 + model: Union[str, Path],
105 + revision: Optional[str] = None,
106 + **kwargs,
107 +) -> Dict[str, Any]:
108 + # Separate model folder from file path for GGUF models
109 + if Path(model).is_file() and Path(model).suffix == ".gguf":
110 + model = Path(model).parent
111 + return get_image_processor_config(model, revision=revision, **kwargs)
112 +
113 +
114 def get_hf_text_config(config: PretrainedConfig):
115 """Get the "sub" config relevant to llm for multi modal models.
116 No op for pure text models.