A high-throughput and memory-efficient inference and serving engine for LLMs
Mirrored from https://github.com/tduncan-lum/Nvidia-Optimization-Stack---VLLM.git
Synced Nov 16, 18:15 UTC
[Model] Fix Phi-3.5-vision-instruct 'num_crops' issue (#7710)
zifeitong committed
Aug 21, 2024 at 18:36 UTC
df1a21131d951ba8ee65363aeb9b9486f569aa4f
6 files changed
+37
-13
docs/source/models/supported_models.rst
+2
-2
@@ -225,9 +225,9 @@ Multimodal Language Models
225
- :code:`google/paligemma-3b-pt-224`, :code:`google/paligemma-3b-mix-224`, etc.
226
-
227
* - :code:`Phi3VForCausalLM`
228
- - Phi-3-Vision
228
+ - Phi-3-Vision, Phi-3.5-Vision
229
- Image
230
- - :code:`microsoft/Phi-3-vision-128k-instruct`, etc.
230
+ - :code:`microsoft/Phi-3-vision-128k-instruct`, :code:`microsoft/Phi-3.5-vision-instruct` etc.
231
-
232
* - :code:`MiniCPMV`
233
- MiniCPM-V
tests/models/test_phi3v.py
+1
-1
@@ -21,7 +21,7 @@ HF_IMAGE_PROMPTS = IMAGE_ASSETS.prompts({
21
"<|user|>\n<|image_1|>\nWhat is the season?<|end|>\n<|assistant|>\n",
22
})
23
24
-models = ["microsoft/Phi-3-vision-128k-instruct"]
24
+models = ["microsoft/Phi-3.5-vision-instruct"]
25
26
27
def vllm_to_hf_output(vllm_output: Tuple[List[int], str,
vllm/config.py
+5
-1
@@ -13,7 +13,9 @@ from vllm.model_executor.layers.quantization import QUANTIZATION_METHODS
13
from vllm.model_executor.models import ModelRegistry
14
from vllm.platforms import current_platform
15
from vllm.tracing import is_otel_available, otel_import_error_traceback
16
-from vllm.transformers_utils.config import get_config, get_hf_text_config
16
+from vllm.transformers_utils.config import (get_config,
17
+ get_hf_image_processor_config,
18
+ get_hf_text_config)
19
from vllm.utils import (STR_NOT_IMPL_ENC_DEC_CUDAGRAPH, GiB_bytes,
20
cuda_device_count_stateless, get_cpu_memory, is_cpu,
21
is_hip, is_neuron, is_openvino, is_xpu,
@@ -167,6 +169,8 @@ class ModelConfig:
169
self.hf_config = get_config(self.model, trust_remote_code, revision,
170
code_revision, rope_scaling, rope_theta)
171
self.hf_text_config = get_hf_text_config(self.hf_config)
172
+ self.hf_image_processor_config = get_hf_image_processor_config(
173
+ self.model, revision)
174
self.dtype = _get_and_verify_dtype(self.hf_text_config, dtype)
175
176
# Choose a default enforce_eager value if the user did not specify
vllm/inputs/registry.py
+9
-2
@@ -2,8 +2,8 @@ import functools
2
from array import array
3
from collections import UserDict
4
from dataclasses import dataclass
5
-from typing import (TYPE_CHECKING, Callable, Dict, Mapping, Optional, Protocol,
6
- Tuple, Type)
5
+from typing import (TYPE_CHECKING, Any, Callable, Dict, Mapping, Optional,
6
+ Protocol, Tuple, Type)
7
8
from torch import nn
9
from transformers import PretrainedConfig
@@ -55,6 +55,13 @@ class InputContext:
55
56
return hf_config
57
58
+ def get_hf_image_processor_config(self) -> Dict[str, Any]:
59
+ """
60
+ Get the HuggingFace image processor configuration of the model.
61
+ """
62
+
63
+ return self.model_config.hf_image_processor_config
64
+
65
66
N = TypeVar("N", bound=Type[nn.Module])
67
vllm/model_executor/models/phi3v.py
+6
-6
@@ -15,8 +15,8 @@
15
# limitations under the License.
16
import re
17
from functools import lru_cache
18
-from typing import (Iterable, List, Literal, Mapping, Optional, Tuple,
19
- TypedDict, Union)
18
+from typing import (Any, Dict, Iterable, List, Literal, Mapping, Optional,
19
+ Tuple, TypedDict, Union)
20
21
import numpy as np
22
import torch
@@ -324,12 +324,12 @@ def _calc_hd_transform_size(*, width: int, height: int, hd_num: int = 16):
324
325
# Based on https://huggingface.co/microsoft/Phi-3-vision-128k-instruct/blob/main/image_processing_phi3_v.py#L181
326
def get_phi3v_image_feature_size(
327
- hf_config: PretrainedConfig,
327
+ hf_config: Dict[str, Any],
328
*,
329
input_height: int,
330
input_width: int,
331
) -> int:
332
- num_crops = getattr(hf_config, "num_crops", 16)
332
+ num_crops = hf_config.get("num_crops", 16)
333
new_width, new_height = _calc_hd_transform_size(width=input_width,
334
height=input_height,
335
hd_num=num_crops)
@@ -341,7 +341,7 @@ def get_phi3v_image_feature_size(
341
def get_max_phi3v_image_tokens(ctx: InputContext):
342
343
return get_phi3v_image_feature_size(
344
- ctx.get_hf_config(),
344
+ ctx.get_hf_image_processor_config(),
345
input_height=MAX_IMAGE_FEATURE_SIZE_HEIGHT,
346
input_width=MAX_IMAGE_FEATURE_SIZE_WIDTH,
347
)
@@ -395,7 +395,7 @@ def input_processor_for_phi3v(ctx: InputContext, llm_inputs: LLMInputs):
395
return llm_inputs
396
397
model_config = ctx.model_config
398
- hf_config = ctx.get_hf_config()
398
+ hf_config = ctx.get_hf_image_processor_config()
399
400
image_data = multi_modal_data["image"]
401
if isinstance(image_data, Image.Image):
vllm/transformers_utils/config.py
+14
-1
@@ -1,8 +1,10 @@
1
import contextlib
2
from pathlib import Path
3
-from typing import Dict, Optional, Type, Union
3
+from typing import Any, Dict, Optional, Type, Union
4
5
from transformers import GenerationConfig, PretrainedConfig
6
+from transformers.models.auto.image_processing_auto import (
7
+ get_image_processor_config)
8
from transformers.models.auto.modeling_auto import (
9
MODEL_FOR_CAUSAL_LM_MAPPING_NAMES)
10
@@ -98,6 +100,17 @@ def get_config(
100
return config
101
102
103
+def get_hf_image_processor_config(
104
+ model: Union[str, Path],
105
+ revision: Optional[str] = None,
106
+ **kwargs,
107
+) -> Dict[str, Any]:
108
+ # Separate model folder from file path for GGUF models
109
+ if Path(model).is_file() and Path(model).suffix == ".gguf":
110
+ model = Path(model).parent
111
+ return get_image_processor_config(model, revision=revision, **kwargs)
112
+
113
+
114
def get_hf_text_config(config: PretrainedConfig):
115
"""Get the "sub" config relevant to llm for multi modal models.
116
No op for pure text models.