A high-throughput and memory-efficient inference and serving engine for LLMs
Mirrored from https://github.com/tduncan-lum/Nvidia-Optimization-Stack---VLLM.git
Synced Nov 14, 05:30 UTC
[Bugfix] Fix MiniCPMV Image input inference failed (#22813)
Signed-off-by: HWH <67449739+jio-H@users.noreply.github.com> Signed-off-by: DarkLight1337 <tlleungac@connect.ust.hk> Signed-off-by: Cyrus Leung <cyrus.tl.leung@gmail.com> Co-authored-by: DarkLight1337 <tlleungac@connect.ust.hk> Co-authored-by: Cyrus Leung <cyrus.tl.leung@gmail.com>
HWH committed
Aug 14, 2025 at 00:41 UTC
9bd9294f0e0434190bc4cb3886288572b26e272b
2 files changed
+60
-27
vllm/model_executor/models/minicpmv.py
+17
@@ -85,6 +85,23 @@ class MiniCPMVImagePixelInputs(TensorSchema):
85
- w: Width
86
"""
87
88
+ def _validate_nested_tensors(
89
+ self,
90
+ value: Union[list[torch.Tensor], tuple[torch.Tensor, ...]],
91
+ field_name: str,
92
+ expected_shape: tuple[Union[int, str], ...],
93
+ dynamic_dims: set[str],
94
+ ) -> tuple[int, ...]:
95
+ # value[0] is the scaled image,
96
+ # and value[1:] is a collection of image slices.
97
+ # It is ensured that all slices in the collection
98
+ # have the same shape.
99
+ if field_name == "pixel_values":
100
+ value = value[1:] if len(value) > 1 else value
101
+
102
+ return super()._validate_nested_tensors(value, field_name,
103
+ expected_shape, dynamic_dims)
104
+
105
type: Literal["pixel_values"] = "pixel_values"
106
107
# Note that the image size may vary, so we pass it as a list instead of a
vllm/utils/tensor_schema.py
+43
-27
@@ -1,6 +1,7 @@
1
# SPDX-License-Identifier: Apache-2.0
2
# SPDX-FileCopyrightText: Copyright contributors to the vLLM project
3
-from typing import Annotated, Any, Union, get_args, get_origin, get_type_hints
3
+from typing import (Annotated, Any, Optional, Union, get_args, get_origin,
4
+ get_type_hints)
5
6
import torch
7
@@ -11,9 +12,13 @@ logger = init_logger(__name__)
12
13
class TensorShape:
14
14
- def __init__(self,
15
- *dims: Union[int, str],
16
- dynamic_dims: set[str, ...] = None) -> None:
15
+ def __init__(
16
+ self,
17
+ *dims: Union[int, str],
18
+ dynamic_dims: Optional[set[str]] = None,
19
+ ) -> None:
20
+ super().__init__()
21
+
22
self.dims = dims
23
self.dynamic_dims = dynamic_dims if dynamic_dims else set()
24
@@ -44,11 +49,15 @@ class TensorShape:
49
50
class TensorSchema:
51
47
- def __init__(self,
48
- *,
49
- validate: bool = True,
50
- resolve_bindings: dict[str, int] = None,
51
- **kwargs: Any) -> None:
52
+ def __init__(
53
+ self,
54
+ *,
55
+ validate: bool = True,
56
+ resolve_bindings: Optional[dict[str, int]] = None,
57
+ **kwargs: Any,
58
+ ) -> None:
59
+ super().__init__()
60
+
61
self._resolve_bindings = resolve_bindings if resolve_bindings else {}
62
63
for key, value in kwargs.items():
@@ -57,16 +66,19 @@ class TensorSchema:
66
if validate:
67
self.validate()
68
60
- def __getitem__(self, item) -> Any:
61
- return getattr(self, item)
69
+ def __getitem__(self, key: str) -> Any:
70
+ return getattr(self, key)
71
63
- def get(self, item, default=None) -> Any:
64
- return getattr(self, item, default)
72
+ def get(self, key: str, default: Any = None) -> Any:
73
+ return getattr(self, key, default)
74
66
- def _match_shape_with_dynamic(self, actual: tuple[int, ...],
67
- reference: tuple[int, ...],
68
- expected_shape: tuple[Union[int, str], ...],
69
- dynamic_dims: set[str, ...]) -> bool:
75
+ def _match_shape_with_dynamic(
76
+ self,
77
+ actual: tuple[int, ...],
78
+ reference: tuple[int, ...],
79
+ expected_shape: tuple[Union[int, str], ...],
80
+ dynamic_dims: set[str],
81
+ ) -> bool:
82
if len(actual) != len(reference) or len(actual) > len(expected_shape):
83
return False
84
@@ -84,10 +96,12 @@ class TensorSchema:
96
return True
97
98
def _validate_nested_tensors(
87
- self, value: Union[list[torch.Tensor, ...],
88
- tuple[torch.Tensor, ...]], field_name: str,
89
- expected_shape: tuple[Union[int, str], ...],
90
- dynamic_dims: set[str, ...]) -> tuple[int, ...]:
99
+ self,
100
+ value: Union[list[torch.Tensor], tuple[torch.Tensor, ...]],
101
+ field_name: str,
102
+ expected_shape: tuple[Union[int, str], ...],
103
+ dynamic_dims: set[str],
104
+ ) -> tuple[int, ...]:
105
"""Validate a list/tuple of tensors and return the actual shape."""
106
# Ensure all tensors in the list have the same
107
# shape, besides dynamic dimensions
@@ -110,12 +124,14 @@ class TensorSchema:
124
# shape = (len(list), *tensor.shape)
125
return (len(value), ) + first.shape
126
113
- def _validate_tensor_shape_expected(self, actual_shape: tuple[int, ...],
114
- expected_shape: tuple[Union[int, str],
115
- ...],
116
- field_name: str, shape_env: dict[str,
117
- int],
118
- dynamic_dims: set[str, ...]) -> None:
127
+ def _validate_tensor_shape_expected(
128
+ self,
129
+ actual_shape: tuple[int, ...],
130
+ expected_shape: tuple[Union[int, str], ...],
131
+ field_name: str,
132
+ shape_env: dict[str, int],
133
+ dynamic_dims: set[str],
134
+ ) -> None:
135
"""Validate that the actual tensor shape matches the expected shape."""
136
137
if len(actual_shape) != len(expected_shape):