A high-throughput and memory-efficient inference and serving engine for LLMs
Mirrored from https://github.com/tduncan-lum/Nvidia-Optimization-Stack---VLLM.git
Synced Nov 14, 05:30 UTC
Migrate Qwen2 inputs to TensorSchema (#23475)
Signed-off-by: Benji Beck <benjibeck@meta.com> Co-authored-by: gemini-code-assist[bot] <176961590+gemini-code-assist[bot]@users.noreply.github.com>
Benji Beck committed
Sep 6, 2025 at 20:07 UTC
37a6fa95fd29e95de6ceaf7ccc249c4523be6c75
4 files changed
+257
-164
vllm/model_executor/models/qwen2_5_omni_thinker.py
+31
-21
@@ -25,7 +25,7 @@
25
from collections.abc import Iterable, Mapping, Sequence
26
from copy import copy
27
from functools import partial
28
-from typing import Any, Callable, Optional, Union
28
+from typing import Annotated, Any, Callable, Literal, Optional, Union
29
30
import torch
31
import torch.nn as nn
@@ -41,15 +41,13 @@ from transformers.models.whisper import WhisperFeatureExtractor
41
from vllm.config import VllmConfig
42
from vllm.logger import init_logger
43
from vllm.model_executor.layers.rotary_embedding import MRotaryEmbedding
44
-from vllm.model_executor.models.module_mapping import MultiModelKeys
44
from vllm.model_executor.models.qwen2_5_vl import (
45
Qwen2_5_VisionTransformer, Qwen2_5_VLImageEmbeddingInputs,
46
Qwen2_5_VLImageInputs, Qwen2_5_VLImagePixelInputs,
47
Qwen2_5_VLProcessingInfo, Qwen2_5_VLVideoEmbeddingInputs,
48
Qwen2_5_VLVideoInputs, Qwen2_5_VLVideoPixelInputs)
49
from vllm.model_executor.models.qwen2_audio import (
51
- Qwen2AudioFeatureInputs, Qwen2AudioProcessingInfo,
52
- _get_feat_extract_output_lengths)
50
+ Qwen2AudioProcessingInfo, _get_feat_extract_output_lengths)
51
from vllm.model_executor.models.qwen2_vl import Qwen2VLMultiModalDataParser
52
from vllm.model_executor.sampling_metadata import SamplingMetadata
53
from vllm.multimodal import MULTIMODAL_REGISTRY
@@ -66,9 +64,9 @@ from vllm.multimodal.processing import (BaseMultiModalProcessor,
64
from vllm.multimodal.profiling import BaseDummyInputsBuilder
65
from vllm.sequence import IntermediateTensors
66
from vllm.transformers_utils.tokenizer import decode_tokens, encode_tokens
67
+from vllm.utils.tensor_schema import TensorSchema, TensorShape
68
70
-from .interfaces import (MultiModalEmbeddings, SupportsLoRA,
71
- SupportsMultiModal, SupportsPP)
69
+from .interfaces import MultiModalEmbeddings, SupportsMultiModal, SupportsPP
70
from .utils import (AutoWeightsLoader, WeightsMapper,
71
init_vllm_registered_model, maybe_prefix,
72
merge_multimodal_embeddings)
@@ -81,6 +79,26 @@ except (ImportError, ModuleNotFoundError):
79
logger = init_logger(__name__)
80
81
82
+class Qwen2_5OmniAudioFeatureInputs(TensorSchema):
83
+ """
84
+ Dimensions:
85
+ - na: Number of audios
86
+ - nmb: Number of mel bins
87
+ - msl: Maximum sequence length
88
+ - tsl: Total sequence length
89
+ """
90
+ type: Literal["audio_features"]
91
+ input_features: Annotated[
92
+ Union[torch.Tensor, list[torch.Tensor]],
93
+ TensorShape("nmb", "tsl"),
94
+ ]
95
+
96
+ feature_attention_mask: Annotated[
97
+ torch.Tensor,
98
+ TensorShape("na", "msl"),
99
+ ]
100
+
101
+
102
def create_qwen2_5_omni_thinker_field_factory(
103
spatial_merge_size: int
104
) -> Callable[[Mapping[str, torch.Tensor]], Mapping[str,
@@ -536,7 +554,7 @@ class Qwen2_5OmniConditionalGenerationMixin:
554
return torch.concat(mm_input, dim=dim)
555
556
def _parse_and_validate_audio_input(
539
- self, **kwargs: object) -> Optional[Qwen2AudioFeatureInputs]:
557
+ self, **kwargs: object) -> Optional[Qwen2_5OmniAudioFeatureInputs]:
558
input_audio_features = kwargs.pop('input_audio_features', None)
559
audio_feature_lengths = kwargs.pop('audio_feature_lengths', None)
560
feature_attention_mask = kwargs.pop('feature_attention_mask', None)
@@ -550,7 +568,8 @@ class Qwen2_5OmniConditionalGenerationMixin:
568
if not isinstance(input_audio_features, (torch.Tensor, list)):
569
raise ValueError("Incorrect type of audio input features. "
570
f"Got type: {type(input_audio_features)}")
553
- return Qwen2AudioFeatureInputs(
571
+ return Qwen2_5OmniAudioFeatureInputs(
572
+ type="audio_features",
573
input_features=input_audio_features,
574
audio_feature_lengths=audio_feature_lengths,
575
feature_attention_mask=feature_attention_mask)
@@ -633,7 +652,7 @@ class Qwen2_5OmniConditionalGenerationMixin:
652
653
def _process_audio_input(
654
self,
636
- audio_input: Qwen2AudioFeatureInputs,
655
+ audio_input: Qwen2_5OmniAudioFeatureInputs,
656
audio_hashes: list[str] = None,
657
cached_audio_features: torch.Tensor = None,
658
) -> torch.Tensor:
@@ -660,8 +679,8 @@ class Qwen2_5OmniConditionalGenerationMixin:
679
feature_lens=audio_feature_lengths,
680
aftercnn_lens=audio_feat_lengths,
681
)
663
- audio_features = audio_outputs.last_hidden_state
664
- return audio_features.split(audio_output_lengths.tolist())
682
+ return audio_outputs.last_hidden_state.split(
683
+ audio_output_lengths.tolist())
684
685
def _process_image_input(
686
self,
@@ -707,7 +726,7 @@ class Qwen2_5OmniConditionalGenerationMixin:
726
dummy_inputs=Qwen2_5OmniThinkerDummyInputsBuilder,
727
)
728
class Qwen2_5OmniThinkerForConditionalGeneration(
710
- nn.Module, SupportsMultiModal, SupportsPP, SupportsLoRA,
729
+ nn.Module, SupportsMultiModal, SupportsPP,
730
Qwen2_5OmniConditionalGenerationMixin):
731
hf_to_vllm_mapper = WeightsMapper(
732
orig_to_new_prefix={
@@ -800,15 +819,6 @@ class Qwen2_5OmniThinkerForConditionalGeneration(
819
def get_language_model(self) -> torch.nn.Module:
820
return self.language_model
821
803
- def get_mm_mapping(self) -> MultiModelKeys:
804
- """Get module prefix for multimodal models to filter LoRA modules."""
805
- return MultiModelKeys.from_string_field(
806
- language_model="language_model",
807
- connector=[], # No explicit connector in this model
808
- tower_model=["visual",
809
- "audio_tower"], # Exclude vision and audio towers
810
- )
811
-
822
def get_multimodal_embeddings(self,
823
**kwargs: object) -> MultiModalEmbeddings:
824
vllm/model_executor/models/qwen2_5_vl.py
+103
-70
@@ -27,7 +27,7 @@
27
"""Inference-only Qwen2.5-VL model compatible with HuggingFace weights."""
28
from collections.abc import Iterable, Mapping
29
from functools import lru_cache, partial
30
-from typing import Callable, Literal, Optional, TypedDict, Union
30
+from typing import Annotated, Callable, Literal, Optional, Union
31
32
import torch
33
import torch.nn as nn
@@ -64,6 +64,7 @@ from vllm.multimodal.utils import run_dp_sharded_mrope_vision_model
64
from vllm.platforms import _Backend
65
from vllm.sequence import IntermediateTensors
66
from vllm.transformers_utils.config import uses_mrope
67
+from vllm.utils.tensor_schema import TensorSchema, TensorShape
68
69
from .interfaces import (MultiModalEmbeddings, SupportsLoRA,
70
SupportsMultiModal, SupportsPP, SupportsQuant)
@@ -80,84 +81,125 @@ logger = init_logger(__name__)
81
# === Vision Inputs === #
82
83
83
-class Qwen2_5_VLImagePixelInputs(TypedDict):
84
- type: Literal["pixel_values"]
85
- pixel_values: torch.Tensor
86
- """Shape:
87
- `(num_patches, num_channels * patch_size * patch_size)`
84
+class Qwen2_5_VLImagePixelInputs(TensorSchema):
85
"""
89
-
90
- image_grid_thw: torch.Tensor
91
- """Shape: `(num_images, 3)`
92
- This should be in `(grid_t, grid_h, grid_w)` format.
86
+ Dimensions:
87
+ - np: Number of patches
88
+ - ni: Number of images
89
+ - cps: Number of channels * patch_size * patch_size
90
+
91
+ Historical context:
92
+ - pixel_values shape: (num_patches, num_channels * patch_size *
93
+ patch_size)
94
+ - image_grid_thw shape: (num_images, 3) in (grid_t, grid_h, grid_w)
95
+ formatnum_channels * patch_size * patch_size
96
"""
97
+ type: Literal["pixel_values"]
98
+
99
+ pixel_values: Annotated[
100
+ torch.Tensor,
101
+ TensorShape("np", "cps"),
102
+ ]
103
104
+ image_grid_thw: Annotated[
105
+ torch.Tensor,
106
+ TensorShape("ni", 3),
107
+ ]
108
96
-class Qwen2_5_VLImageEmbeddingInputs(TypedDict):
97
- type: Literal["image_embeds"]
98
- image_embeds: torch.Tensor
99
- """Supported types:
100
- - list[`torch.Tensor`]: A list of tensors holding all images' features.
101
- Each tensor holds an image's features.
102
- - `torch.Tensor`: A tensor holding all images' features
103
- (concatenation of all images' feature tensors).
104
-
105
- Tensor shape: `(num_image_features, hidden_size)`
106
- - `num_image_features` varies based on
107
- the number and resolution of the images.
108
- - `hidden_size` must match the hidden size of language model backbone.
109
- """
109
111
- image_grid_thw: torch.Tensor
112
- """Shape: `(num_images, 3)`
113
- This should be in `(grid_t, grid_h, grid_w)` format.
110
+class Qwen2_5_VLImageEmbeddingInputs(TensorSchema):
111
"""
112
+ Dimensions:
113
+ - nf: Number of image features
114
+ - hs: Hidden size
115
+ - ni: Number of images
116
+
117
+ Historical context:
118
+ - image_embeds shape: (num_image_features, hidden_size)
119
+ - num_image_features varies based on the number and resolution of the
120
+ images.
121
+ - hidden_size must match the hidden size of language model backbone.
122
+ - image_grid_thw shape: (num_images, 3) in (grid_t, grid_h, grid_w)
123
+ format
124
+ """
125
+ type: Literal["image_embeds"]
126
+
127
+ image_embeds: Annotated[
128
+ torch.Tensor,
129
+ TensorShape("nf", "hs"),
130
+ ]
131
+
132
+ image_grid_thw: Annotated[
133
+ torch.Tensor,
134
+ TensorShape("ni", 3),
135
+ ]
136
137
138
Qwen2_5_VLImageInputs = Union[Qwen2_5_VLImagePixelInputs,
139
Qwen2_5_VLImageEmbeddingInputs]
140
141
121
-class Qwen2_5_VLVideoPixelInputs(TypedDict):
122
- type: Literal["pixel_values_videos"]
123
- pixel_values_videos: torch.Tensor
124
- """Shape:
125
- `(num_patches,
126
- num_channels * temporal_patch_size * patch_size * patch_size)`
142
+class Qwen2_5_VLVideoPixelInputs(TensorSchema):
143
+ """
144
+ Dimensions:
145
+ - np: Number of patches
146
+ - nv: Number of videos
147
+ - ctps: Number of channels * temporal_patch_size * patch_size *
148
+ patch_size
149
+
150
+ Historical context:
151
+ - pixel_values_videos shape: (num_patches, num_channels *
152
+ temporal_patch_size * patch_size * patch_size)
153
+ - video_grid_thw shape: (num_videos, 3) in (grid_t, grid_h, grid_w)
154
+ format
155
+ - second_per_grid_ts: The video time interval (in seconds) for each
156
+ grid along the temporal dimension in the 3D position IDs. Returned
157
+ when `videos` is not `None`.
158
"""
159
+ type: Literal["pixel_values_videos"]
160
129
- video_grid_thw: torch.Tensor
130
- """Shape: `(num_videos, 3)`
161
+ pixel_values_videos: Annotated[
162
+ torch.Tensor,
163
+ TensorShape("np", "ctps"),
164
+ ]
165
132
- This should be in `(grid_t, grid_h, grid_w)` format.
133
- """
166
+ video_grid_thw: Annotated[
167
+ torch.Tensor,
168
+ TensorShape("nv", 3),
169
+ ]
170
135
- second_per_grid_ts: torch.Tensor
136
- """
137
- The video time interval (in seconds) for each grid along the temporal
138
- dimension in the 3D position IDs. Returned when `videos` is not `None`.
139
- """
171
+ second_per_grid_ts: Annotated[
172
+ Optional[torch.Tensor],
173
+ TensorShape("nv"),
174
+ ]
175
176
142
-class Qwen2_5_VLVideoEmbeddingInputs(TypedDict):
143
- type: Literal["video_embeds"]
144
- video_embeds: torch.Tensor
145
- """Supported types:
146
- - list[`torch.Tensor`]: A list of tensors holding all videos' features.
147
- Each tensor holds an video's features.
148
- - `torch.Tensor`: A tensor holding all videos' features
149
- (concatenation of all videos' feature tensors).
150
-
151
- Tensor shape: `(num_image_features, hidden_size)`
152
- - `num_image_features` varies based on
153
- the number and resolution of the videos.
154
- - `hidden_size` must match the hidden size of language model backbone.
177
+class Qwen2_5_VLVideoEmbeddingInputs(TensorSchema):
178
"""
156
-
157
- video_grid_thw: torch.Tensor
158
- """Shape: `(num_videos, 3)`
159
- This should be in `(grid_t, grid_h, grid_w)` format.
179
+ Dimensions:
180
+ - nf: Number of video features
181
+ - hs: Hidden size
182
+ - nv: Number of videos
183
+
184
+ Historical context:
185
+ - video_embeds shape: (num_video_features, hidden_size)
186
+ - num_video_features varies based on the number and resolution of the
187
+ videos.
188
+ - hidden_size must match the hidden size of language model backbone.
189
+ - video_grid_thw shape: (num_videos, 3) in (grid_t, grid_h, grid_w)
190
+ format
191
"""
192
+ type: Literal["video_embeds"]
193
+
194
+ video_embeds: Annotated[
195
+ torch.Tensor,
196
+ TensorShape("nf", "hs"),
197
+ ]
198
+
199
+ video_grid_thw: Annotated[
200
+ torch.Tensor,
201
+ TensorShape("nv", 3),
202
+ ]
203
204
205
Qwen2_5_VLVideoInputs = Union[Qwen2_5_VLVideoPixelInputs,
@@ -936,10 +978,6 @@ class Qwen2_5_VLForConditionalGeneration(nn.Module, SupportsMultiModal,
978
image_grid_thw = self._validate_and_reshape_mm_tensor(
979
image_grid_thw, "image grid_thw")
980
939
- if not isinstance(pixel_values, (torch.Tensor, list)):
940
- raise ValueError("Incorrect type of image pixel values. "
941
- f"Got type: {type(pixel_values)}")
942
-
981
return Qwen2_5_VLImagePixelInputs(type="pixel_values",
982
pixel_values=pixel_values,
983
image_grid_thw=image_grid_thw)
@@ -950,9 +988,6 @@ class Qwen2_5_VLForConditionalGeneration(nn.Module, SupportsMultiModal,
988
image_grid_thw = self._validate_and_reshape_mm_tensor(
989
image_grid_thw, "image grid_thw")
990
953
- if not isinstance(image_embeds, torch.Tensor):
954
- raise ValueError("Incorrect type of image embeddings. "
955
- f"Got type: {type(image_embeds)}")
991
return Qwen2_5_VLImageEmbeddingInputs(
992
type="image_embeds",
993
image_embeds=image_embeds,
@@ -973,7 +1008,8 @@ class Qwen2_5_VLForConditionalGeneration(nn.Module, SupportsMultiModal,
1008
pixel_values_videos, "video pixel values")
1009
video_grid_thw = self._validate_and_reshape_mm_tensor(
1010
video_grid_thw, "video grid_thw")
976
-
1011
+ if second_per_grid_ts is not None and second_per_grid_ts.ndim == 2:
1012
+ second_per_grid_ts = second_per_grid_ts.squeeze(-1)
1013
return Qwen2_5_VLVideoPixelInputs(
1014
type="pixel_values_videos",
1015
pixel_values_videos=pixel_values_videos,
@@ -987,9 +1023,6 @@ class Qwen2_5_VLForConditionalGeneration(nn.Module, SupportsMultiModal,
1023
video_grid_thw = self._validate_and_reshape_mm_tensor(
1024
video_grid_thw, "video grid_thw")
1025
990
- if not isinstance(video_embeds, torch.Tensor):
991
- raise ValueError("Incorrect type of video embeddings. "
992
- f"Got type: {type(video_embeds)}")
1026
return Qwen2_5_VLVideoEmbeddingInputs(
1027
type="video_embeds",
1028
video_embeds=video_embeds,
vllm/model_executor/models/qwen2_audio.py
+29
-11
@@ -23,7 +23,7 @@
23
# limitations under the License.
24
"""Inference-only Qwen2-Audio model compatible with HuggingFace weights."""
25
from collections.abc import Iterable, Mapping, Sequence
26
-from typing import Any, Literal, Optional, TypedDict, Union
26
+from typing import Annotated, Any, Literal, Optional, Union
27
28
import torch
29
import torch.nn as nn
@@ -47,6 +47,7 @@ from vllm.multimodal.processing import (BaseMultiModalProcessor,
47
PromptUpdate, PromptUpdateDetails)
48
from vllm.multimodal.profiling import BaseDummyInputsBuilder
49
from vllm.sequence import IntermediateTensors
50
+from vllm.utils.tensor_schema import TensorSchema, TensorShape
51
52
from .interfaces import MultiModalEmbeddings, SupportsMultiModal, SupportsPP
53
from .utils import (AutoWeightsLoader, init_vllm_registered_model,
@@ -54,21 +55,38 @@ from .utils import (AutoWeightsLoader, init_vllm_registered_model,
55
56
57
# # === Audio Inputs === #
57
-class Qwen2AudioFeatureInputs(TypedDict):
58
+class Qwen2AudioFeatureInputs(TensorSchema):
59
+ """
60
+ Dimensions:
61
+ - na: Number of audios
62
+ - nmb: Number of mel bins
63
+ """
64
type: Literal["audio_features"]
59
- input_features: torch.Tensor
60
- """Shape: `(num_audios, num_mel_bins, 3000)`"""
65
+ input_features: Annotated[
66
+ Union[torch.Tensor, list[torch.Tensor]],
67
+ TensorShape("na", "nmb", 3000),
68
+ ]
69
62
- feature_attention_mask: torch.Tensor
63
- """Shape: `(num_audios, 3000)`"""
70
+ feature_attention_mask: Annotated[
71
+ torch.Tensor,
72
+ TensorShape("na", 3000),
73
+ ]
74
75
66
-class Qwen2AudioEmbeddingInputs(TypedDict):
67
- type: Literal["audio_embeds"]
68
- audio_embeds: list[torch.Tensor]
69
- """Shape: `(num_audio_features, hidden_size)`
70
- `hidden_size` must match the hidden size of language model backbone.
76
+class Qwen2AudioEmbeddingInputs(TensorSchema):
77
"""
78
+ Dimensions:
79
+ - bn: Batch size
80
+ - naf: Number of audio features
81
+ - hs: Hidden size (must match the hidden size of language model
82
+ backbone)
83
+ """
84
+ type: Literal["audio_embeds"] = "audio_embeds"
85
+
86
+ audio_embeds: Annotated[
87
+ list[torch.Tensor],
88
+ TensorShape("bn", "naf", "hs"),
89
+ ]
90
91
92
Qwen2AudioInputs = Union[Qwen2AudioFeatureInputs, Qwen2AudioEmbeddingInputs]
vllm/model_executor/models/qwen2_vl.py
+94
-62
@@ -26,7 +26,7 @@
26
"""Inference-only Qwen2-VL model compatible with HuggingFace weights."""
27
from collections.abc import Iterable, Mapping, Sequence
28
from functools import partial
29
-from typing import Any, Callable, Literal, Optional, TypedDict, Union
29
+from typing import Annotated, Any, Callable, Literal, Optional, Union
30
31
import torch
32
import torch.nn as nn
@@ -70,6 +70,7 @@ from vllm.platforms import _Backend, current_platform
70
from vllm.sequence import IntermediateTensors
71
from vllm.transformers_utils.config import uses_mrope
72
from vllm.transformers_utils.tokenizer import AnyTokenizer
73
+from vllm.utils.tensor_schema import TensorSchema, TensorShape
74
75
from .interfaces import (MultiModalEmbeddings, SupportsLoRA,
76
SupportsMultiModal, SupportsPP)
@@ -86,78 +87,119 @@ _MAX_FRAMES_PER_VIDEO = 16
87
# === Vision Inputs === #
88
89
89
-class Qwen2VLImagePixelInputs(TypedDict):
90
- type: Literal["pixel_values"]
91
- pixel_values: torch.Tensor
92
- """Shape:
93
- `(num_patches, num_channels * patch_size * patch_size)`
90
+class Qwen2VLImagePixelInputs(TensorSchema):
91
"""
95
-
96
- image_grid_thw: torch.Tensor
97
- """Shape: `(num_images, 3)`
98
- This should be in `(grid_t, grid_h, grid_w)` format.
92
+ Dimensions:
93
+ - np: The total number of patches over each image over each prompt in
94
+ the batch
95
+ - ni: Number of images
96
+ - cps: Number of channels * patch_size * patch_size
97
+
98
+ Historical context:
99
+ - pixel_values shape: (num_patches, num_channels * patch_size *
100
+ patch_size)
101
+ - image_grid_thw shape: (num_images, 3) in (grid_t, grid_h, grid_w)
102
+ format
103
"""
104
+ type: Literal["pixel_values"]
105
106
+ pixel_values: Annotated[
107
+ torch.Tensor,
108
+ TensorShape("np", "cps"),
109
+ ]
110
102
-class Qwen2VLImageEmbeddingInputs(TypedDict):
103
- type: Literal["image_embeds"]
104
- image_embeds: torch.Tensor
105
- """Supported types:
106
- - list[`torch.Tensor`]: A list of tensors holding all images' features.
107
- Each tensor holds an image's features.
108
- - `torch.Tensor`: A tensor holding all images' features
109
- (concatenation of all images' feature tensors).
111
+ image_grid_thw: Annotated[
112
+ torch.Tensor,
113
+ TensorShape("ni", 3),
114
+ ]
115
+
116
+
117
+class Qwen2VLImageEmbeddingInputs(TensorSchema):
118
+ """
119
+ Dimensions:
120
+ - nf: Number of image features
121
+ - hs: Hidden size
122
+ - ni: Number of images
123
111
- Tensor shape: `(num_image_features, hidden_size)`
112
- - `num_image_features` varies based on
113
- the number and resolution of the images.
114
- - `hidden_size` must match the hidden size of language model backbone.
124
+ Historical context:
125
+ - image_embeds shape: (num_image_features, hidden_size)
126
+ - num_image_features varies based on the number and resolution of the
127
+ images.
128
+ - hidden_size must match the hidden size of language model backbone.
129
+ - image_grid_thw shape: (num_images, 3) in (grid_t, grid_h, grid_w)
130
+ format
131
"""
132
+ type: Literal["image_embeds"]
133
117
- image_grid_thw: torch.Tensor
118
- """Shape: `(num_images, 3)`
119
- This should be in `(grid_t, grid_h, grid_w)` format.
120
- """
134
+ image_embeds: Annotated[
135
+ torch.Tensor,
136
+ TensorShape("nf", "hs"),
137
+ ]
138
+
139
+ image_grid_thw: Annotated[
140
+ torch.Tensor,
141
+ TensorShape("ni", 3),
142
+ ]
143
144
145
Qwen2VLImageInputs = Union[Qwen2VLImagePixelInputs,
146
Qwen2VLImageEmbeddingInputs]
147
148
127
-class Qwen2VLVideoPixelInputs(TypedDict):
128
- type: Literal["pixel_values_videos"]
129
- pixel_values_videos: torch.Tensor
130
- """Shape:
131
- `(num_patches,
132
- num_channels * temporal_patch_size * patch_size * patch_size)`
149
+class Qwen2VLVideoPixelInputs(TensorSchema):
150
+ """
151
+ Dimensions:
152
+ - np: The total number of patches over each video over each prompt in
153
+ the batch
154
+ - ctps: Number of channels * temporal_patch_size * patch_size *
155
+ patch_size
156
+ - nv: Number of videos
157
+
158
+ Historical context:
159
+ - pixel_values_videos shape: (num_patches, num_channels *
160
+ temporal_patch_size * patch_size * patch_size)
161
+ - video_grid_thw shape: (num_videos, 3) in (grid_t, grid_h, grid_w)
162
+ format
163
"""
164
+ type: Literal["pixel_values_videos"]
165
135
- video_grid_thw: torch.Tensor
136
- """Shape: `(num_videos, 3)`
166
+ pixel_values_videos: Annotated[
167
+ torch.Tensor,
168
+ TensorShape("np", "ctps"),
169
+ ]
170
138
- This should be in `(grid_t, grid_h, grid_w)` format.
139
- """
171
+ video_grid_thw: Annotated[
172
+ torch.Tensor,
173
+ TensorShape("nv", 3),
174
+ ]
175
176
142
-class Qwen2VLVideoEmbeddingInputs(TypedDict):
143
- type: Literal["video_embeds"]
144
- video_embeds: torch.Tensor
145
- """Supported types:
146
- - list[`torch.Tensor`]: A list of tensors holding all videos' features.
147
- Each tensor holds an video's features.
148
- - `torch.Tensor`: A tensor holding all videos' features
149
- (concatenation of all videos' feature tensors).
177
+class Qwen2VLVideoEmbeddingInputs(TensorSchema):
178
+ """
179
+ Dimensions:
180
+ - nf: Number of video features
181
+ - hs: Hidden size
182
+ - nv: Number of videos
183
151
- Tensor shape: `(num_image_features, hidden_size)`
152
- - `num_image_features` varies based on
153
- the number and resolution of the videos.
154
- - `hidden_size` must match the hidden size of language model backbone.
184
+ Historical context:
185
+ - video_embeds shape: (num_video_features, hidden_size)
186
+ - num_video_features varies based on the number and resolution of the
187
+ videos.
188
+ - hidden_size must match the hidden size of language model backbone.
189
+ - video_grid_thw shape: (num_videos, 3) in (grid_t, grid_h, grid_w)
190
+ format
191
"""
192
+ type: Literal["video_embeds"]
193
157
- video_grid_thw: torch.Tensor
158
- """Shape: `(num_videos, 3)`
159
- This should be in `(grid_t, grid_h, grid_w)` format.
160
- """
194
+ video_embeds: Annotated[
195
+ torch.Tensor,
196
+ TensorShape("nf", "hs"),
197
+ ]
198
+
199
+ video_grid_thw: Annotated[
200
+ torch.Tensor,
201
+ TensorShape("nv", 3),
202
+ ]
203
204
205
Qwen2VLVideoInputs = Union[Qwen2VLVideoPixelInputs,
@@ -1126,10 +1168,6 @@ class Qwen2VLForConditionalGeneration(nn.Module, SupportsMultiModal,
1168
image_grid_thw = self._validate_and_reshape_mm_tensor(
1169
image_grid_thw, "image grid_thw")
1170
1129
- if not isinstance(pixel_values, (torch.Tensor, list)):
1130
- raise ValueError("Incorrect type of image pixel values. "
1131
- f"Got type: {type(pixel_values)}")
1132
-
1171
return Qwen2VLImagePixelInputs(type="pixel_values",
1172
pixel_values=pixel_values,
1173
image_grid_thw=image_grid_thw)
@@ -1140,9 +1178,6 @@ class Qwen2VLForConditionalGeneration(nn.Module, SupportsMultiModal,
1178
image_grid_thw = self._validate_and_reshape_mm_tensor(
1179
image_grid_thw, "image grid_thw")
1180
1143
- if not isinstance(image_embeds, torch.Tensor):
1144
- raise ValueError("Incorrect type of image embeddings. "
1145
- f"Got type: {type(image_embeds)}")
1181
return Qwen2VLImageEmbeddingInputs(type="image_embeds",
1182
image_embeds=image_embeds,
1183
image_grid_thw=image_grid_thw)
@@ -1174,9 +1209,6 @@ class Qwen2VLForConditionalGeneration(nn.Module, SupportsMultiModal,
1209
video_grid_thw = self._validate_and_reshape_mm_tensor(
1210
video_grid_thw, "video grid_thw")
1211
1177
- if not isinstance(video_embeds, torch.Tensor):
1178
- raise ValueError("Incorrect type of video embeddings. "
1179
- f"Got type: {type(video_embeds)}")
1212
return Qwen2VLVideoEmbeddingInputs(type="video_embeds",
1213
video_embeds=video_embeds,
1214
video_grid_thw=video_grid_thw)