A high-throughput and memory-efficient inference and serving engine for LLMs
Mirrored from https://github.com/tduncan-lum/Nvidia-Optimization-Stack---VLLM.git
Synced Nov 14, 05:30 UTC
[Model] Remove unnecessary CUDA sync of Qwen2VL image and video preprocess (#24334)
Signed-off-by: Win <chatcharinsang@gmail.com> Co-authored-by: Roger Wang <hey@rogerw.io>
Chatcharin Sangbutsarakum committed
Sep 8, 2025 at 13:11 UTC
60f0843ef8fb4b0c4e6788acc042873a0a2ea2a1
1 file changed
+8
-4
vllm/model_executor/models/qwen2_vl.py
+8
-4
@@ -1218,6 +1218,7 @@ class Qwen2VLForConditionalGeneration(nn.Module, SupportsMultiModal,
1218
1219
grid_thw = image_input["image_grid_thw"]
1220
assert grid_thw.ndim == 2
1221
+ grid_thw_list = grid_thw.tolist()
1222
1223
if image_input["type"] == "image_embeds":
1224
image_embeds = image_input["image_embeds"]
@@ -1227,15 +1228,17 @@ class Qwen2VLForConditionalGeneration(nn.Module, SupportsMultiModal,
1228
1229
# Split concatenated embeddings for each image item.
1230
merge_size = self.visual.spatial_merge_size
1230
- sizes = grid_thw.prod(-1) // merge_size // merge_size
1231
+ sizes = (torch.tensor(grid_thw_list, dtype=torch.long).prod(-1) //
1232
+ (merge_size * merge_size)).tolist()
1233
1232
- return image_embeds.split(sizes.tolist())
1234
+ return image_embeds.split(sizes)
1235
1236
def _process_video_input(
1237
self, video_input: Qwen2VLVideoInputs) -> tuple[torch.Tensor, ...]:
1238
1239
grid_thw = video_input["video_grid_thw"]
1240
assert grid_thw.ndim == 2
1241
+ grid_thw_list = grid_thw.tolist()
1242
1243
if video_input["type"] == "video_embeds":
1244
video_embeds = video_input["video_embeds"]
@@ -1245,9 +1248,10 @@ class Qwen2VLForConditionalGeneration(nn.Module, SupportsMultiModal,
1248
1249
# Split concatenated embeddings for each video item.
1250
merge_size = self.visual.spatial_merge_size
1248
- sizes = grid_thw.prod(-1) // merge_size // merge_size
1251
+ sizes = (torch.tensor(grid_thw_list, dtype=torch.long).prod(-1) //
1252
+ (merge_size * merge_size)).tolist()
1253
1250
- return video_embeds.split(sizes.tolist())
1254
+ return video_embeds.split(sizes)
1255
1256
def _parse_and_validate_multimodal_inputs(self, **kwargs: object) -> dict:
1257
modalities = {}