A high-throughput and memory-efficient inference and serving engine for LLMs
Mirrored from https://github.com/tduncan-lum/Nvidia-Optimization-Stack---VLLM.git
Synced Nov 14, 05:30 UTC
Remove unnecessary CUDA sync of qwen image and video preprocess (#22792)
Signed-off-by: cyy <cyyever@outlook.com> Signed-off-by: Yuanyuan Chen <cyyever@outlook.com> Co-authored-by: Cyrus Leung <cyrus.tl.leung@gmail.com>
Yuanyuan Chen committed
Aug 13, 2025 at 21:07 UTC
6772bb0f7d58d137576f386dd921117a5a00f0fb
1 file changed
+8
-4
vllm/model_executor/models/qwen2_5_vl.py
+8
-4
@@ -976,10 +976,12 @@ class Qwen2_5_VLForConditionalGeneration(nn.Module, SupportsMultiModal,
976
image_embeds = self.visual(pixel_values, grid_thw=grid_thw_list)
977
978
# Split concatenated embeddings for each image item.
979
+ # Using prod on grid_thw_list instead of grid_thw.prod avoids CUDA sync
980
merge_size = self.visual.spatial_merge_size
980
- sizes = grid_thw.prod(-1) // merge_size // merge_size
981
+ sizes = (torch.tensor(grid_thw_list, dtype=torch.long).prod(-1) //
982
+ (merge_size * merge_size)).tolist()
983
982
- return image_embeds.split(sizes.tolist())
984
+ return image_embeds.split(sizes)
985
986
def _process_video_input(
987
self,
@@ -998,9 +1000,11 @@ class Qwen2_5_VLForConditionalGeneration(nn.Module, SupportsMultiModal,
1000
1001
# Split concatenated embeddings for each video item.
1002
merge_size = self.visual.spatial_merge_size
1001
- sizes = grid_thw.prod(-1) // merge_size // merge_size
1003
+ # Using prod on grid_thw_list instead of grid_thw.prod avoids CUDA sync
1004
+ sizes = (torch.tensor(grid_thw_list, dtype=torch.long).prod(-1) //
1005
+ (merge_size * merge_size)).tolist()
1006
1003
- return video_embeds.split(sizes.tolist())
1007
+ return video_embeds.split(sizes)
1008
1009
def _parse_and_validate_multimodal_inputs(self, **kwargs: object) -> dict:
1010
mm_input_by_modality = {}