A high-throughput and memory-efficient inference and serving engine for LLMs
Mirrored from https://github.com/tduncan-lum/Nvidia-Optimization-Stack---VLLM.git
Synced Nov 14, 05:30 UTC
[Model] Remove unnecessary CUDA sync of GLM-4.1V image and video preprocess (#24332)
Signed-off-by: Win <chatcharinsang@gmail.com> Co-authored-by: Roger Wang <hey@rogerw.io>
Chatcharin Sangbutsarakum committed
Sep 8, 2025 at 13:10 UTC
8a4660260624eb34c365da34e4962e6ed15bbc0b
1 file changed
+8
-4
vllm/model_executor/models/glm4_1v.py
+8
-4
@@ -1429,6 +1429,7 @@ class Glm4vForConditionalGeneration(nn.Module, SupportsMultiModal,
1429
self, image_input: Glm4vImageInputs) -> tuple[torch.Tensor, ...]:
1430
grid_thw = image_input["image_grid_thw"]
1431
assert grid_thw.ndim == 2
1432
+ grid_thw_list = grid_thw.tolist()
1433
1434
if image_input["type"] == "image_embeds":
1435
image_embeds = image_input["image_embeds"].type(self.visual.dtype)
@@ -1443,13 +1444,15 @@ class Glm4vForConditionalGeneration(nn.Module, SupportsMultiModal,
1444
image_embeds = self.visual(pixel_values,
1445
grid_thw=grid_thw.tolist())
1446
merge_size = self.visual.spatial_merge_size
1446
- sizes = grid_thw.prod(-1) // merge_size // merge_size
1447
- return image_embeds.split(sizes.tolist())
1447
+ sizes = (torch.tensor(grid_thw_list, dtype=torch.long).prod(-1) //
1448
+ (merge_size * merge_size)).tolist()
1449
+ return image_embeds.split(sizes)
1450
1451
def _process_video_input(
1452
self, video_input: Glm4vVideoInputs) -> tuple[torch.Tensor, ...]:
1453
grid_thw = video_input["video_grid_thw"]
1454
assert grid_thw.ndim == 2
1455
+ grid_thw_list = grid_thw.tolist()
1456
1457
if video_input["type"] == "video_embeds":
1458
video_embeds = video_input["video_embeds"].type(self.visual.dtype)
@@ -1466,8 +1469,9 @@ class Glm4vForConditionalGeneration(nn.Module, SupportsMultiModal,
1469
grid_thw=grid_thw.tolist())
1470
# Split concatenated embeddings for each video item.
1471
merge_size = self.visual.spatial_merge_size
1469
- sizes = grid_thw.prod(-1) // merge_size // merge_size
1470
- return video_embeds.split(sizes.tolist())
1472
+ sizes = (torch.tensor(grid_thw_list, dtype=torch.long).prod(-1) //
1473
+ (merge_size * merge_size)).tolist()
1474
+ return video_embeds.split(sizes)
1475
1476
def _parse_and_validate_multimodal_inputs(self, **kwargs: object) -> dict:
1477
mm_input_by_modality = {}