Fix GLM-4.1V video placeholder token ID handling. (#49484)
Signed-off-by: aarushjain29 <Aarushi.Jain2@amd.com> Co-authored-by: Andreas Karatzas <akaratza@amd.com>
Aarushi Jain committed
Jul 24, 2026 at 14:35 UTC
c064fa52b6425b17dbc51fea2eaf4aac0f30863f
2 files changed
+24
-43
tests/models/multimodal/processing/test_glm4_1v.py
+2
-3
@@ -93,7 +93,6 @@ def test_processor_override(
93
limit_mm_per_prompt={"video": 1},
94
)
95
processor = MULTIMODAL_REGISTRY.create_processor(ctx.model_config)
96
- tokenizer = processor.info.get_tokenizer()
96
hf_processor_mm_kwargs = {"fps": fps}
97
98
# Build the image str / prompt based on the number of images we pass
@@ -112,8 +111,8 @@ def test_processor_override(
111
112
# Ensure we have the right number of placeholders per num_crops size
113
hf_processor = processor.info.get_hf_processor(**hf_processor_mm_kwargs)
115
- video_token_id = tokenizer.convert_tokens_to_ids(hf_processor.video_token)
116
- video_tok_count = processed_inputs["prompt_token_ids"].count(video_token_id)
114
+ image_token_id = hf_processor.image_token_id
115
+ video_tok_count = processed_inputs["prompt_token_ids"].count(image_token_id)
116
grid_t, _, _ = processed_inputs["mm_kwargs"].get_data()["video_grid_thw"][0]
117
118
assert grid_t == expected_grid_t
vllm/model_executor/models/glm4_1v.py
+22
-40
@@ -1403,6 +1403,11 @@ class Glm4vProcessingInfo(BaseProcessingInfo):
1403
timestamps_list = full_second_idxs[::2]
1404
return list(timestamps_list)
1405
1406
+ def _get_video_frame_embed_token_id(self, hf_processor: object) -> int:
1407
+ if isinstance(hf_processor, Glm4vProcessor) or TRANSFORMERS_WITH_GA:
1408
+ return hf_processor.image_token_id
1409
+ return hf_processor.video_token_id
1410
+
1411
def _construct_video_placeholder(
1412
self,
1413
video_array: np.ndarray,
@@ -1440,13 +1445,7 @@ class Glm4vProcessingInfo(BaseProcessingInfo):
1445
num_tokens_per_frame = int(H * W) // merge_length
1446
placeholder = []
1447
placeholder.append(bov_token_id)
1443
- # Glm46VProcessor uses image_token_id for video frame embeddings;
1444
- # Glm4vProcessor uses video_token_id.
1445
- frame_embed_token_id = (
1446
- hf_processor.video_token_id
1447
- if isinstance(hf_processor, Glm4vProcessor) or not TRANSFORMERS_WITH_GA
1448
- else hf_processor.image_token_id
1449
- )
1448
+ frame_embed_token_id = self._get_video_frame_embed_token_id(hf_processor)
1449
for frame_idx in frames_idx_token:
1450
placeholder.append(boi_token_id)
1451
placeholder.extend([frame_embed_token_id] * num_tokens_per_frame)
@@ -1604,11 +1603,6 @@ class Glm4vMultiModalProcessor(BaseMultiModalProcessor[Glm4vProcessingInfo]):
1603
1604
processor = self.info.get_hf_processor(**mm_kwargs)
1605
1607
- # Glm46VProcessor and GLMGA handle image/video placeholders together
1608
- # via the direct path. Only Glm4vProcessor (GLM-4.1V) needs the
1609
- # split-video path because it uses image_token_id as the video
1610
- # placeholder. The direct path requires transformers >= 5.5.0
1611
- # (Glm46VProcessor / GlmgaVideoProcessor support).
1606
use_direct_path = (
1607
not isinstance(processor, Glm4vProcessor) and TRANSFORMERS_WITH_GA
1608
)
@@ -1630,6 +1624,8 @@ class Glm4vMultiModalProcessor(BaseMultiModalProcessor[Glm4vProcessingInfo]):
1624
):
1625
video_grid_thw_lst = []
1626
pixel_values_videos_lst = []
1627
+ frame_embed_token_id = self.info._get_video_frame_embed_token_id(processor)
1628
+ swap_video_frame_tokens = frame_embed_token_id == processor.image_token_id
1629
for item in mm_data.pop("videos", []):
1630
video_array, metadata = item
1631
@@ -1650,9 +1646,10 @@ class Glm4vMultiModalProcessor(BaseMultiModalProcessor[Glm4vProcessingInfo]):
1646
tok_kwargs=tok_kwargs,
1647
)
1648
input_ids = video_outputs.pop("input_ids")
1653
- input_ids[input_ids == processor.image_token_id] = (
1654
- processor.video_token_id
1655
- )
1649
+ if swap_video_frame_tokens:
1650
+ input_ids[input_ids == processor.image_token_id] = (
1651
+ processor.video_token_id
1652
+ )
1653
video_placeholder = processor.tokenizer.batch_decode(input_ids)[0]
1654
prompt = prompt.replace(
1655
"<|begin_of_video|><|video|><|end_of_video|>",
@@ -1668,6 +1665,7 @@ class Glm4vMultiModalProcessor(BaseMultiModalProcessor[Glm4vProcessingInfo]):
1665
)
1666
else:
1667
video_outputs = dict()
1668
+ swap_video_frame_tokens = False
1669
1670
processed_outputs = super()._call_hf_processor(
1671
prompt=prompt,
@@ -1675,6 +1673,10 @@ class Glm4vMultiModalProcessor(BaseMultiModalProcessor[Glm4vProcessingInfo]):
1673
mm_kwargs=mm_kwargs,
1674
tok_kwargs=tok_kwargs,
1675
)
1676
+ if swap_video_frame_tokens:
1677
+ input_ids = processed_outputs["input_ids"]
1678
+ input_ids[input_ids == processor.video_token_id] = processor.image_token_id
1679
+ processed_outputs["input_ids"] = input_ids
1680
combined_outputs = dict(
1681
processed_outputs,
1682
**video_outputs,
@@ -1701,7 +1703,7 @@ class Glm4vMultiModalProcessor(BaseMultiModalProcessor[Glm4vProcessingInfo]):
1703
1704
merge_length = image_processor.merge_size**2
1705
1704
- def get_image_replacement_glm4v(item_idx: int):
1706
+ def get_image_replacement(item_idx: int):
1707
out_item = out_mm_kwargs["image"][item_idx]
1708
grid_thw = out_item["image_grid_thw"].data
1709
assert isinstance(grid_thw, torch.Tensor)
@@ -1709,21 +1711,7 @@ class Glm4vMultiModalProcessor(BaseMultiModalProcessor[Glm4vProcessingInfo]):
1711
num_tokens = int(grid_thw.prod()) // merge_length
1712
return [hf_processor.image_token_id] * num_tokens
1713
1712
- def get_video_replacement_glm4v(item_idx: int):
1713
- out_item = out_mm_kwargs["video"][item_idx]
1714
- grid_thw = out_item["video_grid_thw"].data
1715
- assert isinstance(grid_thw, torch.Tensor)
1716
-
1717
- video, metadata = mm_items["video"][item_idx]
1718
- placeholder = self.info._construct_video_placeholder(
1719
- video, metadata, grid_thw
1720
- )
1721
- return PromptUpdateDetails.select_token_id(
1722
- placeholder,
1723
- embed_token_id=hf_processor.video_token_id,
1724
- )
1725
-
1726
- def get_video_replacement_glm46v(item_idx: int):
1714
+ def get_video_replacement(item_idx: int):
1715
out_item = out_mm_kwargs["video"][item_idx]
1716
grid_thw = out_item["video_grid_thw"].data
1717
assert isinstance(grid_thw, torch.Tensor)
@@ -1734,25 +1722,19 @@ class Glm4vMultiModalProcessor(BaseMultiModalProcessor[Glm4vProcessingInfo]):
1722
)
1723
return PromptUpdateDetails.select_token_id(
1724
placeholder,
1737
- embed_token_id=hf_processor.image_token_id,
1725
+ embed_token_id=self.info._get_video_frame_embed_token_id(hf_processor),
1726
)
1727
1740
- is_glm46v = not isinstance(hf_processor, Glm4vProcessor)
1741
-
1728
return [
1729
PromptReplacement(
1730
modality="image",
1731
target=hf_processor.image_token,
1746
- replacement=get_image_replacement_glm4v,
1732
+ replacement=get_image_replacement,
1733
),
1734
PromptReplacement(
1735
modality="video",
1736
target="<|begin_of_video|><|video|><|end_of_video|>",
1751
- replacement=(
1752
- get_video_replacement_glm46v
1753
- if is_glm46v and TRANSFORMERS_WITH_GA
1754
- else get_video_replacement_glm4v
1755
- ),
1737
+ replacement=get_video_replacement,
1738
),
1739
]
1740