A high-throughput and memory-efficient inference and serving engine for LLMs
Mirrored from https://github.com/tduncan-lum/Nvidia-Optimization-Stack---VLLM.git
Synced Nov 13, 23:15 UTC
[Bugfix] Fix broken MRoPE for GLM-4.1V/GLM-4.5V (#27860)
Signed-off-by: Isotr0py <mozf@mail2.sysu.edu.cn>
Isotr0py committed
Nov 1, 2025 at 01:04 UTC
7e06c40e63c12c0ea5fb400fa8f06007e90ff84f
1 file changed
+147
-2
vllm/model_executor/models/glm4_1v.py
+147
-2
@@ -26,6 +26,7 @@
26
# limitations under the License.
27
"""Inference-only GLM-4V model compatible with HuggingFace weights."""
28
29
+import itertools
30
import math
31
from collections.abc import Callable, Iterable, Mapping, Sequence
32
from functools import partial
@@ -36,7 +37,7 @@ import torch
37
import torch.nn as nn
38
import torch.nn.functional as F
39
from einops import rearrange
39
-from transformers import BatchFeature
40
+from transformers import BatchFeature, PretrainedConfig
41
from transformers.models.glm4v.configuration_glm4v import Glm4vVisionConfig
42
from transformers.models.glm4v.image_processing_glm4v import (
43
Glm4vImageProcessor,
@@ -89,6 +90,7 @@ from ..layers.activation import SiluAndMul
90
from .interfaces import (
91
MultiModalEmbeddings,
92
SupportsLoRA,
93
+ SupportsMRoPE,
94
SupportsMultiModal,
95
SupportsPP,
96
)
@@ -1386,7 +1388,7 @@ class Glm4vMultiModalProcessor(BaseMultiModalProcessor[Glm4vProcessingInfo]):
1388
dummy_inputs=Glm4vDummyInputsBuilder,
1389
)
1390
class Glm4vForConditionalGeneration(
1389
- nn.Module, SupportsMultiModal, SupportsLoRA, SupportsPP
1391
+ nn.Module, SupportsMultiModal, SupportsLoRA, SupportsPP, SupportsMRoPE
1392
):
1393
merge_by_field_config = True
1394
@@ -1613,6 +1615,149 @@ class Glm4vForConditionalGeneration(
1615
multimodal_embeddings += tuple(video_embeddings)
1616
return multimodal_embeddings
1617
1618
+ def get_mrope_input_positions(
1619
+ self,
1620
+ input_tokens: list[int],
1621
+ hf_config: "PretrainedConfig",
1622
+ image_grid_thw: list[list[int]] | torch.Tensor | None,
1623
+ video_grid_thw: list[list[int]] | torch.Tensor | None,
1624
+ second_per_grid_ts: list[float] | None = None,
1625
+ context_len: int = 0,
1626
+ seq_len: int | None = None,
1627
+ audio_feature_lengths: torch.Tensor | None = None,
1628
+ use_audio_in_video: bool = False,
1629
+ ) -> tuple[torch.Tensor, int]:
1630
+ """Get mrope input positions and delta value for GLM4V."""
1631
+
1632
+ image_token_id = hf_config.image_token_id
1633
+ video_start_token_id = hf_config.video_start_token_id
1634
+ video_end_token_id = hf_config.video_end_token_id
1635
+ spatial_merge_size = hf_config.vision_config.spatial_merge_size
1636
+ llm_pos_ids_list: list = []
1637
+
1638
+ if not (image_grid_thw is None and video_grid_thw is None):
1639
+ if isinstance(image_grid_thw, torch.Tensor):
1640
+ image_grid_thw = image_grid_thw.tolist()
1641
+
1642
+ input_token_type: list[str] = []
1643
+ video_check_flg = False
1644
+ for token in input_tokens:
1645
+ if token == video_start_token_id:
1646
+ video_check_flg = True
1647
+ elif token == video_end_token_id:
1648
+ video_check_flg = False
1649
+
1650
+ if (token == image_token_id) and (video_check_flg is False):
1651
+ input_token_type.append("image")
1652
+ elif (token == image_token_id) and (video_check_flg is True):
1653
+ input_token_type.append("video")
1654
+ else:
1655
+ input_token_type.append("text")
1656
+
1657
+ input_type_group: list[tuple[str, int, int]] = []
1658
+ for key, group_iter in itertools.groupby(
1659
+ enumerate(input_token_type), lambda x: x[1]
1660
+ ):
1661
+ group_list = list(group_iter)
1662
+ start_index = group_list[0][0]
1663
+ end_index = group_list[-1][0] + 1
1664
+ input_type_group.append((key, start_index, end_index))
1665
+
1666
+ video_frame_num = 1
1667
+ mm_data_idx = 0
1668
+ for modality_type, start_idx, end_idx in input_type_group:
1669
+ st_idx = (
1670
+ llm_pos_ids_list[-1].max() + 1 if len(llm_pos_ids_list) > 0 else 0
1671
+ )
1672
+ if modality_type == "image":
1673
+ t, h, w = (
1674
+ image_grid_thw[mm_data_idx][0],
1675
+ image_grid_thw[mm_data_idx][1],
1676
+ image_grid_thw[mm_data_idx][2],
1677
+ )
1678
+ llm_grid_t, llm_grid_h, llm_grid_w = (
1679
+ t,
1680
+ h // spatial_merge_size,
1681
+ w // spatial_merge_size,
1682
+ )
1683
+
1684
+ t_index = (
1685
+ torch.arange(llm_grid_t)
1686
+ .view(-1, 1)
1687
+ .expand(-1, llm_grid_h * llm_grid_w)
1688
+ .flatten()
1689
+ )
1690
+ h_index = (
1691
+ torch.arange(llm_grid_h)
1692
+ .view(1, -1, 1)
1693
+ .expand(llm_grid_t, -1, llm_grid_w)
1694
+ .flatten()
1695
+ )
1696
+ w_index = (
1697
+ torch.arange(llm_grid_w)
1698
+ .view(1, 1, -1)
1699
+ .expand(llm_grid_t, llm_grid_h, -1)
1700
+ .flatten()
1701
+ )
1702
+ llm_pos_ids_list.append(
1703
+ torch.stack([t_index, h_index, w_index]) + st_idx
1704
+ )
1705
+ mm_data_idx += 1
1706
+
1707
+ elif modality_type == "video":
1708
+ t, h, w = (
1709
+ video_frame_num,
1710
+ image_grid_thw[mm_data_idx][1],
1711
+ image_grid_thw[mm_data_idx][2],
1712
+ )
1713
+ llm_grid_t, llm_grid_h, llm_grid_w = (
1714
+ t,
1715
+ h // spatial_merge_size,
1716
+ w // spatial_merge_size,
1717
+ )
1718
+
1719
+ for t_idx in range(llm_grid_t):
1720
+ t_index = (
1721
+ torch.tensor(t_idx)
1722
+ .view(-1, 1)
1723
+ .expand(-1, llm_grid_h * llm_grid_w)
1724
+ .flatten()
1725
+ )
1726
+ h_index = (
1727
+ torch.arange(llm_grid_h)
1728
+ .view(1, -1, 1)
1729
+ .expand(1, -1, llm_grid_w)
1730
+ .flatten()
1731
+ )
1732
+ w_index = (
1733
+ torch.arange(llm_grid_w)
1734
+ .view(1, 1, -1)
1735
+ .expand(1, llm_grid_h, -1)
1736
+ .flatten()
1737
+ )
1738
+ llm_pos_ids_list.append(
1739
+ torch.stack([t_index, h_index, w_index]) + st_idx
1740
+ )
1741
+
1742
+ mm_data_idx += 1
1743
+ video_frame_num += 1
1744
+
1745
+ else:
1746
+ text_len = end_idx - start_idx
1747
+ llm_pos_ids_list.append(
1748
+ torch.arange(text_len).view(1, -1).expand(3, -1) + st_idx
1749
+ )
1750
+ video_frame_num = 1
1751
+
1752
+ else:
1753
+ text_len = len(input_tokens)
1754
+ llm_pos_ids_list.append(torch.arange(text_len).view(1, -1).expand(3, -1))
1755
+
1756
+ llm_positions = torch.cat(llm_pos_ids_list, dim=1).reshape(3, -1)
1757
+ llm_positions = llm_positions[:, context_len:seq_len]
1758
+ mrope_position_delta = (llm_positions.max() + 1 - len(input_tokens)).item()
1759
+ return llm_positions, mrope_position_delta
1760
+
1761
def forward(
1762
self,
1763
input_ids: torch.Tensor,