A high-throughput and memory-efficient inference and serving engine for LLMs
Mirrored from https://github.com/tduncan-lum/Nvidia-Optimization-Stack---VLLM.git
Synced Nov 9, 02:30 UTC
[Model] Use mm_position to compute mrope positions for GLM-4.xV (#33039)
Signed-off-by: Yang <lymailforjob@gmail.com>
Yang Liu committed
Feb 2, 2026 at 08:55 UTC
199e3cb4762cf03a2a141a8e5ea9c2a3eea02557
3 files changed
+130
-252
examples/offline_inference/vision_language_multi_image.py
+37
@@ -1283,6 +1283,42 @@ def load_tarsier2(question: str, image_urls: list[str]) -> ModelRequestData:
1283
)
1284
1285
1286
+# GLM-4.1V
1287
+def load_glm4_1v(question: str, image_urls: list[str]) -> ModelRequestData:
1288
+ model_name = "zai-org/GLM-4.1V-9B-Thinking"
1289
+
1290
+ engine_args = EngineArgs(
1291
+ model=model_name,
1292
+ max_model_len=45082,
1293
+ max_num_seqs=2,
1294
+ limit_mm_per_prompt={"image": len(image_urls)},
1295
+ enforce_eager=True,
1296
+ )
1297
+
1298
+ placeholders = [{"type": "image", "image": url} for url in image_urls]
1299
+ messages = [
1300
+ {
1301
+ "role": "user",
1302
+ "content": [
1303
+ *placeholders,
1304
+ {"type": "text", "text": question},
1305
+ ],
1306
+ }
1307
+ ]
1308
+
1309
+ processor = AutoProcessor.from_pretrained(model_name)
1310
+ prompt = processor.apply_chat_template(
1311
+ messages, tokenize=False, add_generation_prompt=True
1312
+ )
1313
+ image_data = [fetch_image(url) for url in image_urls]
1314
+
1315
+ return ModelRequestData(
1316
+ engine_args=engine_args,
1317
+ prompt=prompt,
1318
+ image_data=image_data,
1319
+ )
1320
+
1321
+
1322
# GLM-4.5V
1323
def load_glm4_5v(question: str, image_urls: list[str]) -> ModelRequestData:
1324
model_name = "zai-org/GLM-4.5V"
@@ -1430,6 +1466,7 @@ model_example_map = {
1466
"stepvl": load_step_vl,
1467
"tarsier": load_tarsier,
1468
"tarsier2": load_tarsier2,
1469
+ "glm4_1v": load_glm4_1v,
1470
"glm4_5v": load_glm4_5v,
1471
"glm4_5v_fp8": load_glm4_5v_fp8,
1472
}
vllm/model_executor/models/glm4_1v.py
+49
-126
@@ -27,9 +27,8 @@
27
"""Inference-only GLM-4.1V & GLM-4.6V-Flash, AutoGLM-Phone-9B model
28
compatible with HuggingFace weights."""
29
30
-import itertools
30
import math
32
-from collections.abc import Callable, Iterable, Mapping, Sequence
31
+from collections.abc import Callable, Iterable, Iterator, Mapping, Sequence
32
from functools import partial
33
from typing import Annotated, Any, Literal, TypeAlias
34
@@ -1580,138 +1579,62 @@ class Glm4vForConditionalGeneration(
1579
multimodal_embeddings += tuple(video_embeddings)
1580
return multimodal_embeddings
1581
1582
+ def iter_mm_grid_thw(
1583
+ self, mm_features: list[MultiModalFeatureSpec]
1584
+ ) -> Iterator[tuple[int, int, int, int]]:
1585
+ hf_config = self.config
1586
+ spatial_merge_size = hf_config.vision_config.spatial_merge_size
1587
+ for mm_feature in sorted(mm_features, key=lambda f: f.mm_position.offset):
1588
+ offset = mm_feature.mm_position.offset
1589
+ if mm_feature.modality == "image":
1590
+ t, h, w = mm_feature.data["image_grid_thw"].data.tolist()
1591
+ assert t == 1, f"Image must have 1 frame, got {t}"
1592
+ yield offset, t, h // spatial_merge_size, w // spatial_merge_size
1593
+ elif mm_feature.modality == "video":
1594
+ t, h, w = mm_feature.data["video_grid_thw"].data.tolist()
1595
+ yield (
1596
+ offset,
1597
+ t,
1598
+ h // spatial_merge_size,
1599
+ w // spatial_merge_size,
1600
+ )
1601
+ else:
1602
+ raise ValueError(f"Unsupported modality: {mm_feature.modality}")
1603
+
1604
def get_mrope_input_positions(
1605
self,
1606
input_tokens: list[int],
1607
mm_features: list[MultiModalFeatureSpec],
1608
) -> tuple[torch.Tensor, int]:
1588
- kwargs = MultiModalFeatureSpec.gather_kwargs(
1589
- mm_features,
1590
- {"image_grid_thw", "video_grid_thw"},
1591
- )
1592
- image_grid_thw = [item.tolist() for item in kwargs.get("image_grid_thw", [])]
1593
- video_grid_thw = [item.tolist() for item in kwargs.get("video_grid_thw", [])]
1594
-
1595
- hf_config = self.config
1596
- image_token_id = hf_config.image_token_id
1597
- video_start_token_id = hf_config.video_start_token_id
1598
- video_end_token_id = hf_config.video_end_token_id
1599
- spatial_merge_size = hf_config.vision_config.spatial_merge_size
1609
llm_pos_ids_list: list = []
1610
+ st = 0
1611
+ for (
1612
+ offset,
1613
+ llm_grid_t,
1614
+ llm_grid_h,
1615
+ llm_grid_w,
1616
+ ) in self.iter_mm_grid_thw(mm_features):
1617
+ text_len = offset - st
1618
+ st_idx = llm_pos_ids_list[-1].max() + 1 if len(llm_pos_ids_list) > 0 else 0
1619
+ llm_pos_ids_list.append(
1620
+ np.broadcast_to(np.arange(text_len), (3, text_len)) + st_idx
1621
+ )
1622
+ grid_indices = np.indices((llm_grid_t, llm_grid_h, llm_grid_w)).reshape(
1623
+ 3, -1
1624
+ )
1625
+ llm_pos_ids_list.append(grid_indices + text_len + st_idx)
1626
+ st = offset + llm_grid_t * llm_grid_h * llm_grid_w
1627
+
1628
+ if st < len(input_tokens):
1629
+ text_len = len(input_tokens) - st
1630
+ st_idx = llm_pos_ids_list[-1].max() + 1 if len(llm_pos_ids_list) > 0 else 0
1631
+ llm_pos_ids_list.append(
1632
+ np.broadcast_to(np.arange(text_len), (3, text_len)) + st_idx
1633
+ )
1634
1602
- if image_grid_thw or video_grid_thw:
1603
- input_token_type: list[str] = []
1604
- video_check_flg = False
1605
- for token in input_tokens:
1606
- if token == video_start_token_id:
1607
- video_check_flg = True
1608
- elif token == video_end_token_id:
1609
- video_check_flg = False
1610
-
1611
- if (token == image_token_id) and (video_check_flg is False):
1612
- input_token_type.append("image")
1613
- elif (token == image_token_id) and (video_check_flg is True):
1614
- input_token_type.append("video")
1615
- else:
1616
- input_token_type.append("text")
1617
-
1618
- input_type_group: list[tuple[str, int, int]] = []
1619
- for key, group_iter in itertools.groupby(
1620
- enumerate(input_token_type), lambda x: x[1]
1621
- ):
1622
- group_list = list(group_iter)
1623
- start_index = group_list[0][0]
1624
- end_index = group_list[-1][0] + 1
1625
- input_type_group.append((key, start_index, end_index))
1626
-
1627
- video_frame_num = 1
1628
- mm_data_idx = 0
1629
- for modality_type, start_idx, end_idx in input_type_group:
1630
- st_idx = (
1631
- llm_pos_ids_list[-1].max() + 1 if len(llm_pos_ids_list) > 0 else 0
1632
- )
1633
- if modality_type == "image":
1634
- t, h, w = image_grid_thw[mm_data_idx]
1635
- llm_grid_t, llm_grid_h, llm_grid_w = (
1636
- t,
1637
- h // spatial_merge_size,
1638
- w // spatial_merge_size,
1639
- )
1640
-
1641
- t_index = (
1642
- torch.arange(llm_grid_t)
1643
- .view(-1, 1)
1644
- .expand(-1, llm_grid_h * llm_grid_w)
1645
- .flatten()
1646
- )
1647
- h_index = (
1648
- torch.arange(llm_grid_h)
1649
- .view(1, -1, 1)
1650
- .expand(llm_grid_t, -1, llm_grid_w)
1651
- .flatten()
1652
- )
1653
- w_index = (
1654
- torch.arange(llm_grid_w)
1655
- .view(1, 1, -1)
1656
- .expand(llm_grid_t, llm_grid_h, -1)
1657
- .flatten()
1658
- )
1659
- llm_pos_ids_list.append(
1660
- torch.stack([t_index, h_index, w_index]) + st_idx
1661
- )
1662
- mm_data_idx += 1
1663
-
1664
- elif modality_type == "video":
1665
- t, h, w = (
1666
- video_frame_num,
1667
- *image_grid_thw[mm_data_idx][1:],
1668
- )
1669
- llm_grid_t, llm_grid_h, llm_grid_w = (
1670
- t,
1671
- h // spatial_merge_size,
1672
- w // spatial_merge_size,
1673
- )
1674
-
1675
- for t_idx in range(llm_grid_t):
1676
- t_index = (
1677
- torch.tensor(t_idx)
1678
- .view(-1, 1)
1679
- .expand(-1, llm_grid_h * llm_grid_w)
1680
- .flatten()
1681
- )
1682
- h_index = (
1683
- torch.arange(llm_grid_h)
1684
- .view(1, -1, 1)
1685
- .expand(1, -1, llm_grid_w)
1686
- .flatten()
1687
- )
1688
- w_index = (
1689
- torch.arange(llm_grid_w)
1690
- .view(1, 1, -1)
1691
- .expand(1, llm_grid_h, -1)
1692
- .flatten()
1693
- )
1694
- llm_pos_ids_list.append(
1695
- torch.stack([t_index, h_index, w_index]) + st_idx
1696
- )
1697
-
1698
- mm_data_idx += 1
1699
- video_frame_num += 1
1700
-
1701
- else:
1702
- text_len = end_idx - start_idx
1703
- llm_pos_ids_list.append(
1704
- torch.arange(text_len).view(1, -1).expand(3, -1) + st_idx
1705
- )
1706
- video_frame_num = 1
1707
-
1708
- else:
1709
- text_len = len(input_tokens)
1710
- llm_pos_ids_list.append(torch.arange(text_len).view(1, -1).expand(3, -1))
1711
-
1712
- llm_positions = torch.cat(llm_pos_ids_list, dim=1).reshape(3, -1)
1635
+ llm_positions = np.concatenate(llm_pos_ids_list, axis=1).reshape(3, -1)
1636
mrope_position_delta = (llm_positions.max() + 1 - len(input_tokens)).item()
1714
- return llm_positions, mrope_position_delta
1637
+ return torch.from_numpy(llm_positions), mrope_position_delta
1638
1639
def forward(
1640
self,
vllm/model_executor/models/glm4v.py
+44
-126
@@ -5,11 +5,11 @@
5
# https://github.com/zai-org/CogAgent
6
"""Inference-only CogAgent model compatible with THUDM weights."""
7
8
-import itertools
8
from argparse import Namespace
10
-from collections.abc import Mapping, Sequence
9
+from collections.abc import Iterator, Mapping, Sequence
10
from typing import Annotated, Literal
11
12
+import numpy as np
13
import torch
14
from torch import nn
15
from torch.nn import LayerNorm
@@ -624,138 +624,56 @@ class GLM4VForCausalLM(
624
625
return self.transformer.vision(pixel_values)
626
627
+ def iter_mm_grid_thw(
628
+ self, mm_features: list[MultiModalFeatureSpec]
629
+ ) -> Iterator[tuple[int, int, int, int]]:
630
+ hf_config = self.config
631
+ spatial_merge_size = hf_config.vision_config.spatial_merge_size
632
+ for mm_feature in sorted(mm_features, key=lambda f: f.mm_position.offset):
633
+ offset = mm_feature.mm_position.offset
634
+ if mm_feature.modality == "image":
635
+ t, h, w = mm_feature.data["image_grid_thw"].data.tolist()
636
+ assert t == 1, f"Image must have 1 frame, got {t}"
637
+ yield offset, t, h // spatial_merge_size, w // spatial_merge_size
638
+ else:
639
+ # glm4v only supports image modality
640
+ raise ValueError(f"Unsupported modality: {mm_feature.modality}")
641
+
642
def get_mrope_input_positions(
643
self,
644
input_tokens: list[int],
645
mm_features: list[MultiModalFeatureSpec],
646
) -> tuple[torch.Tensor, int]:
632
- kwargs = MultiModalFeatureSpec.gather_kwargs(
633
- mm_features,
634
- {"image_grid_thw", "video_grid_thw"},
635
- )
636
- image_grid_thw = [item.tolist() for item in kwargs.get("image_grid_thw", [])]
637
- video_grid_thw = [item.tolist() for item in kwargs.get("video_grid_thw", [])]
638
-
639
- hf_config = self.config
640
- image_token_id = hf_config.image_token_id
641
- video_start_token_id = hf_config.video_start_token_id
642
- video_end_token_id = hf_config.video_end_token_id
643
- spatial_merge_size = hf_config.vision_config.spatial_merge_size
647
llm_pos_ids_list: list = []
648
+ st = 0
649
+ for (
650
+ offset,
651
+ llm_grid_t,
652
+ llm_grid_h,
653
+ llm_grid_w,
654
+ ) in self.iter_mm_grid_thw(mm_features):
655
+ text_len = offset - st
656
+ st_idx = llm_pos_ids_list[-1].max() + 1 if len(llm_pos_ids_list) > 0 else 0
657
+ llm_pos_ids_list.append(
658
+ np.broadcast_to(np.arange(text_len), (3, text_len)) + st_idx
659
+ )
660
+ grid_indices = np.indices((llm_grid_t, llm_grid_h, llm_grid_w)).reshape(
661
+ 3, -1
662
+ )
663
+ llm_pos_ids_list.append(grid_indices + text_len + st_idx)
664
+ # EVA2CLIPModel has embeddings for boi and eoi tokens as well
665
+ st = offset + 1 + llm_grid_t * llm_grid_h * llm_grid_w + 1
666
+
667
+ if st < len(input_tokens):
668
+ text_len = len(input_tokens) - st
669
+ st_idx = llm_pos_ids_list[-1].max() + 1 if len(llm_pos_ids_list) > 0 else 0
670
+ llm_pos_ids_list.append(
671
+ np.broadcast_to(np.arange(text_len), (3, text_len)) + st_idx
672
+ )
673
646
- if image_grid_thw or video_grid_thw:
647
- input_token_type: list[str] = []
648
- video_check_flg = False
649
- for token in input_tokens:
650
- if token == video_start_token_id:
651
- video_check_flg = True
652
- elif token == video_end_token_id:
653
- video_check_flg = False
654
-
655
- if (token == image_token_id) and (video_check_flg is False):
656
- input_token_type.append("image")
657
- elif (token == image_token_id) and (video_check_flg is True):
658
- input_token_type.append("video")
659
- else:
660
- input_token_type.append("text")
661
-
662
- input_type_group: list[tuple[str, int, int]] = []
663
- for key, group_iter in itertools.groupby(
664
- enumerate(input_token_type), lambda x: x[1]
665
- ):
666
- group_list = list(group_iter)
667
- start_index = group_list[0][0]
668
- end_index = group_list[-1][0] + 1
669
- input_type_group.append((key, start_index, end_index))
670
-
671
- video_frame_num = 1
672
- mm_data_idx = 0
673
- for modality_type, start_idx, end_idx in input_type_group:
674
- st_idx = (
675
- llm_pos_ids_list[-1].max() + 1 if len(llm_pos_ids_list) > 0 else 0
676
- )
677
- if modality_type == "image":
678
- t, h, w = image_grid_thw[mm_data_idx]
679
- llm_grid_t, llm_grid_h, llm_grid_w = (
680
- t,
681
- h // spatial_merge_size,
682
- w // spatial_merge_size,
683
- )
684
-
685
- t_index = (
686
- torch.arange(llm_grid_t)
687
- .view(-1, 1)
688
- .expand(-1, llm_grid_h * llm_grid_w)
689
- .flatten()
690
- )
691
- h_index = (
692
- torch.arange(llm_grid_h)
693
- .view(1, -1, 1)
694
- .expand(llm_grid_t, -1, llm_grid_w)
695
- .flatten()
696
- )
697
- w_index = (
698
- torch.arange(llm_grid_w)
699
- .view(1, 1, -1)
700
- .expand(llm_grid_t, llm_grid_h, -1)
701
- .flatten()
702
- )
703
- llm_pos_ids_list.append(
704
- torch.stack([t_index, h_index, w_index]) + st_idx
705
- )
706
- mm_data_idx += 1
707
-
708
- elif modality_type == "video":
709
- t, h, w = (
710
- video_frame_num,
711
- *image_grid_thw[mm_data_idx][1:],
712
- )
713
- llm_grid_t, llm_grid_h, llm_grid_w = (
714
- t,
715
- h // spatial_merge_size,
716
- w // spatial_merge_size,
717
- )
718
-
719
- for t_idx in range(llm_grid_t):
720
- t_index = (
721
- torch.tensor(t_idx)
722
- .view(-1, 1)
723
- .expand(-1, llm_grid_h * llm_grid_w)
724
- .flatten()
725
- )
726
- h_index = (
727
- torch.arange(llm_grid_h)
728
- .view(1, -1, 1)
729
- .expand(1, -1, llm_grid_w)
730
- .flatten()
731
- )
732
- w_index = (
733
- torch.arange(llm_grid_w)
734
- .view(1, 1, -1)
735
- .expand(1, llm_grid_h, -1)
736
- .flatten()
737
- )
738
- llm_pos_ids_list.append(
739
- torch.stack([t_index, h_index, w_index]) + st_idx
740
- )
741
-
742
- mm_data_idx += 1
743
- video_frame_num += 1
744
-
745
- else:
746
- text_len = end_idx - start_idx
747
- llm_pos_ids_list.append(
748
- torch.arange(text_len).view(1, -1).expand(3, -1) + st_idx
749
- )
750
- video_frame_num = 1
751
-
752
- else:
753
- text_len = len(input_tokens)
754
- llm_pos_ids_list.append(torch.arange(text_len).view(1, -1).expand(3, -1))
755
-
756
- llm_positions = torch.cat(llm_pos_ids_list, dim=1).reshape(3, -1)
674
+ llm_positions = np.concatenate(llm_pos_ids_list, axis=1).reshape(3, -1)
675
mrope_position_delta = (llm_positions.max() + 1 - len(input_tokens)).item()
758
- return llm_positions, mrope_position_delta
676
+ return torch.from_numpy(llm_positions), mrope_position_delta
677
678
embed_input_ids = SupportsMultiModal.embed_input_ids
679