@tduncan / Nvidia-Optimization-Stack-VLLM Mirror · read-only / commits / ed70f3c64f

Fork 0

A high-throughput and memory-efficient inference and serving engine for LLMs

Mirrored from https://github.com/tduncan-lum/Nvidia-Optimization-Stack---VLLM.git

Synced Nov 15, 03:30 UTC

Add GLM4.1V model (Draft) (#19331)

Signed-off-by: zRzRzRzRzRzRzR <2448370773@qq.com> Signed-off-by: Isotr0py <mozf@mail2.sysu.edu.cn> Co-authored-by: Isotr0py <mozf@mail2.sysu.edu.cn>

Yuxuan Zhang committed Jul 1, 2025 at 20:48 UTC ed70f3c64f684750edea087e286cbf264e7cc3f3

17 files changed +1946 -16

docs/models/supported_models.md

+2 -1

@@ -553,6 +553,7 @@ Specified using `--task generate`.

553 | `FuyuForCausalLM` | Fuyu | T + I | `adept/fuyu-8b` etc. | | ✅︎ | ✅︎ |

554 | `Gemma3ForConditionalGeneration` | Gemma 3 | T + I+ | `google/gemma-3-4b-it`, `google/gemma-3-27b-it`, etc. | ✅︎ | ✅︎ | ⚠️ |

555 | `GLM4VForCausalLM`^ | GLM-4V | T + I | `THUDM/glm-4v-9b`, `THUDM/cogagent-9b-20241220` etc. | ✅︎ | ✅︎ | ✅︎ |

556 +| `Glm4vForConditionalGeneration` | GLM-4.1V-Thinking | T + IE+ + VE+ | `THUDM/GLM-4.1V-9B-Thinkg`, etc. | ✅︎ | ✅︎ | ✅︎ |

557 | `GraniteSpeechForConditionalGeneration` | Granite Speech | T + A | `ibm-granite/granite-speech-3.3-8b` | ✅︎ | ✅︎ | ✅︎ |

558 | `H2OVLChatModel` | H2OVL | T + IE+ | `h2oai/h2ovl-mississippi-800m`, `h2oai/h2ovl-mississippi-2b`, etc. | | ✅︎ | ✅︎\* |

559 | `Idefics3ForConditionalGeneration` | Idefics3 | T + I | `HuggingFaceM4/Idefics3-8B-Llama3` etc. | ✅︎ | | ✅︎ |

@@ -583,7 +584,7 @@ Specified using `--task generate`.

584 | `SkyworkR1VChatModel` | Skywork-R1V-38B | T + I | `Skywork/Skywork-R1V-38B` | | ✅︎ | ✅︎ |

585 | `SmolVLMForConditionalGeneration` | SmolVLM2 | T + I | `SmolVLM2-2.2B-Instruct` | ✅︎ | | ✅︎ |

586 | `TarsierForConditionalGeneration` | Tarsier | T + IE+ | `omni-search/Tarsier-7b`,`omni-search/Tarsier-34b` | | ✅︎ | ✅︎ |

586 -| `Tarsier2ForConditionalGeneration`^ | Tarsier2 | T + IE+ + VE+ | `omni-research/Tarsier2-Recap-7b`,`omni-research/Tarsier2-7b-0115` | | ✅︎ | ✅︎ |

587 +| `Tarsier2ForConditionalGeneration`^ | Tarsier2 | T + IE+ + VE+ | `omni-research/Tarsier2-Recap-7b`,`omni-research/Tarsier2-7b-0115` | | ✅︎ | ✅︎ |

588

589 ^ You need to set the architecture name via `--hf-overrides` to match the one in vLLM.

590     • For example, to use DeepSeek-VL2 series models:

examples/offline_inference/vision_language.py

+39 -1

@@ -248,6 +248,42 @@ def run_glm4v(questions: list[str], modality: str) -> ModelRequestData:

248 )

249

250

251 +# GLM-4.1V

252 +def run_glm4_1v(questions: list[str], modality: str) -> ModelRequestData:

253 + model_name = "THUDM/GLM-4.1V-9B-Thinking"

254 +

255 + engine_args = EngineArgs(

256 + model=model_name,

257 + max_model_len=4096,

258 + max_num_seqs=2,

259 + mm_processor_kwargs={

260 + "size": {"shortest_edge": 12544, "longest_edge": 47040000},

261 + "fps": 1,

262 + },

263 + limit_mm_per_prompt={modality: 1},

264 + enforce_eager=True,

265 + )

266 +

267 + if modality == "image":

268 + placeholder = "<|begin_of_image|><|image|><|end_of_image|>"

269 + elif modality == "video":

270 + placeholder = "<|begin_of_video|><|video|><|end_of_video|>"

271 +

272 + prompts = [

273 + (

274 + "[gMASK]<sop><|system|>\nYou are a helpful assistant.<|user|>\n"

275 + f"{placeholder}"

276 + f"{question}<|assistant|>assistant\n"

277 + )

278 + for question in questions

279 + ]

280 +

281 + return ModelRequestData(

282 + engine_args=engine_args,

283 + prompts=prompts,

284 + )

285 +

286 +

287 # H2OVL-Mississippi

288 def run_h2ovl(questions: list[str], modality: str) -> ModelRequestData:

289 assert modality == "image"

@@ -1114,6 +1150,7 @@ model_example_map = {

1150 "fuyu": run_fuyu,

1151 "gemma3": run_gemma3,

1152 "glm4v": run_glm4v,

1153 + "glm4_1v": run_glm4_1v,

1154 "h2ovl_chat": run_h2ovl,

1155 "idefics3": run_idefics3,

1156 "internvl_chat": run_internvl,

@@ -1172,10 +1209,11 @@ def get_multi_modal_input(args):

1209 if args.modality == "video":

1210 # Input video and question

1211 video = VideoAsset(name="baby_reading", num_frames=args.num_frames).np_ndarrays

1212 + metadata = VideoAsset(name="baby_reading", num_frames=args.num_frames).metadata

1213 vid_questions = ["Why is this video funny?"]

1214

1215 return {

1178 - "data": video,

1216 + "data": [(video, metadata)] if args.model_type == "glm4_1v" else video,

1217 "questions": vid_questions,

1218 }

1219

tests/entrypoints/openai/test_video.py

+1 -1

@@ -50,7 +50,7 @@ async def client(server):

50 @pytest.fixture(scope="session")

51 def base64_encoded_video() -> dict[str, str]:

52 return {

53 - video_url: encode_video_base64(fetch_video(video_url))

53 + video_url: encode_video_base64(fetch_video(video_url)[0])

54 for video_url in TEST_VIDEO_URLS

55 }

tests/models/multimodal/generation/test_common.py

+28

@@ -309,6 +309,34 @@ VLM_TEST_SETTINGS = {

309 num_logprobs=10,

310 marks=[large_gpu_mark(min_gb=32)],

311 ),

312 + "glm4_1v": VLMTestInfo(

313 + models=["THUDM/GLM-4.1V-9B-Thinking"],

314 + test_type=(VLMTestType.IMAGE, VLMTestType.MULTI_IMAGE),

315 + prompt_formatter=lambda img_prompt: f"<|user|>\n{img_prompt}<|assistant|>", # noqa: E501

318 + max_model_len=2048,

319 + max_num_seqs=2,

320 + get_stop_token_ids=lambda tok: [151329, 151336, 151338],

321 + num_logprobs=10,

322 + image_size_factors=[(), (0.25,), (0.25, 0.25, 0.25), (0.25, 0.2, 0.15)],

323 + auto_cls=AutoModelForImageTextToText,

324 + ),

325 + "glm4_1v-video": VLMTestInfo(

326 + models=["THUDM/GLM-4.1V-9B-Thinking"],

327 + # GLM4.1V require include video metadata for input

328 + test_type=VLMTestType.CUSTOM_INPUTS,

329 + max_model_len=4096,

330 + max_num_seqs=2,

331 + auto_cls=AutoModelForImageTextToText,

332 + patch_hf_runner=model_utils.glm4_1v_patch_hf_runner,

333 + custom_test_opts=[CustomTestOptions(

334 + inputs=custom_inputs.video_with_metadata_glm4_1v(),

335 + limit_mm_per_prompt={"video": 1},

336 + )],

337 + # This is needed to run on machine with 24GB VRAM

338 + vllm_runner_kwargs={"gpu_memory_utilization": 0.95},

339 + ),

340 "h2ovl": VLMTestInfo(

341 models = [

342 "h2oai/h2ovl-mississippi-800m",

tests/models/multimodal/generation/vlm_utils/custom_inputs.py

+20

@@ -129,3 +129,23 @@ def windows_attention_image_qwen2_5_vl():

129

130 wrapped_sf = ImageSizeWrapper(type=SizeType.SIZE_FACTOR, data=[0.5])

131 return build_single_image_inputs([image], [prompt], wrapped_sf)

132 +

133 +

134 +def video_with_metadata_glm4_1v():

135 + video_array = VIDEO_ASSETS[0].np_ndarrays

136 + metadata = VIDEO_ASSETS[0].metadata

137 + question = "Describe the video."

138 + video_prompt = "<|begin_of_video|><|video|><|end_of_video|>"

139 + formatted_prompt = f"<|user|>\n{video_prompt}{question}<|assistant|>\n"

140 +

141 + scales = [0.1, 0.2, 0.25]

142 + video_input = [[(rescale_video_size(video_array, scale), metadata)]

143 + for scale in scales]

144 + prompts = [formatted_prompt] * len(video_input)

145 +

146 + return [

147 + PromptWithMultiModalInput(

148 + prompts=prompts,

149 + video_data=video_input,

150 + )

151 + ]

tests/models/multimodal/generation/vlm_utils/model_utils.py

+24

@@ -16,9 +16,11 @@ import torch

16 from PIL.Image import Image

17 from transformers import (AutoConfig, AutoTokenizer, BatchFeature,

18 GenerationConfig, GenerationMixin)

19 +from transformers.video_utils import VideoMetadata

21 from vllm.sequence import SampleLogprobs

22 from vllm.transformers_utils.tokenizer import patch_padding_side

23 +from vllm.utils import is_list_of

25 from .....conftest import HfRunner, ImageAsset, ImageTestAssets

26 from .types import RunnerOutput

@@ -373,6 +375,28 @@ def glm4v_patch_hf_runner(hf_model: HfRunner) -> HfRunner:

375 return hf_model

376

377

378 +def glm4_1v_patch_hf_runner(hf_model: HfRunner) -> HfRunner:

379 + """Patches and returns an instance of the HfRunner to use for GLM4.1V."""

380 + hf_processor = hf_model.processor

381 +

382 + def processor(*args, videos=None, **kwargs):

383 + if videos is not None and is_list_of(videos, tuple):

384 + # If videos is a list of tuples, we assume each tuple contains

385 + # (video_array, metadata) as in the case of GLM4.1V.

386 + video_metadata = [[VideoMetadata(**video[1])] for video in videos]

387 + videos = [[video[0]] for video in videos]

388 + else:

389 + video_metadata = None

390 +

391 + return hf_processor(*args,

392 + videos=videos,

393 + video_metadata=video_metadata,

394 + **kwargs)

395 +

396 + hf_model.processor = processor

397 + return hf_model

398 +

399 +

400 def h2ovl_patch_hf_runner(hf_model: HfRunner) -> HfRunner:

401 """Patches and returns an instance of the HfRunner to use for H2OVL."""

402

tests/models/multimodal/processing/test_common.py

+24

@@ -24,6 +24,22 @@ from ....multimodal.utils import random_audio, random_image, random_video

24 from ...registry import HF_EXAMPLE_MODELS

27 +def glm4_1v_patch_mm_data(mm_data: MultiModalDataDict) -> MultiModalDataDict:

28 + """

29 + Patch the multimodal data for GLM4.1V model.

30 + """

31 + # Ensure video metadata is included

32 + if "video" in mm_data:

33 + video = mm_data["video"]

34 + mm_data["video"] = (video, {

35 + "total_num_frames": len(video),

36 + "fps": len(video),

37 + "duration": 1,

38 + "video_backend": "opencv"

39 + })

40 + return mm_data

41 +

42 +

43 def _test_processing_correctness(

44 model_id: str,

45 hit_rate: float,

@@ -154,6 +170,11 @@ _IGNORE_MM_KEYS = {

170 "ultravox": {"audio_features"},

171 }

172

173 +MM_DATA_PATCHES = {

174 + # GLM4.1V requires video metadata to be included in the input

175 + "glm4v": glm4_1v_patch_mm_data,

176 +}

177 +

178

179 def _test_processing_correctness_one(

180 model_config: ModelConfig,

@@ -166,6 +187,8 @@ def _test_processing_correctness_one(

187 ):

188 model_type = model_config.hf_config.model_type

189 ignore_mm_keys = _IGNORE_MM_KEYS.get(model_type, set[str]())

190 + if model_type in MM_DATA_PATCHES:

191 + mm_data = MM_DATA_PATCHES[model_type](mm_data)

192

193 if isinstance(prompt, str):

194 text_prompt = prompt

@@ -245,6 +268,7 @@ def _test_processing_correctness_one(

268 "adept/fuyu-8b",

269 "google/gemma-3-4b-it",

270 "THUDM/glm-4v-9b",

271 + "THUDM/GLM-4.1V-9B-Thinking",

272 "ibm-granite/granite-speech-3.3-2b",

273 "h2oai/h2ovl-mississippi-800m",

274 "OpenGVLab/InternVL2-1B",

tests/models/registry.py

@@ -338,6 +338,7 @@ _MULTIMODAL_EXAMPLE_MODELS = {

338 "GLM4VForCausalLM": _HfExamplesInfo("THUDM/glm-4v-9b",

339 trust_remote_code=True,

340 hf_overrides={"architectures": ["GLM4VForCausalLM"]}), # noqa: E501

341 + "Glm4vForConditionalGeneration": _HfExamplesInfo("THUDM/GLM-4.1V-9B-Thinking", min_transformers_version="4.53"), # noqa: E501

342 "H2OVLChatModel": _HfExamplesInfo("h2oai/h2ovl-mississippi-800m",

343 extras={"2b": "h2oai/h2ovl-mississippi-2b"}, # noqa: E501

344 max_transformers_version="4.48", # noqa: E501

tests/multimodal/test_utils.py

+3 -1

@@ -172,7 +172,9 @@ async def test_fetch_video_http(video_url: str, num_frames: int):

172 video_sync = connector.fetch_video(video_url, num_frames=num_frames)

173 video_async = await connector.fetch_video_async(video_url,

174 num_frames=num_frames)

175 - assert np.array_equal(video_sync, video_async)

175 + # Check that the video frames are equal and metadata are same

176 + assert np.array_equal(video_sync[0], video_async[0])

177 + assert video_sync[1] == video_async[1]

178

179

180 # Used for the next two tests related to `merge_and_sort_multimodal_metadata`.

vllm/assets/video.py

+25 -1

@@ -3,7 +3,7 @@

4 from dataclasses import dataclass

5 from functools import lru_cache

6 -from typing import ClassVar, Literal, Optional

6 +from typing import Any, ClassVar, Literal, Optional

8 import cv2

9 import numpy as np

@@ -77,6 +77,24 @@ def video_to_pil_images_list(path: str,

77 ]

80 +def video_get_metadata(path: str) -> dict[str, Any]:

81 + cap = cv2.VideoCapture(path)

82 + if not cap.isOpened():

83 + raise ValueError(f"Could not open video file {path}")

84 +

85 + total_frames = int(cap.get(cv2.CAP_PROP_FRAME_COUNT))

86 + fps = cap.get(cv2.CAP_PROP_FPS)

87 + duration = total_frames / fps if fps > 0 else 0

88 +

89 + metadata = {

90 + "total_num_frames": total_frames,

91 + "fps": fps,

92 + "duration": duration,

93 + "video_backend": "opencv"

94 + }

95 + return metadata

96 +

97 +

98 VideoAssetName = Literal["baby_reading"]

100

@@ -105,6 +123,12 @@ class VideoAsset:

123 ret = video_to_ndarrays(video_path, self.num_frames)

124 return ret

125

126 + @property

127 + def metadata(self) -> dict[str, Any]:

128 + video_path = download_video_asset(self.filename)

129 + ret = video_get_metadata(video_path)

130 + return ret

131 +

132 def get_audio(self, sampling_rate: Optional[float] = None) -> npt.NDArray:

133 """

134 Read audio data from the video asset, used in Qwen2.5-Omni examples.

vllm/entrypoints/chat_utils.py

@@ -515,6 +515,8 @@ class BaseMultiModalItemTracker(ABC, Generic[_T]):

515 if modality in ("image", "image_embeds"):

516 if model_type == "chatglm":

517 return "<|begin_of_image|><|endoftext|><|end_of_image|>"

518 + if model_type == "glm4v":

519 + return "<|begin_of_image|><|image|><|end_of_image|>"

520 if model_type in ("phi3_v", "phi4mm"):

521 return f"<|image_{current_count}|>"

522 if model_type in ("minicpmo", "minicpmv"):

@@ -563,6 +565,8 @@ class BaseMultiModalItemTracker(ABC, Generic[_T]):

565 elif modality == "video":

566 if model_type == "internvl_chat":

567 return "<video>"

568 + if model_type == "glm4v":

569 + return "<|begin_of_video|><|video|><|end_of_video|>"

570 if model_type in ("qwen2_vl", "qwen2_5_vl"):

571 return "<|vision_start|><|video_pad|><|vision_end|>"

572 if model_type == "qwen2_5_omni":

vllm/model_executor/layers/rotary_embedding.py

+119

@@ -23,6 +23,7 @@

23 # See the License for the specific language governing permissions and

24 # limitations under the License.

25 """Rotary Positional Embeddings."""

26 +import itertools

27 import math

28 from typing import Any, Optional, Union

@@ -1118,6 +1119,15 @@ class MRotaryEmbedding(RotaryEmbedding):

1119 audio_feature_lengths=audio_feature_lengths,

1120 use_audio_in_video=use_audio_in_video,

1121 )

1122 + elif "glm4v" in hf_config.model_type:

1123 + return cls._glm4v_get_input_positions_tensor(

1124 + input_tokens=input_tokens,

1125 + hf_config=hf_config,

1126 + image_grid_thw=image_grid_thw,

1127 + video_grid_thw=video_grid_thw,

1128 + context_len=context_len,

1129 + seq_len=seq_len,

1130 + )

1131 else:

1132 return cls._vl_get_input_positions_tensor(

1133 input_tokens=input_tokens,

@@ -1129,6 +1139,115 @@ class MRotaryEmbedding(RotaryEmbedding):

1139 seq_len=seq_len,

1140 )

1141

1142 + @classmethod

1143 + def _glm4v_get_input_positions_tensor(

1144 + cls,

1145 + input_tokens: list[int],

1146 + hf_config: PretrainedConfig,

1147 + image_grid_thw: Union[list[list[int]], torch.Tensor],

1148 + video_grid_thw: Union[list[list[int]], torch.Tensor],

1149 + context_len: int = 0,

1150 + seq_len: Optional[int] = None,

1151 + ) -> tuple[torch.Tensor, int]:

1152 + """Get mrope input positions and delta value for GLM4V."""

1153 +

1154 + image_token_id = hf_config.image_token_id

1155 + video_start_token_id = hf_config.video_start_token_id

1156 + video_end_token_id = hf_config.video_end_token_id

1157 + spatial_merge_size = hf_config.vision_config.spatial_merge_size

1158 + llm_pos_ids_list: list = []

1159 +

1160 + if not (image_grid_thw is None and video_grid_thw is None):

1161 + if isinstance(image_grid_thw, torch.Tensor):

1162 + image_grid_thw = image_grid_thw.tolist()

1163 +

1164 + input_token_type: list[str] = []

1165 + video_check_flg = False

1166 + for token in input_tokens:

1167 + if token == video_start_token_id:

1168 + video_check_flg = True

1169 + elif token == video_end_token_id:

1170 + video_check_flg = False

1171 +

1172 + if (token == image_token_id) and (video_check_flg is False):

1173 + input_token_type.append("image")

1174 + elif (token == image_token_id) and (video_check_flg is True):

1175 + input_token_type.append("video")

1176 + else:

1177 + input_token_type.append("text")

1178 +

1179 + input_type_group: list[tuple[str, int, int]] = []

1180 + for key, group_iter in itertools.groupby(

1181 + enumerate(input_token_type), lambda x: x[1]):

1182 + group_list = list(group_iter)

1183 + start_index = group_list[0][0]

1184 + end_index = group_list[-1][0] + 1

1185 + input_type_group.append((key, start_index, end_index))

1186 +

1187 + video_frame_num = 1

1188 + mm_data_idx = 0

1189 + for modality_type, start_idx, end_idx in input_type_group:

1190 + st_idx = llm_pos_ids_list[-1].max() + 1 if len(

1191 + llm_pos_ids_list) > 0 else 0

1192 + if modality_type == "image":

1193 + t, h, w = (

1194 + image_grid_thw[mm_data_idx][0],

1195 + image_grid_thw[mm_data_idx][1],

1196 + image_grid_thw[mm_data_idx][2],

1197 + )

1198 + llm_grid_t, llm_grid_h, llm_grid_w = \

1199 + t, h // spatial_merge_size, w // spatial_merge_size

1200 +

1201 + t_index = torch.arange(llm_grid_t).view(-1, 1).expand(

1202 + -1, llm_grid_h * llm_grid_w).flatten()

1203 + h_index = torch.arange(llm_grid_h).view(1, -1, 1).expand(

1204 + llm_grid_t, -1, llm_grid_w).flatten()

1205 + w_index = torch.arange(llm_grid_w).view(1, 1, -1).expand(

1206 + llm_grid_t, llm_grid_h, -1).flatten()

1207 + llm_pos_ids_list.append(

1208 + torch.stack([t_index, h_index, w_index]) + st_idx)

1209 + mm_data_idx += 1

1210 +

1211 + elif modality_type == "video":

1212 + t, h, w = (

1213 + video_frame_num,

1214 + image_grid_thw[mm_data_idx][1],

1215 + image_grid_thw[mm_data_idx][2],

1216 + )

1217 + llm_grid_t, llm_grid_h, llm_grid_w = \

1218 + t, h // spatial_merge_size, w // spatial_merge_size

1219 +

1220 + for t_idx in range(llm_grid_t):

1221 + t_index = torch.tensor(t_idx).view(-1, 1).expand(

1222 + -1, llm_grid_h * llm_grid_w).flatten()

1223 + h_index = torch.arange(llm_grid_h).view(

1224 + 1, -1, 1).expand(1, -1, llm_grid_w).flatten()

1225 + w_index = torch.arange(llm_grid_w).view(

1226 + 1, 1, -1).expand(1, llm_grid_h, -1).flatten()

1227 + llm_pos_ids_list.append(

1228 + torch.stack([t_index, h_index, w_index]) + st_idx)

1229 +

1230 + mm_data_idx += 1

1231 + video_frame_num += 1

1232 +

1233 + else:

1234 + text_len = end_idx - start_idx

1235 + llm_pos_ids_list.append(

1236 + torch.arange(text_len).view(1, -1).expand(3, -1) +

1237 + st_idx)

1238 + video_frame_num = 1

1239 +

1240 + else:

1241 + text_len = len(input_tokens)

1242 + llm_pos_ids_list.append(

1243 + torch.arange(text_len).view(1, -1).expand(3, -1))

1244 +

1245 + llm_positions = torch.cat(llm_pos_ids_list, dim=1).reshape(3, -1)

1246 + llm_positions = llm_positions[:, context_len:seq_len]

1247 + mrope_position_delta = (llm_positions.max() + 1 -

1248 + len(input_tokens)).item()

1249 + return llm_positions, mrope_position_delta

1250 +

1251 @classmethod

1252 def _vl_get_input_positions_tensor(

1253 cls,

vllm/model_executor/models/glm4_1v.py new

+1589

@@ -0,0 +1,1589 @@

1 +# SPDX-License-Identifier: Apache-2.0

2 +

3 +# Adapted from

4 +# https://github.com/huggingface/transformers/blob/main/src/transformers/models/Glm4v/modeling_Glm4v.py

9 +#

10 +# This code is based on EleutherAI's GPT-NeoX library and the GPT-NeoX

11 +# and OPT implementations in this library. It has been modified from its

12 +# original forms to accommodate minor architectural differences compared

13 +# to GPT-NeoX and OPT used by the Meta AI team that trained the model.

14 +#

15 +# Licensed under the Apache License, Version 2.0 (the "License");

16 +# you may not use this file except in compliance with the License.

17 +# You may obtain a copy of the License at

18 +#

19 +# http://www.apache.org/licenses/LICENSE-2.0

20 +#

21 +# Unless required by applicable law or agreed to in writing, software

22 +# distributed under the License is distributed on an "AS IS" BASIS,

23 +# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.

24 +# See the License for the specific language governing permissions and

25 +# limitations under the License.

26 +"""Inference-only GLM-4V model compatible with HuggingFace weights."""

27 +

28 +import math

29 +from collections.abc import Iterable, Mapping, Sequence

30 +from functools import partial

31 +from typing import Any, Callable, Literal, Optional, TypedDict, Union

32 +

33 +import numpy as np

34 +import torch

35 +import torch.nn as nn

36 +import torch.nn.functional as F

37 +from einops import rearrange

38 +from transformers import BatchFeature

39 +from transformers.models.glm4v.configuration_glm4v import (Glm4vConfig,

40 + Glm4vVisionConfig)

41 +from transformers.models.glm4v.image_processing_glm4v import (

42 + Glm4vImageProcessor, smart_resize)

43 +from transformers.models.glm4v.video_processing_glm4v import (

44 + Glm4vVideoProcessor)

45 +from transformers.video_utils import VideoMetadata

46 +

47 +from vllm.config import VllmConfig

48 +from vllm.distributed import parallel_state

49 +from vllm.distributed import utils as dist_utils

50 +from vllm.logger import init_logger

51 +from vllm.model_executor import SamplingMetadata

52 +from vllm.model_executor.layers.layernorm import RMSNorm

53 +from vllm.model_executor.layers.linear import (ColumnParallelLinear,

54 + MergedColumnParallelLinear,

55 + QKVParallelLinear,

56 + RowParallelLinear)

57 +from vllm.model_executor.layers.quantization import QuantizationConfig

58 +from vllm.model_executor.layers.quantization.gptq import GPTQConfig

59 +from vllm.model_executor.layers.quantization.gptq_marlin import (

60 + GPTQMarlinConfig)

61 +from vllm.model_executor.model_loader.weight_utils import default_weight_loader

62 +from vllm.model_executor.models.module_mapping import MultiModelKeys

63 +from vllm.multimodal import MULTIMODAL_REGISTRY

64 +from vllm.multimodal.inputs import (MultiModalDataDict, MultiModalFieldConfig,

65 + MultiModalKwargs, VideoItem)

66 +from vllm.multimodal.parse import (ImageSize, MultiModalDataItems,

67 + MultiModalDataParser)

68 +from vllm.multimodal.processing import (BaseMultiModalProcessor,

69 + BaseProcessingInfo, PromptReplacement,

70 + PromptUpdate)

71 +from vllm.multimodal.profiling import BaseDummyInputsBuilder

72 +from vllm.platforms import _Backend

73 +from vllm.sequence import IntermediateTensors

74 +from vllm.transformers_utils.config import uses_mrope

75 +

76 +from ..layers.activation import SiluAndMul

77 +from .interfaces import (MultiModalEmbeddings, SupportsLoRA,

78 + SupportsMultiModal, SupportsPP)

79 +from .qwen2_vl import _qwen2vl_field_config, apply_rotary_pos_emb_vision

80 +from .utils import (AutoWeightsLoader, WeightsMapper,

81 + init_vllm_registered_model, maybe_prefix,

82 + merge_multimodal_embeddings)

83 +from .vision import get_vit_attn_backend

84 +

85 +logger = init_logger(__name__)

86 +

87 +# For profile run

88 +_MAX_FRAMES_PER_VIDEO = 600

89 +

90 +# === Vision Inputs === #

91 +

92 +

93 +class Glm4vImagePixelInputs(TypedDict):

94 + type: Literal["pixel_values"]

95 + pixel_values: torch.Tensor

96 + """Shape:

97 + `(num_patches, num_channels * patch_size * patch_size)`

98 + """

99 +

100 + image_grid_thw: torch.Tensor

101 + """Shape: `(num_images, 3)`

102 + This should be in `(grid_t, grid_h, grid_w)` format.

103 + """

104 +

105 +

106 +class Glm4vImageEmbeddingInputs(TypedDict):

107 + type: Literal["image_embeds"]

108 + image_embeds: torch.Tensor

109 + """Supported types:

110 + - List[`torch.Tensor`]: A list of tensors holding all images' features.

111 + Each tensor holds an image's features.

112 + - `torch.Tensor`: A tensor holding all images' features

113 + (concatenation of all images' feature tensors).

114 +

115 + Tensor shape: `(num_image_features, hidden_size)`

116 + - `num_image_features` varies based on

117 + the number and resolution of the images.

118 + - `hidden_size` must match the hidden size of language model backbone.

119 + """

120 +

121 + image_grid_thw: torch.Tensor

122 + """Shape: `(num_images, 3)`

123 + This should be in `(grid_t, grid_h, grid_w)` format.

124 + """

125 +

126 +

127 +Glm4vImageInputs = Union[Glm4vImagePixelInputs, Glm4vImageEmbeddingInputs]

128 +

129 +

130 +class Glm4vVideoPixelInputs(TypedDict):

131 + type: Literal["pixel_values_videos"]

132 + pixel_values_videos: torch.Tensor

133 + """Shape:

134 + `(num_patches,

135 + num_channels * temporal_patch_size * patch_size * patch_size)`

136 + """

137 + # video_metadata: Union[list[VideoMetadata], list[dict]]

138 + video_grid_thw: Union[list[torch.Tensor], torch.Tensor]

139 + """Shape: `(num_videos, num_frames, 3)` or `(1, num_frames, 3)`

140 + for single video.

141 + Each entry represents [grid_t, grid_h, grid_w] format where:

142 + - grid_t: Temporal grid size (usually 1 for processed video)

143 + - grid_h: Height grid size

144 + - grid_w: Width grid size

145 + This describes the grid structure of the video patches.

146 + """

147 +

148 +

149 +class Glm4vVideoEmbeddingInputs(TypedDict):

150 + type: Literal["video_embeds"]

151 +

152 + video_embeds: torch.Tensor

153 + """

154 + Tensor shape: `(num_video_patches, hidden_size)`

155 + - `num_video_patches`: Total number of video patches across all frames

156 + - `hidden_size`: Must match the hidden size of language model backbone

157 + """

158 +

159 + video_grid_thw: torch.Tensor

160 + """Shape: `(num_videos, 1, 3)` or `(1, 1, 3)` for single video

161 + Each entry represents [grid_t, grid_h, grid_w] format where:

162 + - grid_t: Temporal grid size (usually 1 for processed video)

163 + - grid_h: Height grid size

164 + - grid_w: Width grid size

165 + This describes the grid structure of the video patches.

166 + """

167 +

168 +

169 +Glm4vVideoInputs = Union[Glm4vVideoPixelInputs, Glm4vVideoEmbeddingInputs]

170 +

171 +# === Vision Encoder === #

172 +

173 +

174 +class Glm4vVisionMLP(nn.Module):

175 +

176 + def __init__(

177 + self,

178 + in_features: int,

179 + hidden_features: int,

180 + bias: bool = False,

181 + quant_config: Optional[QuantizationConfig] = None,

182 + ):

183 + super().__init__()

184 + self.gate_up_proj = MergedColumnParallelLinear(

185 + input_size=in_features,

186 + output_sizes=[hidden_features] * 2,

187 + bias=bias,

188 + quant_config=quant_config,

189 + )

190 + self.down_proj = RowParallelLinear(

191 + hidden_features,

192 + in_features,

193 + bias=bias,

194 + quant_config=quant_config,

195 + )

196 + self.act_fn = SiluAndMul()

197 +

198 + def forward(self, x: torch.Tensor):

199 + x, _ = self.gate_up_proj(x)

200 + x = self.act_fn(x)

201 + x, _ = self.down_proj(x)

202 + return x

203 +

204 +

205 +def all_gather_interleave(local_tensor, hidden_size: int, tp_size: int):

206 + """All-gather the input tensor interleavely across model parallel group."""

207 + import torch.distributed as dist

208 +

209 + gathered_tensors = [torch.zeros_like(local_tensor) for _ in range(tp_size)]

210 + dist.all_gather(

211 + gathered_tensors,

212 + local_tensor,

213 + group=parallel_state.get_tp_group().device_group,

214 + )

215 +

216 + gathered_tensors_split = [

217 + torch.split(tensor, hidden_size // tp_size, -1)

218 + for tensor in gathered_tensors

219 + ]

220 + ordered_tensors = [

221 + tensor for pair in zip(*gathered_tensors_split) for tensor in pair

222 + ]

223 + result_tensor = torch.cat(ordered_tensors, dim=-1)

224 + return result_tensor

225 +

226 +

227 +class Glm4vVisionAttention(nn.Module):

228 +

229 + def __init__(

230 + self,

231 + embed_dim: int,

232 + num_heads: int,

233 + projection_size: int,

234 + quant_config: Optional[QuantizationConfig] = None,

235 + prefix: str = "",

236 + ) -> None:

237 + super().__init__()

238 + # Per attention head and per partition values.

239 + self.tp_size = parallel_state.get_tensor_model_parallel_world_size()

240 + self.tp_rank = parallel_state.get_tensor_model_parallel_rank()

241 + self.hidden_size_per_attention_head = dist_utils.divide(

242 + projection_size, num_heads)

243 + self.num_attention_heads_per_partition = dist_utils.divide(

244 + num_heads, self.tp_size)

245 +

246 + self.qkv = QKVParallelLinear(

247 + hidden_size=embed_dim,

248 + head_size=self.hidden_size_per_attention_head,

249 + total_num_heads=num_heads,

250 + total_num_kv_heads=num_heads,

251 + bias=False,

252 + quant_config=quant_config,

253 + prefix=f"{prefix}.qkv",

254 + )

255 + self.proj = RowParallelLinear(

256 + input_size=projection_size,

257 + output_size=embed_dim,

258 + quant_config=quant_config,

259 + prefix=f"{prefix}.proj",

260 + bias=False,

261 + )

262 +

263 + # Detect attention implementation.

264 + self.attn_backend: _Backend = get_vit_attn_backend(support_fa=True)

265 + if self.attn_backend not in {

266 + _Backend.FLASH_ATTN,

267 + _Backend.TORCH_SDPA,

268 + _Backend.XFORMERS,

269 + }:

270 + raise RuntimeError(

271 + f"GLM-4V does not support {self.attn_backend} backend now.")

272 +

273 + def split_qkv(self, qkv: torch.Tensor) -> tuple[torch.Tensor, ...]:

274 + # [s, b, 3 * head * head_dim]

275 + seq_len, bs, _ = qkv.shape

276 + if self.tp_size > 1:

277 + qkv = all_gather_interleave(qkv, self.qkv.hidden_size,

278 + self.tp_size)

279 +

280 + # [s, b, 3 * head * head_dim] -> 3 * [s, b, head * head_dim]

281 + q, k, v = qkv.chunk(3, dim=2)

282 +

283 + # 3 * [s, b, head * head_dim]

284 + if self.tp_size > 1:

285 + splitter = partial(

286 + dist_utils.split_tensor_along_last_dim,

287 + num_partitions=self.tp_size,

288 + )

289 + q = splitter(q)[self.tp_rank]

290 + k = splitter(k)[self.tp_rank]

291 + v = splitter(v)[self.tp_rank]

292 +

293 + # 3 * [s, b, head * head_dim] -> 3 * [s, b, head, head_dim]

294 + new_shape = (

295 + seq_len,

296 + bs,

297 + self.num_attention_heads_per_partition,

298 + self.hidden_size_per_attention_head,

299 + )

300 + q, k, v = (x.view(*new_shape) for x in (q, k, v))

301 + return q, k, v

302 +

303 + def forward(

304 + self,

305 + x: torch.Tensor,

306 + cu_seqlens: torch.Tensor,

307 + rotary_pos_emb: torch.Tensor,

308 + max_seqlen: Optional[int] = None, # Only used for Flash Attention

309 + seqlens: Optional[list[int]] = None, # Only used for xFormers

310 + ) -> torch.Tensor:

311 + # [s, b, c] --> [s, b, head * 3 * head_dim]

312 + x, _ = self.qkv(x)

313 +

314 + # [s, b, 3 * head * head_dim] -> 3 * [s, b, head, head_dim]

315 + q, k, v = self.split_qkv(x)

316 + batch_size = q.shape[1]

317 +

318 + q, k, v = (rearrange(x, "s b ... -> b s ...").contiguous()

319 + for x in (q, k, v))

320 + if rotary_pos_emb is not None:

321 + q = apply_rotary_pos_emb_vision(q, rotary_pos_emb)

322 + k = apply_rotary_pos_emb_vision(k, rotary_pos_emb)

323 +

324 + if self.attn_backend == _Backend.FLASH_ATTN:

325 + # from vllm_flash_attn.flash_attn_interface import (

326 + # flash_attn_varlen_func)

327 + from flash_attn import flash_attn_varlen_func

328 +

329 + q, k, v = (rearrange(x, "b s ... -> (b s) ...") for x in [q, k, v])

330 +

331 + output = flash_attn_varlen_func(

332 + q,

333 + k,

334 + v,

335 + cu_seqlens_q=cu_seqlens,

336 + cu_seqlens_k=cu_seqlens,

337 + max_seqlen_q=max_seqlen,

338 + max_seqlen_k=max_seqlen,

339 + dropout_p=0,

340 + causal=False,

341 + )

342 +

343 + context_layer = rearrange(output,

344 + "(b s) ... -> b s ...",

345 + b=batch_size)

346 + elif self.attn_backend == _Backend.TORCH_SDPA:

347 + # Execute attention entry by entry for speed & less VRAM.

348 + outputs = []

349 + for i in range(1, len(cu_seqlens)):

350 + start_idx = cu_seqlens[i - 1]

351 + end_idx = cu_seqlens[i]

352 + q_i = q[:, start_idx:end_idx]

353 + k_i = k[:, start_idx:end_idx]

354 + v_i = v[:, start_idx:end_idx]

355 + q_i, k_i, v_i = (rearrange(x, "b s h d -> b h s d")

356 + for x in [q_i, k_i, v_i])

357 + output_i = F.scaled_dot_product_attention(q_i,

358 + k_i,

359 + v_i,

360 + dropout_p=0.0)

361 + output_i = rearrange(output_i, "b h s d -> b s h d ")

362 + outputs.append(output_i)

363 + context_layer = torch.cat(outputs, dim=1)

364 + elif self.attn_backend == _Backend.XFORMERS:

365 + from xformers import ops as xops

366 + from xformers.ops.fmha.attn_bias import BlockDiagonalMask

367 +

368 + attn_bias = BlockDiagonalMask.from_seqlens(q_seqlen=seqlens,

369 + kv_seqlen=None,

370 + device=q.device)

371 +

372 + context_layer = xops.memory_efficient_attention_forward(

373 + q, k, v, attn_bias=attn_bias, p=0, scale=None)

374 +

375 + context_layer = rearrange(context_layer,

376 + "b s h d -> s b (h d)").contiguous()

377 +

378 + output, _ = self.proj(context_layer)

379 + return output

380 +

381 +

382 +class Glm4vVisionBlock(nn.Module):

383 +

384 + def __init__(

385 + self,

386 + dim: int,

387 + num_heads: int,

388 + mlp_hidden_dim: int,

389 + norm_layer: Optional[Callable[[int], nn.Module]] = None,

390 + quant_config: Optional[QuantizationConfig] = None,

391 + prefix: str = "",

392 + ) -> None:

393 + super().__init__()

394 + if norm_layer is None:

395 + norm_layer = partial(nn.LayerNorm, eps=1e-6)

396 + self.norm1 = norm_layer(dim)

397 + self.norm2 = norm_layer(dim)

398 + self.attn = Glm4vVisionAttention(

399 + embed_dim=dim,

400 + num_heads=num_heads,

401 + projection_size=dim,

402 + quant_config=quant_config,

403 + prefix=f"{prefix}.attn",

404 + )

405 + self.mlp = Glm4vVisionMLP(

406 + dim,

407 + mlp_hidden_dim,

408 + bias=False,

409 + quant_config=quant_config,

410 + )

411 +

412 + def forward(

413 + self,

414 + x: torch.Tensor,

415 + cu_seqlens: torch.Tensor,

416 + rotary_pos_emb: torch.Tensor,

417 + max_seqlen: Optional[int] = None, # Only used for Flash Attention

418 + seqlens: Optional[list[int]] = None, # Only used for xFormers

419 + ) -> torch.Tensor:

420 + x = x + self.attn(

421 + self.norm1(x),

422 + cu_seqlens=cu_seqlens,

423 + rotary_pos_emb=rotary_pos_emb,

424 + max_seqlen=max_seqlen,

425 + seqlens=seqlens,

426 + )

427 +

428 + x = x + self.mlp(self.norm2(x))

429 + return x

430 +

431 +

432 +class Glm4vVisionPatchEmbed(nn.Module):

433 +

434 + def __init__(

435 + self,

436 + patch_size: int = 14,

437 + temporal_patch_size: int = 1,

438 + in_channels: int = 3,

439 + hidden_size: int = 1536,

440 + ) -> None:

441 + super().__init__()

442 + self.patch_size = patch_size

443 + self.temporal_patch_size = temporal_patch_size

444 + self.hidden_size = hidden_size

445 +

446 + kernel_size = (temporal_patch_size, patch_size, patch_size)

447 + self.proj = nn.Conv3d(

448 + in_channels,

449 + hidden_size,

450 + kernel_size=kernel_size,

451 + stride=kernel_size,

452 + bias=True,

453 + )

454 +

455 + def forward(self, x: torch.Tensor) -> torch.Tensor:

456 + L, C = x.shape

457 + x = x.view(L, -1, self.temporal_patch_size, self.patch_size,

458 + self.patch_size)

459 + x = self.proj(x).view(L, self.hidden_size)

460 + return x

461 +

462 +

463 +class Glm4vPatchMerger(nn.Module):

464 +

465 + def __init__(

466 + self,

467 + d_model: int,

468 + context_dim: int,

469 + quant_config: Optional[QuantizationConfig] = None,

470 + bias: bool = False,

471 + ) -> None:

472 + super().__init__()

473 + self.hidden_size = d_model

474 + self.proj = ColumnParallelLinear(self.hidden_size,

475 + self.hidden_size,

476 + bias=bias,

477 + gather_output=True)

478 + self.post_projection_norm = nn.LayerNorm(self.hidden_size)

479 + self.gate_up_proj = MergedColumnParallelLinear(

480 + input_size=self.hidden_size,

481 + output_sizes=[context_dim] * 2,

482 + bias=bias,

483 + quant_config=quant_config,

484 + )

485 + self.down_proj = RowParallelLinear(

486 + context_dim,

487 + self.hidden_size,

488 + bias=bias,

489 + quant_config=quant_config,

490 + )

491 + self.act_fn = SiluAndMul()

492 + self.extra_activation_func = nn.GELU()

493 +

494 + def forward(self, x: torch.Tensor):

495 + x, _ = self.proj(x)

496 + x = self.extra_activation_func(self.post_projection_norm(x))

497 + gate_up, _ = self.gate_up_proj(x)

498 + x = self.act_fn(gate_up)

499 + x, _ = self.down_proj(x)

500 + return x

501 +

502 +

503 +class Glm4vVisionEmbeddings(nn.Module):

504 +

505 + def __init__(self, config: Glm4vVisionConfig):

506 + super().__init__()

507 + self.config = config

508 + self.embed_dim = config.hidden_size

509 + self.image_size = config.image_size

510 + self.patch_size = config.patch_size

511 +

512 + self.num_patches = (self.image_size // self.patch_size)**2

513 + self.num_positions = self.num_patches

514 + self.position_embedding = nn.Embedding(self.num_positions,

515 + self.embed_dim)

516 + self.register_buffer(

517 + "position_ids",

518 + torch.arange(self.num_positions).expand((1, -1)),

519 + persistent=False,

520 + )

521 +

522 + def forward(self, embeddings, lengths, image_shapes, h_coords,

523 + w_coords) -> torch.Tensor:

524 + pos_embed_weight = self.position_embedding.weight

525 + hidden_size = pos_embed_weight.shape[1]

526 + total_seq = h_coords.shape[0]

527 + device = pos_embed_weight.device

528 +

529 + # Move coordinates to correct device

530 + h_coords, w_coords = h_coords.to(device), w_coords.to(device)

531 +

532 + # Handle empty sequence case

533 + if total_seq == 0:

534 + adapted_pos_embed = torch.empty(0,

535 + hidden_size,

536 + device=device,

537 + dtype=pos_embed_weight.dtype)

538 + else:

539 + # Convert inputs to tensors if needed

540 + if isinstance(lengths, list):

541 + lengths = torch.tensor(lengths,

542 + device=device,

543 + dtype=torch.long)

544 + if not isinstance(image_shapes, torch.Tensor):

545 + image_shapes = torch.tensor(image_shapes,

546 + device=device,

547 + dtype=torch.long)

548 +

549 + # Prepare 2D position embedding

550 + orig_size_sq = pos_embed_weight.shape[0]

551 + orig_size = int(orig_size_sq**0.5)

552 + pos_embed_2d = (pos_embed_weight.view(

553 + orig_size, orig_size,

554 + hidden_size).permute(2, 0,

555 + 1).unsqueeze(0).to(device=device,

556 + dtype=torch.float32))

557 +

558 + # Calculate target dimensions for each patch

559 + target_h = torch.cat([

560 + image_shapes[i, 1].repeat(lengths[i])

561 + for i in range(len(lengths))

562 + ]).to(device=device, dtype=torch.float32)

563 + target_w = torch.cat([

564 + image_shapes[i, 2].repeat(lengths[i])

565 + for i in range(len(lengths))

566 + ]).to(device=device, dtype=torch.float32)

567 +

568 + # Normalize coordinates to [-1, 1] range for grid_sample

569 + h_coords = h_coords.to(device=device, dtype=torch.float32)

570 + w_coords = w_coords.to(device=device, dtype=torch.float32)

571 + norm_w = ((w_coords + 0.5) / target_w) * 2 - 1

572 + norm_h = ((h_coords + 0.5) / target_h) * 2 - 1

573 +

574 + # Create sampling grid

575 + grid = (torch.stack((norm_w, norm_h),

576 + dim=-1).unsqueeze(0).unsqueeze(2))

577 +

578 + # Perform bicubic interpolation

579 + interpolated_embed_fp32 = F.grid_sample(

580 + pos_embed_2d,

581 + grid,

582 + mode="bicubic",

583 + align_corners=False,

584 + padding_mode="border",

585 + )

586 +

587 + # Reshape and convert back to original dtype

588 + adapted_pos_embed_fp32 = (

589 + interpolated_embed_fp32.squeeze(0).squeeze(-1).permute(1, 0))

590 + adapted_pos_embed = adapted_pos_embed_fp32.to(

591 + pos_embed_weight.dtype).to(embeddings.device)

592 +

593 + # Add adapted position encoding to embeddings

594 + embeddings = embeddings + adapted_pos_embed

595 + return embeddings

596 +

597 +

598 +class Glm4vVisionRotaryEmbedding(nn.Module):

599 +

600 + def __init__(self, dim: int, theta: float = 10000.0) -> None:

601 + super().__init__()

602 + self.dim = dim

603 + self.theta = theta

604 + inv_freq = 1.0 / (theta

605 + **(torch.arange(0, dim, 2, dtype=torch.float) / dim))

606 + self.register_buffer("inv_freq", inv_freq, persistent=False)

607 + self._seq_len_cached = 0

608 + self._freqs_cached = None

609 +

610 + def update_freqs_cache(self, seqlen: int) -> None:

611 + if seqlen > self._seq_len_cached:

612 + seqlen *= 2

613 + self._seq_len_cached = seqlen

614 + self.inv_freq = 1.0 / (self.theta**(torch.arange(

615 + 0,

616 + self.dim,

617 + 2,

618 + dtype=torch.float,

619 + device=self.inv_freq.device,

620 + ) / self.dim))

621 + seq = torch.arange(seqlen,

622 + device=self.inv_freq.device,

623 + dtype=self.inv_freq.dtype)

624 + freqs = torch.outer(seq, self.inv_freq)

625 + self._freqs_cached = freqs

626 +

627 + def forward(self, seqlen: int) -> torch.Tensor:

628 + self.update_freqs_cache(seqlen)

629 + return self._freqs_cached[:seqlen]

630 +

631 +

632 +class Glm4vVisionTransformer(nn.Module):

633 +

634 + def __init__(

635 + self,

636 + vision_config: Glm4vVisionConfig,

637 + norm_eps: float = 1e-6,

638 + quant_config: Optional[QuantizationConfig] = None,

639 + prefix: str = "",

640 + ) -> None:

641 + super().__init__()

642 +

643 + patch_size = vision_config.patch_size

644 + temporal_patch_size = vision_config.temporal_patch_size

645 + in_channels = vision_config.in_channels

646 + depth = vision_config.depth

647 + self.hidden_size = vision_config.hidden_size

648 + self.num_heads = vision_config.num_heads

649 +

650 + self.patch_size = vision_config.patch_size

651 + self.spatial_merge_size = vision_config.spatial_merge_size

652 + self.out_hidden_size = vision_config.out_hidden_size

653 +

654 + self.patch_embed = Glm4vVisionPatchEmbed(

655 + patch_size=patch_size,

656 + temporal_patch_size=temporal_patch_size,

657 + in_channels=in_channels,

658 + hidden_size=self.hidden_size,

659 + )

660 +

661 + norm_layer = partial(RMSNorm, eps=norm_eps)

662 + head_dim = self.hidden_size // self.num_heads

663 + self.rotary_pos_emb = Glm4vVisionRotaryEmbedding(head_dim // 2)

664 + self.blocks = nn.ModuleList([

665 + Glm4vVisionBlock(

666 + dim=self.hidden_size,

667 + num_heads=self.num_heads,

668 + mlp_hidden_dim=vision_config.out_hidden_size,

669 + norm_layer=norm_layer,

670 + quant_config=quant_config,

671 + prefix=f"{prefix}.blocks.{layer_idx}",

672 + ) for layer_idx in range(depth)

673 + ])

674 + self.merger = Glm4vPatchMerger(

675 + d_model=vision_config.out_hidden_size,

676 + context_dim=vision_config.intermediate_size,

677 + quant_config=quant_config,

678 + bias=False,

679 + )

680 + self.embeddings = Glm4vVisionEmbeddings(vision_config)

681 +

682 + self.post_conv_layernorm = RMSNorm(vision_config.hidden_size,

683 + eps=vision_config.rms_norm_eps)

684 + self.downsample = nn.Conv2d(

685 + in_channels=vision_config.hidden_size,

686 + out_channels=vision_config.out_hidden_size,

687 + kernel_size=vision_config.spatial_merge_size,

688 + stride=vision_config.spatial_merge_size,

689 + )

690 + self.post_layernorm = RMSNorm(vision_config.hidden_size,

691 + eps=vision_config.rms_norm_eps)

692 +

693 + self.attn_backend: _Backend = get_vit_attn_backend(support_fa=True)

694 +

695 + @property

696 + def dtype(self) -> torch.dtype:

697 + return self.patch_embed.proj.weight.dtype

698 +

699 + @property

700 + def device(self) -> torch.device:

701 + return self.patch_embed.proj.weight.device

702 +

703 + def rot_pos_emb(self, grid_thw: torch.Tensor) -> torch.Tensor:

704 + pos_ids = []

705 + for t, h, w in grid_thw:

706 + hpos_ids = torch.arange(h).unsqueeze(1).expand(-1, w)

707 + wpos_ids = torch.arange(w).unsqueeze(0).expand(h, -1)

708 + hpos_ids = (hpos_ids.reshape(

709 + h // self.spatial_merge_size,

710 + self.spatial_merge_size,

711 + w // self.spatial_merge_size,

712 + self.spatial_merge_size,

713 + ).permute(0, 2, 1, 3).flatten())

714 + wpos_ids = (wpos_ids.reshape(

715 + h // self.spatial_merge_size,

716 + self.spatial_merge_size,

717 + w // self.spatial_merge_size,

718 + self.spatial_merge_size,

719 + ).permute(0, 2, 1, 3).flatten())

720 + pos_ids.append(

721 + torch.stack([hpos_ids, wpos_ids], dim=-1).repeat(t, 1))

722 + pos_ids = torch.cat(pos_ids, dim=0)

723 + max_grid_size = grid_thw[:, 1:].max()

724 + rotary_pos_emb_full = self.rotary_pos_emb(max_grid_size)

725 + rotary_pos_emb = rotary_pos_emb_full[pos_ids].flatten(1)

726 + return rotary_pos_emb, pos_ids

727 +

728 + def compute_attn_mask_seqlen(

729 + self,

730 + cu_seqlens: torch.Tensor,

731 + ) -> tuple[Optional[int], Optional[list[int]]]:

732 + max_seqlen, seqlens = None, None

733 + seqlens = (cu_seqlens[1:] - cu_seqlens[:-1]).tolist()

734 + if self.attn_backend == _Backend.FLASH_ATTN:

735 + max_seqlen = (cu_seqlens[1:] - cu_seqlens[:-1]).max().item()

736 + return max_seqlen, seqlens

737 +

738 + def forward(

739 + self,

740 + x: torch.Tensor,

741 + grid_thw: torch.Tensor,

742 + ) -> torch.Tensor:

743 + # patchify

744 + x = x.to(device=self.device, dtype=self.dtype)

745 + x = self.patch_embed(x)

746 + x = self.post_conv_layernorm(x)

747 +

748 + # compute position embedding

749 + rotary_pos_emb, image_type_ids = self.rot_pos_emb(grid_thw)

750 + # compute cu_seqlens

751 + cu_seqlens = torch.repeat_interleave(grid_thw[:, 1] * grid_thw[:, 2],

752 + grid_thw[:, 0]).cumsum(

753 + dim=0, dtype=torch.int32)

754 + cu_seqlens = F.pad(cu_seqlens, (1, 0), "constant", 0)

755 +

756 + # pre-compute seqlens for attn mask to reduce cuMemcpy operations

757 + max_seqlen, seqlens = self.compute_attn_mask_seqlen(cu_seqlens)

758 + x = self.embeddings(x, seqlens, grid_thw, image_type_ids[:, 0],

759 + image_type_ids[:, 1])

760 +

761 + # transformers

762 + x = x.unsqueeze(1)

763 + for blk in self.blocks:

764 + x = blk(

765 + x,

766 + cu_seqlens=cu_seqlens,

767 + rotary_pos_emb=rotary_pos_emb,

768 + max_seqlen=max_seqlen,

769 + seqlens=seqlens,

770 + )

771 +

772 + # adapter

773 + x = self.post_layernorm(x)

774 +

775 + x = x.view(-1, self.spatial_merge_size, self.spatial_merge_size,

776 + x.shape[-1])

777 + x = x.permute(0, 3, 1, 2)

778 + x = self.downsample(x).view(-1, self.out_hidden_size)

779 + x = self.merger(x)

780 +

781 + return x

782 +

783 + def load_weights(self, weights: Iterable[tuple[str,

784 + torch.Tensor]]) -> set[str]:

785 + stacked_params_mapping = [

786 + # (param_name, shard_name, shard_id)

787 + ("attn.qkv.", "attn.q.", "q"),

788 + ("attn.qkv.", "attn.k.", "k"),

789 + ("attn.qkv.", "attn.v.", "v"),

790 + ("gate_up_proj", "gate_proj", 0),

791 + ("gate_up_proj", "up_proj", 1),

792 + ]

793 + params_dict = dict(self.named_parameters(remove_duplicate=False))

794 + loaded_params: set[str] = set()

795 +

796 + for name, loaded_weight in weights:

797 + for param_name, weight_name, shard_id in stacked_params_mapping:

798 + if weight_name not in name:

799 + continue

800 + name = name.replace(weight_name, param_name)

801 +

802 + param = params_dict[name]

803 + weight_loader = param.weight_loader

804 + weight_loader(param, loaded_weight, shard_id)

805 + break

806 + else:

807 + param = params_dict[name]

808 + weight_loader = getattr(param, "weight_loader",

809 + default_weight_loader)

810 + weight_loader(param, loaded_weight)

811 + loaded_params.add(name)

812 + return loaded_params

813 +

814 +

815 +class Glm4vProcessingInfo(BaseProcessingInfo):

816 +

817 + def get_hf_config(self):

818 + return self.ctx.get_hf_config(Glm4vConfig)

819 +

820 + def get_tokenizer(self):

821 + return self.ctx.tokenizer

822 +

823 + def get_supported_mm_limits(self) -> Mapping[str, Optional[int]]:

824 + return {"image": None, "video": 1}

825 +

826 + def get_image_processor(self) -> Glm4vImageProcessor:

827 + return self.get_hf_processor().image_processor

828 +

829 + def get_video_processor(self) -> Glm4vVideoProcessor:

830 + return self.get_hf_processor().video_processor

831 +

832 + def _get_vision_info(

833 + self,

834 + *,

835 + image_width: int,

836 + image_height: int,

837 + num_frames: int = 16,

838 + do_resize: bool = True,

839 + max_image_pixels: int = 28 * 28 * 2 * 30000,

840 + ) -> tuple[ImageSize, int]:

841 + hf_config = self.get_hf_config()

842 + vision_config = hf_config.vision_config

843 + patch_size = vision_config.patch_size

844 + merge_size = vision_config.spatial_merge_size

845 + temporal_patch_size = vision_config.temporal_patch_size

846 + if do_resize:

847 + resized_height, resized_width = smart_resize(

848 + num_frames=num_frames

849 + if num_frames > temporal_patch_size else temporal_patch_size,

850 + height=image_height,

851 + width=image_width,

852 + factor=patch_size * merge_size,

853 + max_pixels=max_image_pixels,

854 + )

855 + preprocessed_size = ImageSize(width=resized_width,

856 + height=resized_height)

857 + else:

858 + preprocessed_size = ImageSize(width=image_width,

859 + height=image_height)

860 +

861 + # NOTE: Frames are padded to be divisible by `temporal_patch_size`

862 + # https://github.com/huggingface/transformers/blob/v4.48.3/src/transformers/models/qwen2_vl/image_processing_qwen2_vl.py#L294

863 + padded_num_frames = num_frames + num_frames % temporal_patch_size

864 +

865 + grid_t = max(padded_num_frames // temporal_patch_size, 1)

866 + grid_h = preprocessed_size.height // patch_size

867 + grid_w = preprocessed_size.width // patch_size

868 +

869 + num_patches = grid_t * grid_h * grid_w

870 + num_vision_tokens = num_patches // (merge_size**2)

871 +

872 + return preprocessed_size, num_vision_tokens

873 +

874 + def get_image_size_with_most_features(self) -> ImageSize:

875 + max_image_size, _ = self._get_vision_info(image_width=9999999,

876 + image_height=9999999)

877 + return max_image_size

878 +

879 + def get_num_image_tokens(

880 + self,

881 + *,

882 + image_width: int,

883 + image_height: int,

884 + ) -> int:

885 + _, num_image_tokens = self._get_vision_info(

886 + image_width=image_width,

887 + image_height=image_height,

888 + max_image_pixels=28 * 28 * 2 * 6144,

889 + )

890 + return num_image_tokens

891 +

892 + def get_max_image_tokens(self) -> int:

893 + target_width, target_height = self.get_image_size_with_most_features()

894 +

895 + return self.get_num_image_tokens(

896 + image_width=target_width,

897 + image_height=target_height,

898 + )

899 +

900 + def get_num_video_tokens(

901 + self,

902 + *,

903 + image_width: int,

904 + image_height: int,

905 + num_frames: int,

906 + ) -> int:

907 + _, num_video_tokens = self._get_vision_info(

908 + image_width=image_width,

909 + image_height=image_height,

910 + num_frames=num_frames,

911 + max_image_pixels=28 * 28 * 2 * 30000,

912 + )

913 + return num_video_tokens

914 +

915 + def _get_max_video_frames(self, max_tokens: int) -> int:

916 + target_width, target_height = self.get_image_size_with_most_features()

917 +

918 + num_frames = 0

919 +

920 + while True:

921 + next_num_frames = num_frames + 1

922 + next_max_tokens = self.get_num_video_tokens(

923 + image_width=target_width,

924 + image_height=target_height,

925 + num_frames=next_num_frames,

926 + )

927 + if next_max_tokens > max_tokens or next_max_tokens == 0:

928 + break

929 +

930 + num_frames = next_num_frames

931 +

932 + return num_frames

933 +

934 + def get_num_frames_with_most_features(

935 + self,

936 + seq_len: int,

937 + mm_counts: Mapping[str, int],

938 + ) -> int:

939 + max_images = mm_counts.get("image", 0)

940 + max_videos = mm_counts.get("video", 0)

941 +

942 + max_image_tokens = self.get_max_image_tokens() * max_images

943 + max_total_frames = self._get_max_video_frames(seq_len -

944 + max_image_tokens)

945 + max_frames_per_video = min(max_total_frames // max(max_videos, 1),

946 + _MAX_FRAMES_PER_VIDEO)

947 +

948 + return max(max_frames_per_video, 1)

949 +

950 + def _get_video_second_idx(self, metadata: dict[str, Any],

951 + total_frames: int) -> list[int]:

952 + video_processor = self.get_video_processor()

953 +

954 + video_fps = metadata.get("fps", 2.0)

955 + meta_frames = metadata.get("total_num_frames", total_frames)

956 + max_frame_idx = meta_frames - 1

957 + duration = metadata.get("duration",

958 + round(max_frame_idx / video_fps) + 1)

959 + if duration <= video_processor.max_duration:

960 + n = int(math.floor(duration * video_processor.fps))

961 + frame_indices = [

962 + min(

963 + max_frame_idx,

964 + int(math.ceil(i * video_fps / video_processor.fps)),

965 + ) for i in range(n)

966 + ]

967 + else:

968 + num_samples = int(video_processor.max_duration *

969 + video_processor.fps)

970 + if num_samples >= meta_frames:

971 + frame_indices = list(range(meta_frames))

972 + else:

973 + target_seconds = np.linspace(0,

974 + duration,

975 + num_samples,

976 + endpoint=True)

977 + frame_indices = [

978 + min(max_frame_idx, int(math.ceil(t * video_fps)))

979 + for t in target_seconds

980 + ]

981 +

982 + seen, uniq = set(), []

983 + for idx in frame_indices:

984 + if idx not in seen:

985 + seen.add(idx)

986 + uniq.append(idx)

987 + if len(uniq) & 1:

988 + uniq.append(uniq[-1])

989 + frame_indices = uniq

990 +

991 + full_second_idxs = [int(idx / video_fps) for idx in frame_indices]

992 + timestamps_list = full_second_idxs[::2]

993 + selected_timestamps = []

994 + for idx in range(0, len(timestamps_list)):

995 + selected_timestamps.append(timestamps_list[idx])

996 + return selected_timestamps

997 +

998 +

999 +class Glm4vDummyInputsBuilder(BaseDummyInputsBuilder[Glm4vProcessingInfo]):

1000 +

1001 + def get_dummy_text(self, mm_counts: Mapping[str, int]) -> str:

1002 + num_images = mm_counts.get("image", 0)

1003 + num_videos = mm_counts.get("video", 0)

1004 +

1005 + hf_config = self.info.get_hf_config()

1006 + hf_processor = self.info.get_hf_processor()

1007 + tokenizer = self.info.get_tokenizer()

1008 +

1009 + image_token: str = hf_processor.image_token

1010 + video_token_ids = [

1011 + hf_config.video_start_token_id,

1012 + hf_processor.video_token_id,

1013 + hf_config.video_end_token_id,

1014 + ]

1015 + video_token = tokenizer.decode(video_token_ids)

1016 +

1017 + return image_token * num_images + video_token * num_videos

1018 +

1019 + def get_dummy_mm_data(

1020 + self,

1021 + seq_len: int,

1022 + mm_counts: Mapping[str, int],

1023 + ) -> MultiModalDataDict:

1024 + num_images = mm_counts.get("image", 0)

1025 + num_videos = mm_counts.get("video", 0)

1026 +

1027 + target_width, target_height = (

1028 + self.info.get_image_size_with_most_features())

1029 + target_num_frames = self.info.get_num_frames_with_most_features(

1030 + seq_len, mm_counts)

1031 + return {

1032 + "image":

1033 + self._get_dummy_images(width=target_width,

1034 + height=target_height,

1035 + num_images=num_images),

1036 + "video":

1037 + self._get_dummy_videos(

1038 + width=target_width,

1039 + height=target_height,

1040 + num_frames=target_num_frames,

1041 + num_videos=num_videos,

1042 + ),

1043 + }

1044 +

1045 + def _get_dummy_videos(

1046 + self,

1047 + *,

1048 + width: int,

1049 + height: int,

1050 + num_frames: int,

1051 + num_videos: int,

1052 + ) -> list[VideoItem]:

1053 + video = np.full((num_frames, width, height, 3), 255, dtype=np.uint8)

1054 + video_items = []

1055 + for i in range(num_videos):

1056 + video_metadata = {

1057 + "fps": 2.0,

1058 + "duration": num_frames / 2.0,

1059 + "total_num_frames": num_frames,

1060 + "video_backend": "opencv",

1061 + }

1062 + video_item = (video.copy(), video_metadata)

1063 + video_items.append(video_item)

1064 +

1065 + return video_items

1066 +

1067 +

1068 +class Glm4vMultiModalProcessor(BaseMultiModalProcessor[Glm4vProcessingInfo]):

1069 +

1070 + def _get_data_parser(self) -> MultiModalDataParser:

1071 + return MultiModalDataParser(video_needs_metadata=True)

1072 +

1073 + def _call_hf_processor(

1074 + self,

1075 + prompt: str,

1076 + mm_data: Mapping[str, object],

1077 + mm_kwargs: Mapping[str, object],

1078 + tok_kwargs: Mapping[str, object],

1079 + ) -> BatchFeature:

1080 + mm_data = dict(mm_data)

1081 + processor = self.info.get_hf_processor(**mm_kwargs)

1082 +

1083 + # GLM-4.1V use `image_token_id` as video placeholder, we need to

1084 + # replace it with `video_token_id` for video processing. So we

1085 + # separate video processing from image processing.

1086 + if ("videos" in mm_data and isinstance(mm_data["videos"], list)

1087 + and len(mm_data["videos"]) > 0):

1088 + video_grid_thw_lst = []

1089 + pixel_values_videos_lst = []

1090 + for item in mm_data.pop("videos", []):

1091 + video_array, metadata = item

1092 +

1093 + # FIXME(Isotr0py): Activate the below logic after we can disable

1094 + # resampling from video loader backend.

1095 + # assert metadata["total_num_frames"] == len(video_array), (

1096 + # f"Total frames {metadata['total_num_frames']} does not "

1097 + # f"match the length of video array {len(video_array)}.")

1098 +

1099 + # NOTE: Temporary workaround for resampled videos.

1100 + # this can cause a divergence with HF implementation if

1101 + # the input video is resampled in advance.

1102 +

1103 + if metadata["total_num_frames"] != len(video_array):

1104 + logger.warning(

1105 + "Total frames in metadata "

1106 + "(%s) does not match the length of "

1107 + "video array %s. This can "

1108 + "be because the video is resampled "

1109 + "in advance. This may cause "

1110 + "a divergence with HF implementation.",

1111 + metadata["total_num_frames"],

1112 + len(video_array),

1113 + )

1114 + metadata["total_num_frames"] = len(video_array)

1115 + metadata = VideoMetadata(**metadata)

1116 +

1117 + video_mm_data = dict()

1118 + video_mm_data["videos"] = [[video_array]]

1119 + video_mm_data["video_metadata"] = [[metadata]]

1120 +

1121 + video_outputs = super()._call_hf_processor(

1122 + prompt="<|begin_of_video|><|video|><|end_of_video|>",

1123 + mm_data=video_mm_data,

1124 + mm_kwargs=mm_kwargs,

1125 + tok_kwargs=tok_kwargs,

1126 + )

1127 + input_ids = video_outputs.pop("input_ids")

1128 + input_ids[input_ids == processor.image_token_id] = (

1129 + processor.video_token_id)

1130 + video_placeholder = processor.tokenizer.batch_decode(

1131 + input_ids)[0]

1132 + prompt = prompt.replace(

1133 + "<|begin_of_video|><|video|><|end_of_video|>",

1134 + video_placeholder,

1135 + )

1136 +

1137 + grid_t = len(video_outputs["video_grid_thw"])

1138 + _, grid_h, grid_w = video_outputs["video_grid_thw"][0]

1139 + grid_thw = torch.tensor([[grid_t, grid_h, grid_w]])

1140 +

1141 + video_grid_thw_lst.append(grid_thw)

1142 + pixel_values_videos_lst.append(

1143 + video_outputs["pixel_values_videos"])

1144 + video_outputs = dict(

1145 + pixel_values_videos=torch.cat(pixel_values_videos_lst),

1146 + video_grid_thw=torch.cat(video_grid_thw_lst),

1147 + )

1148 + else:

1149 + video_outputs = dict()

1150 +

1151 + processed_outputs = super()._call_hf_processor(

1152 + prompt=prompt,

1153 + mm_data=mm_data,

1154 + mm_kwargs=mm_kwargs,

1155 + tok_kwargs=tok_kwargs,

1156 + )

1157 + combined_outputs = dict(

1158 + processed_outputs,

1159 + **video_outputs,

1160 + )

1161 + return BatchFeature(combined_outputs)

1162 +

1163 + def _get_mm_fields_config(

1164 + self,

1165 + hf_inputs: BatchFeature,

1166 + hf_processor_mm_kwargs: Mapping[str, object],

1167 + ) -> Mapping[str, MultiModalFieldConfig]:

1168 + return _qwen2vl_field_config(hf_inputs)

1169 +

1170 + def _get_prompt_updates(

1171 + self,

1172 + mm_items: MultiModalDataItems,

1173 + hf_processor_mm_kwargs: Mapping[str, Any],

1174 + out_mm_kwargs: MultiModalKwargs,

1175 + ) -> Sequence[PromptUpdate]:

1176 + hf_processor = self.info.get_hf_processor(**hf_processor_mm_kwargs)

1177 + image_processor = self.info.get_image_processor(

1178 + **hf_processor_mm_kwargs)

1179 + tokenizer = self.info.get_tokenizer()

1180 + hf_config = self.info.get_hf_config()

1181 +

1182 + boi_token_id = hf_config.image_start_token_id

1183 + eoi_token_id = hf_config.image_end_token_id

1184 +

1185 + bov_token_id = hf_config.video_start_token_id

1186 + eov_token_id = hf_config.video_end_token_id

1187 +

1188 + merge_length = image_processor.merge_size**2

1189 +

1190 + def get_image_replacement_glm4v(item_idx: int):

1191 + grid_thw = out_mm_kwargs["image_grid_thw"][item_idx]

1192 + assert isinstance(grid_thw, torch.Tensor)

1193 +

1194 + num_tokens = int(grid_thw.prod()) // merge_length

1195 + return [hf_processor.image_token_id] * num_tokens

1196 +

1197 + def get_video_replacement_glm4v(item_idx: int):

1198 + grid_thw = out_mm_kwargs["video_grid_thw"][item_idx]

1199 + assert isinstance(grid_thw, torch.Tensor)

1200 +

1201 + video, metadata = mm_items["video"][item_idx]

1202 + timestamps = self.info._get_video_second_idx(metadata, len(video))

1203 + frames_idx_token = [

1204 + tokenizer.encode(str(i), add_special_tokens=False)

1205 + for i in timestamps

1206 + ]

1207 + num_tokens_per_frame = int(grid_thw[1:].prod()) // merge_length

1208 + placeholder = []

1209 + placeholder.append(bov_token_id)

1210 + for frame_idx in frames_idx_token:

1211 + placeholder.append(boi_token_id)

1212 + placeholder.extend([hf_processor.video_token_id] *

1213 + num_tokens_per_frame)

1214 + placeholder.append(eoi_token_id)

1215 + placeholder.extend(frame_idx)

1216 + placeholder.append(eov_token_id)

1217 + return placeholder

1218 +

1219 + return [

1220 + PromptReplacement(

1221 + modality="image",

1222 + target=hf_processor.image_token,

1223 + replacement=get_image_replacement_glm4v,

1224 + ),

1225 + PromptReplacement(

1226 + modality="video",

1227 + target="<|begin_of_video|><|video|><|end_of_video|>",

1228 + replacement=get_video_replacement_glm4v,

1229 + ),

1230 + ]

1231 +

1232 +

1233 +@MULTIMODAL_REGISTRY.register_processor(

1234 + Glm4vMultiModalProcessor,

1235 + info=Glm4vProcessingInfo,

1236 + dummy_inputs=Glm4vDummyInputsBuilder,

1237 +)

1238 +class Glm4vForConditionalGeneration(nn.Module, SupportsMultiModal,

1239 + SupportsLoRA, SupportsPP):

1240 + packed_modules_mapping = {

1241 + "qkv_proj": [

1242 + "q_proj",

1243 + "k_proj",

1244 + "v_proj",

1245 + ],

1246 + "gate_up_proj": [

1247 + "gate_proj",

1248 + "up_proj",

1249 + ],

1250 + }

1251 +

1252 + # To ensure correct weight loading and mapping.

1253 + hf_to_vllm_mapper = WeightsMapper(

1254 + orig_to_new_prefix={

1255 + "lm_head.": "language_model.lm_head.",

1256 + "model.language_model.": "language_model.model.",

1257 + "model.visual.": "visual.",

1258 + })

1259 +

1260 + def __init__(self, *, vllm_config: VllmConfig, prefix: str = ""):

1261 + super().__init__()

1262 + config: Glm4vConfig = vllm_config.model_config.hf_config

1263 + quant_config = vllm_config.quant_config

1264 + multimodal_config = vllm_config.model_config.multimodal_config

1265 +

1266 + self.config = config

1267 + self.multimodal_config = multimodal_config

1268 +

1269 + self.visual = Glm4vVisionTransformer(

1270 + config.vision_config,

1271 + norm_eps=getattr(config, "rms_norm_eps", 1e-5),

1272 + quant_config=self._maybe_ignore_quant_config(quant_config),

1273 + prefix=maybe_prefix(prefix, "visual"),

1274 + )

1275 +

1276 + self.language_model = init_vllm_registered_model(

1277 + vllm_config=vllm_config,

1278 + prefix=maybe_prefix(prefix, ""),

1279 + architectures=["Glm4ForCausalLM"],

1280 + )

1281 +

1282 + self.make_empty_intermediate_tensors = (

1283 + self.language_model.make_empty_intermediate_tensors)

1284 +

1285 + def _maybe_ignore_quant_config(self, quant_config: QuantizationConfig):

1286 + # GPTQ configs do not have a list of ignored modules, however AutoGPTQ

1287 + # seems to avoid vision encoder sections for some models.

1288 + if isinstance(quant_config, (GPTQConfig, GPTQMarlinConfig)):

1289 + return None

1290 + return quant_config

1291 +

1292 + def _validate_and_reshape_mm_tensor(self, mm_input: object,

1293 + name: str) -> torch.Tensor:

1294 + if not isinstance(mm_input, (torch.Tensor, list)):

1295 + raise ValueError(

1296 + f"Incorrect type of {name}. Got type: {type(mm_input)}")

1297 + if isinstance(mm_input, torch.Tensor):

1298 + if mm_input.ndim == 2:

1299 + return mm_input

1300 + if mm_input.ndim != 3:

1301 + raise ValueError(f"{name} should be 2D or batched 3D tensor. "

1302 + f"Got ndim: {mm_input.ndim} "

1303 + f"(shape={mm_input.shape})")

1304 + return torch.concat(list(mm_input))

1305 + else:

1306 + return torch.concat(mm_input)

1307 +

1308 + def _parse_and_validate_image_input(

1309 + self, **kwargs: object) -> Optional[Glm4vImageInputs]:

1310 + pixel_values = kwargs.pop("pixel_values", None)

1311 + image_embeds = kwargs.pop("image_embeds", None)

1312 + image_grid_thw = kwargs.pop("image_grid_thw", None)

1313 +

1314 + if pixel_values is None and image_embeds is None:

1315 + return None

1316 +

1317 + if pixel_values is not None:

1318 + pixel_values = self._validate_and_reshape_mm_tensor(

1319 + pixel_values, "image pixel values")

1320 + image_grid_thw = self._validate_and_reshape_mm_tensor(

1321 + image_grid_thw, "image grid_thw")

1322 +

1323 + if not isinstance(pixel_values, (torch.Tensor, list)):

1324 + raise ValueError("Incorrect type of image pixel values. "

1325 + f"Got type: {type(pixel_values)}")

1326 +

1327 + return Glm4vImagePixelInputs(

1328 + type="pixel_values",

1329 + pixel_values=pixel_values,

1330 + image_grid_thw=image_grid_thw,

1331 + )

1332 +

1333 + if image_embeds is not None:

1334 + image_embeds = self._validate_and_reshape_mm_tensor(

1335 + image_embeds, "image embeds")

1336 + image_grid_thw = self._validate_and_reshape_mm_tensor(

1337 + image_grid_thw, "image grid_thw")

1338 +

1339 + if not isinstance(image_embeds, torch.Tensor):

1340 + raise ValueError("Incorrect type of image embeddings. "

1341 + f"Got type: {type(image_embeds)}")

1342 + return Glm4vImageEmbeddingInputs(

1343 + type="image_embeds",

1344 + image_embeds=image_embeds,

1345 + image_grid_thw=image_grid_thw,

1346 + )

1347 +

1348 + def _parse_and_validate_video_input(

1349 + self, **kwargs: object) -> Optional[Glm4vVideoInputs]:

1350 + pixel_values_videos = kwargs.pop("pixel_values_videos", None)

1351 + video_embeds = kwargs.pop("video_embeds", None)

1352 + video_grid_thw = kwargs.pop("video_grid_thw", None)

1353 + if pixel_values_videos is None and video_embeds is None:

1354 + return None

1355 + if pixel_values_videos is not None:

1356 + pixel_values_videos = self._validate_and_reshape_mm_tensor(

1357 + pixel_values_videos, "video pixel values")

1358 + video_grid_thw = self._validate_and_reshape_mm_tensor(

1359 + video_grid_thw, "video grid_thw")

1360 +

1361 + return Glm4vVideoPixelInputs(

1362 + type="pixel_values_videos",

1363 + # video_metadata=video_metadata,

1364 + pixel_values_videos=pixel_values_videos,

1365 + video_grid_thw=video_grid_thw,

1366 + )

1367 +

1368 + if video_embeds is not None:

1369 + video_embeds = self._validate_and_reshape_mm_tensor(

1370 + video_embeds, "video embeds")

1371 + video_grid_thw = self._validate_and_reshape_mm_tensor(

1372 + video_grid_thw, "video grid_thw")

1373 +

1374 + if not isinstance(video_embeds, torch.Tensor):

1375 + raise ValueError("Incorrect type of video embeddings. "

1376 + f"Got type: {type(video_embeds)}")

1377 + return Glm4vVideoEmbeddingInputs(

1378 + type="video_embeds",

1379 + video_embeds=video_embeds,

1380 + video_grid_thw=video_grid_thw,

1381 + )

1382 +

1383 + def _process_image_input(

1384 + self, image_input: Glm4vImageInputs) -> tuple[torch.Tensor, ...]:

1385 + grid_thw = image_input["image_grid_thw"]

1386 + assert grid_thw.ndim == 2

1387 +

1388 + if image_input["type"] == "image_embeds":

1389 + image_embeds = image_input["image_embeds"].type(self.visual.dtype)

1390 + else:

1391 + pixel_values = image_input["pixel_values"].type(self.visual.dtype)

1392 + image_embeds = self.visual(pixel_values, grid_thw=grid_thw)

1393 +

1394 + merge_size = self.visual.spatial_merge_size

1395 + sizes = grid_thw.prod(-1) // merge_size // merge_size

1396 + return image_embeds.split(sizes.tolist())

1397 +

1398 + def _process_video_input(

1399 + self, video_input: Glm4vVideoInputs) -> tuple[torch.Tensor, ...]:

1400 + grid_thw = video_input["video_grid_thw"]

1401 + assert grid_thw.ndim == 2

1402 +

1403 + device = self.visual.device

1404 + flat_grid_thw = torch.cat([

1405 + torch.tensor([[1, h, w]] * t, device=device)

1406 + for t, h, w in grid_thw

1407 + ])

1408 + if video_input["type"] == "video_embeds":

1409 + video_embeds = video_input["video_embeds"].type(self.visual.dtype)

1410 + else:

1411 + pixel_values_videos = video_input["pixel_values_videos"].type(

1412 + self.visual.dtype)

1413 + video_embeds = self.visual(pixel_values_videos,

1414 + grid_thw=flat_grid_thw)

1415 +

1416 + # Split concatenated embeddings for each video item.

1417 + merge_size = self.visual.spatial_merge_size

1418 + sizes = grid_thw.prod(-1) // merge_size // merge_size

1419 +

1420 + return video_embeds.split(sizes.tolist())

1421 +

1422 + def _parse_and_validate_multimodal_inputs(self, **kwargs: object) -> dict:

1423 + mm_input_by_modality = {}

1424 +

1425 + # Preserve the order of modalities if there are multiple of them

1426 + # from the order of kwargs.

1427 + for input_key in kwargs:

1428 + if (input_key in ("pixel_values", "image_embeds")

1429 + and "image" not in mm_input_by_modality):

1430 + mm_input_by_modality["image"] = (

1431 + self._parse_and_validate_image_input(**kwargs))

1432 + if (input_key in ("pixel_values_videos", "video_embeds")

1433 + and "video" not in mm_input_by_modality):

1434 + mm_input_by_modality["video"] = (

1435 + self._parse_and_validate_video_input(**kwargs))

1436 + return mm_input_by_modality

1437 +

1438 + def get_language_model(self) -> torch.nn.Module:

1439 + return self.language_model

1440 +

1441 + def get_multimodal_embeddings(

1442 + self, **kwargs: object) -> Optional[MultiModalEmbeddings]:

1443 + mm_input_by_modality = self._parse_and_validate_multimodal_inputs(

1444 + **kwargs)

1445 + if not mm_input_by_modality:

1446 + return None

1447 +

1448 + # The result multimodal_embeddings is tuple of tensors, with each

1449 + # tensor correspoending to a multimodal data item (image or video).

1450 + multimodal_embeddings: tuple[torch.Tensor, ...] = ()

1451 +

1452 + # NOTE: It is important to iterate over the keys in this dictionary

1453 + # to preserve the order of the modalities.

1454 + for modality in mm_input_by_modality:

1455 + multimodal_input = mm_input_by_modality[modality]

1456 + if modality == "image":

1457 + vision_embeddings = self._process_image_input(multimodal_input)

1458 + multimodal_embeddings += vision_embeddings

1459 + if modality == "video":

1460 + video_embeddings = self._process_video_input(multimodal_input)

1461 + multimodal_embeddings += video_embeddings

1462 + return multimodal_embeddings

1463 +

1464 + def get_input_embeddings(

1465 + self,

1466 + input_ids: torch.Tensor,

1467 + multimodal_embeddings: Optional[MultiModalEmbeddings] = None,

1468 + ) -> torch.Tensor:

1469 + inputs_embeds = self.language_model.get_input_embeddings(input_ids)

1470 + if (multimodal_embeddings is not None

1471 + and len(multimodal_embeddings) != 0

1472 + and all(embed.numel() > 0 for embed in multimodal_embeddings)):

1473 + inputs_embeds = merge_multimodal_embeddings(

1474 + input_ids,

1475 + inputs_embeds,

1476 + multimodal_embeddings,

1477 + [self.config.image_token_id, self.config.video_token_id],

1478 + )

1479 + return inputs_embeds

1480 +

1481 + def get_input_embeddings_v0(

1482 + self,

1483 + input_ids: torch.Tensor,

1484 + image_input: Optional[Glm4vImageInputs] = None,

1485 + video_input: Optional[Glm4vVideoInputs] = None,

1486 + ) -> torch.Tensor:

1487 + inputs_embeds = self.get_input_embeddings(input_ids)

1488 + if image_input is not None:

1489 + image_embeds = self._process_image_input(image_input)

1490 + inputs_embeds = merge_multimodal_embeddings(

1491 + input_ids,

1492 + inputs_embeds,

1493 + image_embeds,

1494 + placeholder_token_id=self.config.image_token_id,

1495 + )

1496 +

1497 + if video_input is not None:

1498 + video_embeds = self._process_video_input(video_input)

1499 + inputs_embeds = merge_multimodal_embeddings(

1500 + input_ids,

1501 + inputs_embeds,

1502 + video_embeds,

1503 + placeholder_token_id=self.config.video_token_id,

1504 + )

1505 + return inputs_embeds

1506 +

1507 + def forward(

1508 + self,

1509 + input_ids: torch.Tensor,

1510 + positions: torch.Tensor,

1511 + intermediate_tensors: Optional[IntermediateTensors] = None,

1512 + inputs_embeds: Optional[torch.Tensor] = None,

1513 + **kwargs: object,

1514 + ) -> Union[torch.Tensor, IntermediateTensors]:

1515 + """Run forward pass for GLM-4V.

1516 +

1517 + Args:

1518 + input_ids: Flattened (concatenated) input_ids corresponding to a

1519 + batch.

1520 + positions: Flattened (concatenated) position ids corresponding to a

1521 + batch.

1522 + **NOTE**: If mrope is enabled (default setting for GLM-4V

1523 + opensource models), the shape will be `(3, seq_len)`,

1524 + otherwise it will be `(seq_len,).

1525 + pixel_values: Pixel values to be fed to a model.

1526 + `None` if no images are passed.

1527 + image_grid_thw: Tensor `(n_images, 3)` of image 3D grid in LLM.

1528 + `None` if no images are passed.

1529 + pixel_values_videos: Pixel values of videos to be fed to a model.

1530 + `None` if no videos are passed.

1531 + video_grid_thw: Tensor `(n_videos, 3)` of video 3D grid in LLM.

1532 + `None` if no videos are passed.

1533 + second_per_grid_ts: Tensor `(num_videos)` of video time interval (

1534 + in seconds) for each grid along the temporal dimension in the

1535 + 3D position IDs. `None` if no videos are passed.

1536 + """

1537 + if intermediate_tensors is not None:

1538 + inputs_embeds = None

1539 +

1540 + # NOTE: In v1, inputs_embeds is always generated at model runner from

1541 + # `get_multimodal_embeddings` and `get_input_embeddings`, this

1542 + # condition is only for v0 compatibility.

1543 + elif inputs_embeds is None:

1544 + image_input = self._parse_and_validate_image_input(**kwargs)

1545 + video_input = self._parse_and_validate_video_input(**kwargs)

1546 +

1547 + if image_input is None and video_input is None:

1548 + inputs_embeds = None

1549 + else:

1550 + if uses_mrope(self.config):

1551 + assert positions.ndim == 2 and positions.size(0) == 3, (

1552 + "multimodal section rotary embedding requires "

1553 + f"(3, seq_len) positions, but got {positions.size()}")

1554 + inputs_embeds = self.get_input_embeddings_v0(

1555 + input_ids,

1556 + image_input=image_input,

1557 + video_input=video_input)

1558 + input_ids = None

1559 +

1560 + hidden_states = self.language_model.model(

1561 + input_ids=input_ids,

1562 + positions=positions,

1563 + intermediate_tensors=intermediate_tensors,

1564 + inputs_embeds=inputs_embeds,

1565 + )

1566 + return hidden_states

1567 +

1568 + def compute_logits(

1569 + self,

1570 + hidden_states: torch.Tensor,

1571 + sampling_metadata: SamplingMetadata,

1572 + ) -> Optional[torch.Tensor]:

1573 + return self.language_model.compute_logits(hidden_states,

1574 + sampling_metadata)

1575 +

1576 + def load_weights(self, weights: Iterable[tuple[str,

1577 + torch.Tensor]]) -> set[str]:

1578 + loader = AutoWeightsLoader(self)

1579 + return loader.load_weights(weights, mapper=self.hf_to_vllm_mapper)

1580 +

1581 + def get_mm_mapping(self) -> MultiModelKeys:

1582 + """

1583 + Get the module prefix in multimodal models

1584 + """

1585 + return MultiModelKeys.from_string_field(

1586 + language_model="language_model",

1587 + connector="visual.merger.",

1588 + tower_model="visual.",

1589 + )

vllm/model_executor/models/registry.py

@@ -190,6 +190,7 @@ _MULTIMODAL_MODELS = {

190 "FuyuForCausalLM": ("fuyu", "FuyuForCausalLM"),

191 "Gemma3ForConditionalGeneration": ("gemma3_mm", "Gemma3ForConditionalGeneration"), # noqa: E501

192 "GLM4VForCausalLM": ("glm4v", "GLM4VForCausalLM"),

193 + "Glm4vForConditionalGeneration": ("glm4_1v", "Glm4vForConditionalGeneration"), # noqa: E501

194 "GraniteSpeechForConditionalGeneration": ("granite_speech", "GraniteSpeechForConditionalGeneration"), # noqa: E501

195 "H2OVLChatModel": ("h2ovl", "H2OVLChatModel"),

196 "InternVLChatModel": ("internvl", "InternVLChatModel"),

vllm/multimodal/inputs.py

+5 -3

@@ -57,10 +57,12 @@ which are treated as image embeddings;

57 these are directly passed to the model without HF processing.

58 """

60 -VideoItem: TypeAlias = Union[HfVideoItem, "torch.Tensor"]

60 +VideoItem: TypeAlias = Union[HfVideoItem, "torch.Tensor",

61 + tuple[HfVideoItem, dict[str, Any]]]

62 """

62 -A `transformers.image_utils.VideoInput` representing a single video

63 -item, which can be passed to a HuggingFace `VideoProcessor`.

63 +A `transformers.video_utils.VideoInput` representing a single video item.

64 +This can be passed to a HuggingFace `VideoProcessor`

65 +with `transformers.video_utils.VideoMetadata`.

67 Alternatively, a 3-D tensor or batch of 2-D tensors,

68 which are treated as video embeddings;

vllm/multimodal/parse.py

+40 -2

@@ -224,8 +224,14 @@ class ImageEmbeddingItems(EmbeddingItems):

224

225 class VideoProcessorItems(ProcessorBatchItems[HfVideoItem]):

226

227 - def __init__(self, data: Sequence[HfVideoItem]) -> None:

227 + def __init__(

228 + self,

229 + data: Sequence[HfVideoItem],

230 + metadata: Optional[Union[dict[str, Any],

231 + list[Optional[dict[str, Any]]]]] = None,

232 + ) -> None:

233 super().__init__(data, "video")

234 + self.metadata = metadata

235

236 def get_num_frames(self, item_idx: int) -> int:

237 return len(self.get(item_idx))

@@ -320,6 +326,7 @@ class MultiModalDataParser:

326 *,

327 target_sr: Optional[float] = None,

328 audio_resample_method: Literal["librosa", "scipy"] = "librosa",

329 + video_needs_metadata: bool = False,

330 ) -> None:

331 super().__init__()

332

@@ -327,6 +334,7 @@ class MultiModalDataParser:

334 target_sr=target_sr,

335 method=audio_resample_method,

336 )

337 + self.video_needs_metadata = video_needs_metadata

338

339 def _is_embeddings(

340 self, data: object

@@ -361,6 +369,21 @@ class MultiModalDataParser:

369

370 assert_never(audio)

371

372 + def _get_video_with_metadata(

373 + self,

374 + video: VideoItem,

375 + ) -> tuple[np.ndarray, Optional[dict[str, Any]]]:

376 + if isinstance(video, tuple):

377 + return video

378 + if isinstance(video, list):

379 + return np.array(video), None

380 + if isinstance(video, np.ndarray):

381 + return video, None

382 + if isinstance(video, torch.Tensor):

383 + return video.numpy(), None

384 +

385 + assert_never(video)

386 +

387 def _parse_audio_data(

388 self,

389 data: ModalityData[AudioItem],

@@ -433,10 +456,25 @@ class MultiModalDataParser:

456 data_items = [data]

457 elif isinstance(data, (np.ndarray, torch.Tensor)):

458 data_items = [elem for elem in data]

459 + elif isinstance(data, tuple) and len(data) == 2:

460 + data_items = [data]

461 else:

462 data_items = data

463

439 - return VideoProcessorItems(data_items)

464 + new_videos = list[tuple[np.ndarray, Optional[dict[str, Any]]]]()

465 + metadata_lst: list[Optional[dict[str, Any]]] = []

466 + for data_item in data_items:

467 + video, metadata = self._get_video_with_metadata(data_item)

468 + if self.video_needs_metadata:

469 + new_videos.append((video, metadata))

470 + metadata_lst.append(metadata)

471 + else:

472 + new_videos.append(video)

473 +

474 + if not self.video_needs_metadata:

475 + metadata = None

476 +

477 + return VideoProcessorItems(new_videos, metadata=metadata_lst)

478

479 def _get_subparsers(self) -> Mapping[str, ModalityDataParser]:

480 return {

vllm/multimodal/video.py

+21 -6

@@ -24,6 +24,7 @@ def resize_video(frames: npt.NDArray, size: tuple[int, int]) -> npt.NDArray:

24 dtype=frames.dtype)

25 # lazy import cv2 to avoid bothering users who only use text models

26 import cv2

27 +

28 for i, frame in enumerate(frames):

29 resized_frame = cv2.resize(frame, (new_width, new_height))

30 resized_frames[i] = resized_frame

@@ -92,14 +93,16 @@ class OpenCVVideoBackend(VideoLoader):

93 continue

94 if not vr.isBackendBuiltIn(backend):

95 _, abi, api = vr.getStreamBufferedBackendPluginVersion(backend)

95 - if (abi < 1 or (abi == 1 and api < 2)):

96 + if abi < 1 or (abi == 1 and api < 2):

97 continue

98 api_pref = backend

99 break

100 return api_pref

101

102 @classmethod

102 - def load_bytes(cls, data: bytes, num_frames: int = -1) -> npt.NDArray:

103 + def load_bytes(cls,

104 + data: bytes,

105 + num_frames: int = -1) -> tuple[npt.NDArray, dict]:

106 import cv2

107

108 backend = cls().get_cv2_video_api()

@@ -108,6 +111,9 @@ class OpenCVVideoBackend(VideoLoader):

111 raise ValueError("Could not open video stream")

112

113 total_frames_num = int(cap.get(cv2.CAP_PROP_FRAME_COUNT))

114 + original_fps = cap.get(cv2.CAP_PROP_FPS)

115 + duration = total_frames_num / original_fps if original_fps > 0 else 0

116 +

117 full_read = num_frames == -1 or total_frames_num < num_frames

118 if full_read:

119 num_frames = total_frames_num

@@ -125,18 +131,27 @@ class OpenCVVideoBackend(VideoLoader):

131

132 i = 0

133 for idx in range(total_frames_num):

128 - ok = cap.grab() # next img

134 + ok = cap.grab()

135 if not ok:

136 break

131 - if idx in frame_idx: # only decompress needed

137 + if idx in frame_idx:

138 ret, frame = cap.retrieve()

139 if ret:

140 frames[i] = cv2.cvtColor(frame, cv2.COLOR_BGR2RGB)

141 i += 1

136 - # we expect all frames loaded

142 +

143 assert i == num_frames, (f"Expected reading {num_frames} frames, "

144 f"but only loaded {i} frames from video.")

139 - return frames

145 +

146 + # Use transformers transformers.video_utils.VideoMetadata format

147 + metadata = {

148 + "total_num_frames": total_frames_num,

149 + "fps": original_fps,

150 + "duration": duration,

151 + "video_backend": "opencv"

152 + }

153 +

154 + return frames, metadata

155

156

157 class VideoMediaIO(MediaIO[npt.NDArray]):