A high-throughput and memory-efficient inference and serving engine for LLMs
Mirrored from https://github.com/tduncan-lum/Nvidia-Optimization-Stack---VLLM.git
Synced Nov 14, 05:30 UTC
Fix GLM-4.5V-FP8 numerical issue (#22949)
Signed-off-by: qizixi <qizixi@meta.com> Co-authored-by: Cyrus Leung <tlleungac@connect.ust.hk>
qizixi committed
Aug 19, 2025 at 16:56 UTC
4efd43e9b45cec5a77c92bce3721774e8e51f533
3 files changed
+154
-5
examples/offline_inference/vision_language.py
+78
-2
@@ -333,6 +333,80 @@ def run_glm4_1v(questions: list[str], modality: str) -> ModelRequestData:
333
)
334
335
336
+# GLM-4.5V
337
+def run_glm4_5v(questions: list[str], modality: str) -> ModelRequestData:
338
+ model_name = "zai-org/GLM-4.5V"
339
+
340
+ engine_args = EngineArgs(
341
+ model=model_name,
342
+ max_model_len=4096,
343
+ max_num_seqs=2,
344
+ mm_processor_kwargs={
345
+ "size": {"shortest_edge": 12544, "longest_edge": 47040000},
346
+ "fps": 1,
347
+ },
348
+ limit_mm_per_prompt={modality: 1},
349
+ enforce_eager=True,
350
+ tensor_parallel_size=4,
351
+ )
352
+
353
+ if modality == "image":
354
+ placeholder = "<|begin_of_image|><|image|><|end_of_image|>"
355
+ elif modality == "video":
356
+ placeholder = "<|begin_of_video|><|video|><|end_of_video|>"
357
+
358
+ prompts = [
359
+ (
360
+ "[gMASK]<sop><|system|>\nYou are a helpful assistant.<|user|>\n"
361
+ f"{placeholder}"
362
+ f"{question}<|assistant|>assistant\n"
363
+ )
364
+ for question in questions
365
+ ]
366
+
367
+ return ModelRequestData(
368
+ engine_args=engine_args,
369
+ prompts=prompts,
370
+ )
371
+
372
+
373
+# GLM-4.5V-FP8
374
+def run_glm4_5v_fp8(questions: list[str], modality: str) -> ModelRequestData:
375
+ model_name = "zai-org/GLM-4.5V-FP8"
376
+
377
+ engine_args = EngineArgs(
378
+ model=model_name,
379
+ max_model_len=4096,
380
+ max_num_seqs=2,
381
+ mm_processor_kwargs={
382
+ "size": {"shortest_edge": 12544, "longest_edge": 47040000},
383
+ "fps": 1,
384
+ },
385
+ limit_mm_per_prompt={modality: 1},
386
+ enforce_eager=True,
387
+ tensor_parallel_size=4,
388
+ )
389
+
390
+ if modality == "image":
391
+ placeholder = "<|begin_of_image|><|image|><|end_of_image|>"
392
+ elif modality == "video":
393
+ placeholder = "<|begin_of_video|><|video|><|end_of_video|>"
394
+
395
+ prompts = [
396
+ (
397
+ "[gMASK]<sop><|system|>\nYou are a helpful assistant.<|user|>\n"
398
+ f"{placeholder}"
399
+ f"{question}<|assistant|>assistant\n"
400
+ )
401
+ for question in questions
402
+ ]
403
+
404
+ return ModelRequestData(
405
+ engine_args=engine_args,
406
+ prompts=prompts,
407
+ )
408
+
409
+
410
# H2OVL-Mississippi
411
def run_h2ovl(questions: list[str], modality: str) -> ModelRequestData:
412
assert modality == "image"
@@ -383,8 +457,8 @@ def run_hyperclovax_seed_vision(
457
for question in questions:
458
if modality == "image":
459
"""
386
- ocr: List the words in the image in raster order.
387
- Even if the word order feels unnatural for reading,
460
+ ocr: List the words in the image in raster order.
461
+ Even if the word order feels unnatural for reading,
462
the model will handle it as long as it follows raster order.
463
e.g. "Naver, CLOVA, bigshane"
464
lens_keywords: List the entity names in the image.
@@ -1448,6 +1522,8 @@ model_example_map = {
1522
"gemma3n": run_gemma3n,
1523
"glm4v": run_glm4v,
1524
"glm4_1v": run_glm4_1v,
1525
+ "glm4_5v": run_glm4_5v,
1526
+ "glm4_5v_fp8": run_glm4_5v_fp8,
1527
"h2ovl_chat": run_h2ovl,
1528
"hyperclovax_seed_vision": run_hyperclovax_seed_vision,
1529
"idefics3": run_idefics3,
examples/offline_inference/vision_language_multi_image.py
+72
@@ -1064,6 +1064,76 @@ def load_tarsier2(question: str, image_urls: list[str]) -> ModelRequestData:
1064
)
1065
1066
1067
+# GLM-4.5V
1068
+def load_glm4_5v(question: str, image_urls: list[str]) -> ModelRequestData:
1069
+ model_name = "zai-org/GLM-4.5V"
1070
+
1071
+ engine_args = EngineArgs(
1072
+ model=model_name,
1073
+ max_model_len=32768,
1074
+ max_num_seqs=2,
1075
+ limit_mm_per_prompt={"image": len(image_urls)},
1076
+ enforce_eager=True,
1077
+ tensor_parallel_size=4,
1078
+ )
1079
+ placeholders = [{"type": "image", "image": url} for url in image_urls]
1080
+ messages = [
1081
+ {
1082
+ "role": "user",
1083
+ "content": [
1084
+ *placeholders,
1085
+ {"type": "text", "text": question},
1086
+ ],
1087
+ }
1088
+ ]
1089
+ processor = AutoProcessor.from_pretrained(model_name)
1090
+ prompt = processor.apply_chat_template(
1091
+ messages, tokenize=False, add_generation_prompt=True
1092
+ )
1093
+ image_data = [fetch_image(url) for url in image_urls]
1094
+
1095
+ return ModelRequestData(
1096
+ engine_args=engine_args,
1097
+ prompt=prompt,
1098
+ image_data=image_data,
1099
+ )
1100
+
1101
+
1102
+# GLM-4.5V-FP8
1103
+def load_glm4_5v_fp8(question: str, image_urls: list[str]) -> ModelRequestData:
1104
+ model_name = "zai-org/GLM-4.5V-FP8"
1105
+
1106
+ engine_args = EngineArgs(
1107
+ model=model_name,
1108
+ max_model_len=32768,
1109
+ max_num_seqs=2,
1110
+ limit_mm_per_prompt={"image": len(image_urls)},
1111
+ enforce_eager=True,
1112
+ tensor_parallel_size=4,
1113
+ )
1114
+ placeholders = [{"type": "image", "image": url} for url in image_urls]
1115
+ messages = [
1116
+ {
1117
+ "role": "user",
1118
+ "content": [
1119
+ *placeholders,
1120
+ {"type": "text", "text": question},
1121
+ ],
1122
+ }
1123
+ ]
1124
+ processor = AutoProcessor.from_pretrained(model_name)
1125
+ prompt = processor.apply_chat_template(
1126
+ messages, tokenize=False, add_generation_prompt=True
1127
+ )
1128
+ image_data = [fetch_image(url) for url in image_urls]
1129
+
1130
+ return ModelRequestData(
1131
+ engine_args=engine_args,
1132
+ prompt=prompt,
1133
+ image_data=image_data,
1134
+ )
1135
+
1136
+
1137
model_example_map = {
1138
"aria": load_aria,
1139
"aya_vision": load_aya_vision,
@@ -1096,6 +1166,8 @@ model_example_map = {
1166
"step3": load_step3,
1167
"tarsier": load_tarsier,
1168
"tarsier2": load_tarsier2,
1169
+ "glm4_5v": load_glm4_5v,
1170
+ "glm4_5v_fp8": load_glm4_5v_fp8,
1171
}
1172
1173
vllm/model_executor/models/glm4_1v.py
+4
-3
@@ -126,7 +126,7 @@ class Glm4vVideoPixelInputs(TensorSchema):
126
- ctpp: Number of channels * temporal_patch_size *
127
patch_size * patch_size
128
- f: Number of frames
129
- - g: Grid dimensions (3 for grid_t which is usually 1 for processed
129
+ - g: Grid dimensions (3 for grid_t which is usually 1 for processed
130
video, grid_h, grid_w)
131
"""
132
type: Literal["pixel_values_videos"] = "pixel_values_videos"
@@ -141,7 +141,7 @@ class Glm4vVideoEmbeddingInputs(TensorSchema):
141
- p: Number of video patches across all frames
142
- h: Hidden size (must match language model backbone)
143
- f: Number of frames
144
- - g: Grid dimensions (3 for grid_t which is usually 1 for processed
144
+ - g: Grid dimensions (3 for grid_t which is usually 1 for processed
145
video, grid_h, grid_w)
146
"""
147
type: Literal["video_embeds"] = "video_embeds"
@@ -234,7 +234,8 @@ class Glm4vVisionAttention(nn.Module):
234
total_num_kv_heads=num_heads,
235
bias=False,
236
quant_config=quant_config,
237
- prefix=f"{prefix}.qkv",
237
+ # Change qkv prefix to align with GLM-4.5V-FP8 quantization config
238
+ prefix=f"{prefix}.qkv_proj" if quant_config else f"{prefix}.qkv",
239
)
240
self.proj = RowParallelLinear(
241
input_size=projection_size,