A high-throughput and memory-efficient inference and serving engine for LLMs
Mirrored from https://github.com/tduncan-lum/Nvidia-Optimization-Stack---VLLM.git
Synced Nov 15, 19:45 UTC
[Bugfix] Fix prompt format of GLM4V (#14539)
Signed-off-by: DarkLight1337 <tlleungac@connect.ust.hk>
Cyrus Leung committed
Mar 13, 2025 at 19:37 UTC
f53a0586b9c88a78167157296555b7664c398055
7 files changed
+43
-19
tests/models/decoder_only/vision_language/test_models.py
+11
-3
@@ -254,13 +254,21 @@ VLM_TEST_SETTINGS = {
254
"glm4v": VLMTestInfo(
255
models=["THUDM/glm-4v-9b"],
256
test_type=VLMTestType.IMAGE,
257
- prompt_formatter=identity,
258
- img_idx_to_prompt=lambda idx: "",
257
+ prompt_formatter=lambda img_prompt: f"<|user|>\n{img_prompt}<|assistant|>", # noqa: E501
258
+ single_image_prompts=IMAGE_ASSETS.prompts({
259
+ "stop_sign": "<|begin_of_image|><|endoftext|><|end_of_image|>What's the content in the center of the image?", # noqa: E501
260
+ "cherry_blossom": "<|begin_of_image|><|endoftext|><|end_of_image|>What is the season?", # noqa: E501
261
+ }),
262
max_model_len=2048,
263
max_num_seqs=2,
264
dtype="bfloat16",
265
get_stop_token_ids=lambda tok: [151329, 151336, 151338],
263
- patch_hf_runner=model_utils.glm_patch_hf_runner,
266
+ patch_hf_runner=model_utils.glm4v_patch_hf_runner,
267
+ # The image embeddings match with HF but the outputs of the language
268
+ # decoder are only consistent up to 2 decimal places.
269
+ # So, we need to reduce the number of tokens for the test to pass.
270
+ max_tokens=8,
271
+ num_logprobs=10,
272
marks=[large_gpu_mark(min_gb=32)],
273
),
274
"h2ovl": VLMTestInfo(
tests/models/decoder_only/vision_language/vlm_utils/core.py
+3
-1
@@ -61,7 +61,9 @@ def run_test(
61
# if we run HF first, the cuda initialization will be done and it
62
# will hurt multiprocessing backend with fork method (the default method).
63
64
- vllm_runner_kwargs_: dict[str, Any] = {}
64
+ vllm_runner_kwargs_: dict[str, Any] = {
65
+ "disable_mm_preprocessor_cache": True,
66
+ }
67
if model_info.tokenizer:
68
vllm_runner_kwargs_["tokenizer"] = model_info.tokenizer
69
if model_info.tokenizer_mode:
tests/models/decoder_only/vision_language/vlm_utils/model_utils.py
+13
-5
@@ -316,8 +316,8 @@ def gemma3_patch_hf_runner(hf_model: HfRunner) -> HfRunner:
316
return hf_model
317
318
319
-def glm_patch_hf_runner(hf_model: HfRunner) -> HfRunner:
320
- """Patches and returns an instance of the HfRunner to use for GLM4."""
319
+def glm4v_patch_hf_runner(hf_model: HfRunner) -> HfRunner:
320
+ """Patches and returns an instance of the HfRunner to use for GLM4V."""
321
hf_processor = hf_model.processor
322
patch_padding_side(hf_processor)
323
@@ -325,12 +325,20 @@ def glm_patch_hf_runner(hf_model: HfRunner) -> HfRunner:
325
if images is None:
326
return hf_processor(*args, **kwargs)
327
328
+ images = [images] if isinstance(images, Image) else images
329
+
330
+ contents = re.findall(
331
+ r"<\|begin_of_image\|><\|endoftext\|><\|end_of_image\|>(.*?)<\|assistant\|>",
332
+ text,
333
+ )
334
+ assert len(contents) == len(images)
335
+
336
return hf_processor.apply_chat_template(
337
[{
338
"role": "user",
331
- "image": images,
332
- "content": text
333
- }],
339
+ "image": image,
340
+ "content": content
341
+ } for image, content in zip(images, contents)],
342
add_generation_prompt=True,
343
tokenize=True,
344
return_dict=True,
vllm/config.py
+8
-4
@@ -286,14 +286,18 @@ class ModelConfig:
286
if rope_scaling is not None:
287
hf_override: dict[str, Any] = {"rope_scaling": rope_scaling}
288
hf_overrides_kw.update(hf_override)
289
- msg = ("`--rope-scaling` will be removed in a future release. "
290
- f"'Please instead use `--hf-overrides '{hf_override!r}'`")
289
+ hf_overrides_str = json.dumps(hf_overrides)
290
+ msg = (
291
+ "`--rope-scaling` will be removed in a future release. "
292
+ f"'Please instead use `--hf-overrides '{hf_overrides_str}'`")
293
warnings.warn(DeprecationWarning(msg), stacklevel=2)
294
if rope_theta is not None:
295
hf_override = {"rope_theta": rope_theta}
296
hf_overrides_kw.update(hf_override)
295
- msg = ("`--rope-theta` will be removed in a future release. "
296
- f"'Please instead use `--hf-overrides '{hf_override!r}'`")
297
+ hf_overrides_str = json.dumps(hf_overrides)
298
+ msg = (
299
+ "`--rope-theta` will be removed in a future release. "
300
+ f"'Please instead use `--hf-overrides '{hf_overrides_str}'`")
301
warnings.warn(DeprecationWarning(msg), stacklevel=2)
302
303
self.maybe_pull_model_tokenizer_for_s3(model, tokenizer)
vllm/entrypoints/chat_utils.py
+4
-3
@@ -403,7 +403,9 @@ class BaseMultiModalItemTracker(ABC, Generic[_T]):
403
hf_config = self._model_config.hf_config
404
model_type = hf_config.model_type
405
406
- if modality in ["image", "image_embeds"]:
406
+ if modality in ("image", "image_embeds"):
407
+ if model_type == "chatglm":
408
+ return "<|begin_of_image|><|endoftext|><|end_of_image|>"
409
if model_type == "phi3_v":
410
# Workaround since this token is not defined in the tokenizer
411
return f"<|image_{current_count}|>"
@@ -411,8 +413,7 @@ class BaseMultiModalItemTracker(ABC, Generic[_T]):
413
return "<|endoftext10|>" # 200010 (see vocab.json in hf model)
414
if model_type in ("minicpmo", "minicpmv"):
415
return "(<image>./</image>)"
414
- if model_type in ("blip-2", "chatglm", "fuyu", "paligemma",
415
- "pixtral"):
416
+ if model_type in ("blip-2", "fuyu", "paligemma", "pixtral"):
417
# These models do not use image tokens in the prompt
418
return None
419
if model_type == "qwen":
vllm/model_executor/models/chatglm.py
+2
-1
@@ -2,6 +2,7 @@
2
# Adapted from
3
# https://github.com/THUDM/ChatGLM2-6B
4
"""Inference-only ChatGLM model compatible with THUDM weights."""
5
+import json
6
from typing import Iterable, Optional, Set, Tuple, Union
7
8
import torch
@@ -463,7 +464,7 @@ class ChatGLMForCausalLM(ChatGLMBaseModel, SupportsLoRA, SupportsPP):
464
"The configuration of this model indicates that it supports "
465
"vision inputs, but you instantiated the text-only version "
466
"of this model. Please use the vision model by setting "
466
- f"`--hf-overrides {hf_overrides!r}`")
467
+ f"`--hf-overrides '{json.dumps(hf_overrides)}'`")
468
469
super().__init__(vllm_config=vllm_config, prefix=prefix)
470
vllm/model_executor/models/qwen.py
+2
-2
@@ -5,7 +5,7 @@
5
# Copyright (c) Alibaba Cloud.
6
# LICENSE: https://huggingface.co/Qwen/Qwen-7B/blob/main/LICENSE
7
"""Inference-only QWen model compatible with HuggingFace weights."""
8
-
8
+import json
9
from typing import Any, Dict, Iterable, Optional, Set, Tuple, Union
10
11
import torch
@@ -354,7 +354,7 @@ class QWenLMHeadModel(QWenBaseModel, SupportsPP, SupportsLoRA):
354
"The configuration of this model indicates that it supports "
355
"vision inputs, but you instantiated the text-only version "
356
"of this model. Please use the vision model by setting "
357
- f"`--hf-overrides {hf_overrides!r}`")
357
+ f"`--hf-overrides '{json.dumps(hf_overrides)}'`")
358
359
super().__init__(vllm_config=vllm_config, prefix=prefix)
360