A high-throughput and memory-efficient inference and serving engine for LLMs
Mirrored from https://github.com/tduncan-lum/Nvidia-Optimization-Stack---VLLM.git
Synced Nov 12, 01:00 UTC
[ROCm][CI][Bugfix] Fixing the `Multi-Modal Models Test (Extended) 1` group (#30013)
Signed-off-by: Andreas Karatzas <akaratza@amd.com>
Andreas Karatzas committed
Dec 4, 2025 at 05:00 UTC
e96a6a6dca930d00902852ea6937a214a584b89b
10 files changed
+139
-9
.buildkite/test-amd.yaml
+4
-2
@@ -987,7 +987,8 @@ steps:
987
commands:
988
- pytest -s -v test_lm_eval_correctness.py --config-list-file=configs/models-mm-small.txt --tp-size=1
989
990
-- label: Multi-Modal Models Test (Extended) 1
990
+- label: Multi-Modal Models Test (Extended) 1 # 60min
991
+ timeout_in_minutes: 120
992
mirror_hardwares: [amdexperimental]
993
agent_pool: mi325_1
994
# grade: Blocking
@@ -1011,7 +1012,8 @@ steps:
1012
- pip install git+https://github.com/TIGER-AI-Lab/Mantis.git
1013
- pytest -v -s models/multimodal/generation/test_common.py -m 'split(group=0) and not core_model'
1014
1014
-- label: Multi-Modal Models Test (Extended) 3
1015
+- label: Multi-Modal Models Test (Extended) 3 # 75min
1016
+ timeout_in_minutes: 150
1017
mirror_hardwares: [amdexperimental]
1018
agent_pool: mi325_1
1019
# grade: Blocking
tests/models/multimodal/generation/conftest.py
+16
@@ -2,6 +2,8 @@
2
# SPDX-FileCopyrightText: Copyright contributors to the vLLM project
3
"""Pytest configuration for vLLM tests."""
4
5
+import warnings
6
+
7
import torch
8
9
from vllm.platforms import current_platform
@@ -14,6 +16,20 @@ def pytest_configure(config):
16
if not current_platform.is_rocm():
17
return
18
19
+ skip_patterns = ["test_granite_speech.py"]
20
+ if any(pattern in str(arg) for arg in config.args for pattern in skip_patterns):
21
+ # Skip disabling SDP for Granite Speech tests on ROCm
22
+ return
23
+
24
+ # Disable Flash/MemEfficient SDP on ROCm to avoid HF Transformers
25
+ # accuracy issues
26
+ # TODO: Remove once ROCm SDP accuracy issues are resolved on HuggingFace
27
torch.backends.cuda.enable_flash_sdp(False)
28
torch.backends.cuda.enable_mem_efficient_sdp(False)
29
torch.backends.cuda.enable_math_sdp(True)
30
+ warnings.warn(
31
+ "ROCm: Disabled flash_sdp and mem_efficient_sdp, enabled math_sdp "
32
+ "to avoid HuggingFace Transformers accuracy issues",
33
+ UserWarning,
34
+ stacklevel=1,
35
+ )
tests/models/multimodal/generation/test_common.py
+10
-2
@@ -403,12 +403,13 @@ VLM_TEST_SETTINGS = {
403
# So, we need to reduce the number of tokens for the test to pass.
404
max_tokens=8,
405
num_logprobs=10,
406
+ auto_cls=AutoModelForCausalLM,
407
marks=[large_gpu_mark(min_gb=32)],
408
),
409
"glm4_1v": VLMTestInfo(
410
models=["zai-org/GLM-4.1V-9B-Thinking"],
411
test_type=(VLMTestType.IMAGE, VLMTestType.MULTI_IMAGE),
411
- prompt_formatter=lambda img_prompt: f"<|user|>\n{img_prompt}<|assistant|>",
412
+ prompt_formatter=lambda img_prompt: f"[gMASK]<|user|>\n{img_prompt}<|assistant|>\n", # noqa: E501
413
img_idx_to_prompt=lambda idx: "<|begin_of_image|><|image|><|end_of_image|>",
414
video_idx_to_prompt=lambda idx: "<|begin_of_video|><|video|><|end_of_video|>",
415
max_model_len=2048,
@@ -423,6 +424,7 @@ VLM_TEST_SETTINGS = {
424
models=["zai-org/GLM-4.1V-9B-Thinking"],
425
# GLM4.1V require include video metadata for input
426
test_type=VLMTestType.CUSTOM_INPUTS,
427
+ prompt_formatter=lambda vid_prompt: f"[gMASK]<|user|>\n{vid_prompt}<|assistant|>\n", # noqa: E501
428
max_model_len=4096,
429
max_num_seqs=2,
430
auto_cls=AutoModelForImageTextToText,
@@ -737,7 +739,13 @@ VLM_TEST_SETTINGS = {
739
max_model_len=8192,
740
max_num_seqs=2,
741
auto_cls=AutoModelForImageTextToText,
740
- marks=[large_gpu_mark(min_gb=48)],
742
+ marks=[
743
+ large_gpu_mark(min_gb=48),
744
+ pytest.mark.skipif(
745
+ current_platform.is_rocm(),
746
+ reason="Model produces a vector of <UNK> output in HF on ROCm",
747
+ ),
748
+ ],
749
),
750
"qwen_vl": VLMTestInfo(
751
models=["Qwen/Qwen-VL"],
tests/models/multimodal/generation/test_granite_speech.py
+13
-2
@@ -8,6 +8,7 @@ from transformers import AutoModelForSpeechSeq2Seq
8
9
from vllm.logprobs import SampleLogprobs
10
from vllm.lora.request import LoRARequest
11
+from vllm.platforms import current_platform
12
13
from ....conftest import AudioTestAssets, HfRunner, PromptAudioInput, VllmRunner
14
from ...registry import HF_EXAMPLE_MODELS
@@ -34,6 +35,12 @@ audio_lora_path = MODEL_NAME
35
models = [MODEL_NAME]
36
37
38
+@pytest.fixture(autouse=True)
39
+def set_attention_backend_for_rocm(monkeypatch):
40
+ if current_platform.is_rocm():
41
+ monkeypatch.setenv("VLLM_ATTENTION_BACKEND", "TRITON_ATTN")
42
+
43
+
44
def run_test(
45
hf_runner: type[HfRunner],
46
vllm_runner: type[VllmRunner],
@@ -111,8 +118,12 @@ def run_test(
118
119
120
@pytest.mark.parametrize("model", models)
114
-@pytest.mark.parametrize("dtype", ["bfloat16"])
115
-@pytest.mark.parametrize("max_model_len", [2048])
121
+@pytest.mark.parametrize(
122
+ "dtype", ["float16"] if current_platform.is_rocm() else ["bfloat16"]
123
+)
124
+@pytest.mark.parametrize(
125
+ "max_model_len", [512] if current_platform.is_rocm() else [2048]
126
+)
127
@pytest.mark.parametrize("max_tokens", [128])
128
@pytest.mark.parametrize("num_logprobs", [10])
129
def test_models(
tests/models/multimodal/generation/test_pixtral.py
+10
@@ -15,6 +15,7 @@ from transformers import AutoProcessor
15
from vllm import SamplingParams, TextPrompt, TokensPrompt
16
from vllm.logprobs import Logprob, SampleLogprobs
17
from vllm.multimodal import MultiModalDataBuiltins
18
+from vllm.platforms import current_platform
19
20
from ....utils import VLLM_PATH, large_gpu_test
21
from ...utils import check_logprobs_close
@@ -165,6 +166,15 @@ def load_outputs_w_logprobs(filename: "StrPath") -> OutputsLogprobs:
166
def test_chat(
167
vllm_runner, max_model_len: int, model: str, dtype: str, local_asset_server
168
) -> None:
169
+ if (
170
+ model == MISTRAL_SMALL_3_1_ID
171
+ and max_model_len == 65536
172
+ and current_platform.is_rocm()
173
+ ):
174
+ pytest.skip(
175
+ "OOM on ROCm: 24B model with 65536 context length exceeds GPU memory"
176
+ )
177
+
178
EXPECTED_CHAT_LOGPROBS = load_outputs_w_logprobs(FIXTURE_LOGPROBS_CHAT[model])
179
with vllm_runner(
180
model,
tests/models/multimodal/generation/vlm_utils/custom_inputs.py
+1
-1
@@ -140,7 +140,7 @@ def video_with_metadata_glm4_1v():
140
metadata = VIDEO_ASSETS[0].metadata
141
question = "Describe the video."
142
video_prompt = "<|begin_of_video|><|video|><|end_of_video|>"
143
- formatted_prompt = f"<|user|>\n{video_prompt}{question}<|assistant|>\n"
143
+ formatted_prompt = f"[gMASK]<|user|>\n{video_prompt}{question}<|assistant|>\n"
144
145
scales = [0.1, 0.2, 0.25]
146
video_input = [
tests/models/multimodal/generation/vlm_utils/model_utils.py
+44
-1
@@ -25,6 +25,7 @@ from transformers import (
25
from transformers.video_utils import VideoMetadata
26
27
from vllm.logprobs import SampleLogprobs
28
+from vllm.platforms import current_platform
29
from vllm.utils.collection_utils import is_list_of
30
31
from .....conftest import HfRunner, ImageAsset, ImageTestAssets
@@ -366,6 +367,40 @@ def gemma3_vllm_to_hf_output(vllm_output: RunnerOutput, model: str) -> RunnerOut
367
368
def glm4v_patch_hf_runner(hf_model: HfRunner) -> HfRunner:
369
"""Patches and returns an instance of the HfRunner to use for GLM4V."""
370
+ if current_platform.is_rocm():
371
+ import types
372
+
373
+ config = hf_model.model.config
374
+ if hasattr(config, "num_layers") and not hasattr(config, "num_hidden_layers"):
375
+ config.num_hidden_layers = config.num_layers
376
+ config.output_hidden_states = True
377
+
378
+ def patched_prepare_cache(
379
+ self, generation_config, model_kwargs, *args, **kwargs
380
+ ):
381
+ model_kwargs["past_key_values"] = None
382
+ model_kwargs["use_cache"] = False
383
+ return model_kwargs
384
+
385
+ hf_model.model._prepare_cache_for_generation = types.MethodType(
386
+ patched_prepare_cache, hf_model.model
387
+ )
388
+ original_generate = hf_model.model.generate
389
+
390
+ def patched_generate(*args, **kwargs):
391
+ kwargs["output_hidden_states"] = True
392
+ kwargs["return_dict_in_generate"] = True
393
+ return original_generate(*args, **kwargs)
394
+
395
+ hf_model.model.generate = patched_generate
396
+ original_forward = hf_model.model.forward
397
+
398
+ def patched_forward(*args, **kwargs):
399
+ kwargs["output_hidden_states"] = True
400
+ return original_forward(*args, **kwargs)
401
+
402
+ hf_model.model.forward = patched_forward
403
+
404
hf_processor = hf_model.processor
405
406
def processor(*args, text="", images=None, **kwargs):
@@ -406,7 +441,15 @@ def glm4_1v_patch_hf_runner(hf_model: HfRunner) -> HfRunner:
441
if videos is not None and is_list_of(videos, tuple):
442
# If videos is a list of tuples, we assume each tuple contains
443
# (video_array, metadata) as in the case of GLM4.1V.
409
- video_metadata = [[VideoMetadata(**video[1])] for video in videos]
444
+ # Filter out 'do_sample_frames' as it's not a valid VideoMetadata arg
445
+ video_metadata = [
446
+ [
447
+ VideoMetadata(
448
+ **{k: v for k, v in video[1].items() if k != "do_sample_frames"}
449
+ )
450
+ ]
451
+ for video in videos
452
+ ]
453
videos = [[video[0]] for video in videos]
454
else:
455
video_metadata = None
tests/models/multimodal/pooling/conftest.py
new
+24
@@ -0,0 +1,24 @@
1
+# SPDX-License-Identifier: Apache-2.0
2
+# SPDX-FileCopyrightText: Copyright contributors to the vLLM project
3
+"""Pytest configuration for vLLM pooling tests."""
4
+
5
+import os
6
+import warnings
7
+
8
+from vllm.platforms import current_platform
9
+
10
+
11
+def pytest_collection_modifyitems(config, items):
12
+ """Set FLEX_ATTENTION backend for SigLIP tests on ROCm."""
13
+ if not current_platform.is_rocm():
14
+ return
15
+
16
+ siglip_tests = [item for item in items if "test_siglip" in item.nodeid]
17
+
18
+ if siglip_tests:
19
+ os.environ["VLLM_ATTENTION_BACKEND"] = "FLEX_ATTENTION"
20
+ warnings.warn(
21
+ "ROCm: Set VLLM_ATTENTION_BACKEND=FLEX_ATTENTION for SigLIP tests",
22
+ UserWarning,
23
+ stacklevel=1,
24
+ )
tests/models/registry.py
+4
@@ -667,6 +667,10 @@ _MULTIMODAL_EXAMPLE_MODELS = {
667
"moonshotai/Kimi-VL-A3B-Instruct",
668
extras={"thinking": "moonshotai/Kimi-VL-A3B-Thinking"},
669
trust_remote_code=True,
670
+ max_transformers_version="4.53.3",
671
+ transformers_version_reason="HF model uses deprecated transformers API "
672
+ "(PytorchGELUTanh, DynamicCache.seen_tokens, and more). See: "
673
+ "https://huggingface.co/moonshotai/Kimi-VL-A3B-Instruct/discussions/31",
674
),
675
"LightOnOCRForConditionalGeneration": _HfExamplesInfo(
676
"lightonai/LightOnOCR-1B",
vllm/v1/attention/backends/flex_attention.py
+13
-1
@@ -31,6 +31,7 @@ from vllm.logger import init_logger
31
from vllm.model_executor.layers.batch_invariant import (
32
vllm_is_batch_invariant,
33
)
34
+from vllm.platforms import current_platform
35
from vllm.utils.math_utils import cdiv
36
from vllm.utils.torch_utils import is_torch_equal_or_newer
37
from vllm.v1.attention.backends.utils import (
@@ -927,7 +928,18 @@ def get_kernel_options(
928
929
if torch.cuda.is_available():
930
device_props = torch.cuda.get_device_properties()
930
- max_shared_memory = device_props.shared_memory_per_block_optin
931
+ # ROCm doesn't expose shared_memory_per_block_optin attribute
932
+ # AMD GPUs typically have 64KB LDS (Local Data Share) per workgroup
933
+ if hasattr(device_props, "shared_memory_per_block_optin"):
934
+ max_shared_memory = device_props.shared_memory_per_block_optin
935
+ elif current_platform.is_rocm():
936
+ # ROCm fallback: use 64KB
937
+ max_shared_memory = 65536
938
+ else:
939
+ raise RuntimeError(
940
+ "Unable to determine shared memory size on this hardware."
941
+ )
942
+
943
if max_shared_memory < 144 * 1024:
944
block_m_candidate = ensure_divisible(
945
max(1, block_m_candidate // 2), block_m