@tduncan / Nvidia-Optimization-Stack-VLLM Mirror · read-only / commits / 7b43db210c

Fork 0

A high-throughput and memory-efficient inference and serving engine for LLMs

Mirrored from https://github.com/tduncan-lum/Nvidia-Optimization-Stack---VLLM.git

Synced Nov 11, 05:00 UTC

[ROCm][CI][Bugfix] Multi-Modal Model Support Fixes and Attention Backend Improvements (#30270)

Signed-off-by: Andreas Karatzas <akaratza@amd.com>

Andreas Karatzas committed Dec 18, 2025 at 20:17 UTC 7b43db210c34023cb1e8bcec1d3c49537556856f

7 files changed +109 -47

.buildkite/test-amd.yaml

+16 -5

@@ -964,7 +964,7 @@ steps:

964 - pytest -v -s models/multimodal/processing

965

966 - label: Multi-Modal Models Test (Standard) # 60min

967 - timeout_in_minutes: 80

967 + timeout_in_minutes: 100

968 mirror_hardwares: [amdexperimental]

969 agent_pool: mi325_1

970 # grade: Blocking

@@ -973,13 +973,15 @@ steps:

973 - vllm/

974 - tests/models/multimodal

975 commands:

976 + - export MIOPEN_DEBUG_CONV_DIRECT=0

977 + - export MIOPEN_DEBUG_CONV_GEMM=0

978 - pip install git+https://github.com/TIGER-AI-Lab/Mantis.git

979 - pip freeze | grep -E 'torch'

980 - pytest -v -s models/multimodal -m core_model --ignore models/multimodal/generation/test_whisper.py --ignore models/multimodal/processing

981 - cd .. && VLLM_WORKER_MULTIPROC_METHOD=spawn pytest -v -s tests/models/multimodal/generation/test_whisper.py -m core_model # Otherwise, mp_method="spawn" doesn't work

982

981 -- label: Multi-Modal Accuracy Eval (Small Models) # 150min - 180min

982 - timeout_in_minutes: 180

983 +- label: Multi-Modal Accuracy Eval (Small Models) # 5min

984 + timeout_in_minutes: 10

985 mirror_hardwares: [amdexperimental, amdproduction]

986 agent_pool: mi325_1

987 # grade: Blocking

@@ -989,7 +991,9 @@ steps:

991 - vllm/inputs/

992 - vllm/v1/core/

993 commands:

992 - - pytest -s -v test_lm_eval_correctness.py --config-list-file=configs/models-mm-small.txt --tp-size=1

994 + - export MIOPEN_DEBUG_CONV_DIRECT=0

995 + - export MIOPEN_DEBUG_CONV_GEMM=0

996 + - pytest -s -v test_lm_eval_correctness.py --config-list-file=configs/models-mm-small.txt

997

998 - label: Multi-Modal Models Test (Extended) 1 # 60min

999 timeout_in_minutes: 120

@@ -1001,10 +1005,13 @@ steps:

1005 - vllm/

1006 - tests/models/multimodal

1007 commands:

1008 + - export MIOPEN_DEBUG_CONV_DIRECT=0

1009 + - export MIOPEN_DEBUG_CONV_GEMM=0

1010 - pip install git+https://github.com/TIGER-AI-Lab/Mantis.git

1011 - pytest -v -s models/multimodal -m 'not core_model' --ignore models/multimodal/generation/test_common.py --ignore models/multimodal/processing

1012

1007 -- label: Multi-Modal Models Test (Extended) 2

1013 +- label: Multi-Modal Models Test (Extended) 2 #60min

1014 + timeout_in_minutes: 120

1015 mirror_hardwares: [amdexperimental]

1016 agent_pool: mi325_1

1017 # grade: Blocking

@@ -1013,6 +1020,8 @@ steps:

1020 - vllm/

1021 - tests/models/multimodal

1022 commands:

1023 + - export MIOPEN_DEBUG_CONV_DIRECT=0

1024 + - export MIOPEN_DEBUG_CONV_GEMM=0

1025 - pip install git+https://github.com/TIGER-AI-Lab/Mantis.git

1026 - pytest -v -s models/multimodal/generation/test_common.py -m 'split(group=0) and not core_model'

1027

@@ -1026,6 +1035,8 @@ steps:

1035 - vllm/

1036 - tests/models/multimodal

1037 commands:

1038 + - export MIOPEN_DEBUG_CONV_DIRECT=0

1039 + - export MIOPEN_DEBUG_CONV_GEMM=0

1040 - pip install git+https://github.com/TIGER-AI-Lab/Mantis.git

1041 - pytest -v -s models/multimodal/generation/test_common.py -m 'split(group=1) and not core_model'

1042

tests/models/multimodal/conftest.py renamed

+3 -6

@@ -1,6 +1,6 @@

1 # SPDX-License-Identifier: Apache-2.0

2 # SPDX-FileCopyrightText: Copyright contributors to the vLLM project

3 -"""Pytest configuration for vLLM tests."""

3 +"""Pytest configuration for vLLM multimodal tests."""

5 import warnings

@@ -9,16 +9,13 @@ import torch

9 from vllm.platforms import current_platform

12 -def pytest_configure(config):

13 - """Disable Flash/MemEfficient SDP on ROCm to avoid HF

14 - Transformers accuracy issues.

15 - """

12 +def pytest_collection_modifyitems(config, items):

13 + """Configure ROCm-specific settings based on collected tests."""

14 if not current_platform.is_rocm():

15 return

17 skip_patterns = ["test_granite_speech.py"]

18 if any(pattern in str(arg) for arg in config.args for pattern in skip_patterns):

21 - # Skip disabling SDP for Granite Speech tests on ROCm

19 return

21 # Disable Flash/MemEfficient SDP on ROCm to avoid HF Transformers

tests/models/multimodal/generation/test_common.py

+30 -2

@@ -173,6 +173,13 @@ VLM_TEST_SETTINGS = {

173 auto_cls=AutoModelForImageTextToText,

174 vllm_output_post_proc=model_utils.qwen2_vllm_to_hf_output,

175 patch_hf_runner=model_utils.qwen3_vl_patch_hf_runner,

176 + vllm_runner_kwargs={

177 + "attention_config": {

178 + "backend": "ROCM_AITER_FA",

179 + },

180 + }

181 + if current_platform.is_rocm()

182 + else None,

183 image_size_factors=[(0.25,), (0.25, 0.25, 0.25), (0.25, 0.2, 0.15)],

184 marks=[

185 pytest.mark.core_model,

@@ -253,8 +260,19 @@ VLM_TEST_SETTINGS = {

260 image_size_factors=[(0.25, 0.2, 0.15)],

261 vllm_runner_kwargs={

262 "model_impl": "transformers",

263 + # TODO: [ROCm] Revert this once issue #30167 is resolved

264 + **(

265 + {

266 + "mm_processor_kwargs": {

267 + "min_pixels": 256 * 28 * 28,

268 + "max_pixels": 1280 * 28 * 28,

269 + },

270 + }

271 + if current_platform.is_rocm()

272 + else {}

273 + ),

274 },

257 - marks=[large_gpu_mark(min_gb=32)],

275 + marks=[large_gpu_mark(min_gb=80 if current_platform.is_rocm() else 32)],

276 ),

277 #### Extended model tests

278 "aria": VLMTestInfo(

@@ -645,7 +663,17 @@ VLM_TEST_SETTINGS = {

663 hf_output_post_proc=model_utils.minimax_vl_01_hf_output,

664 patch_hf_runner=model_utils.minimax_vl_01_patch_hf_runner,

665 auto_cls=AutoModelForImageTextToText,

648 - marks=[large_gpu_mark(min_gb=80)],

666 + marks=[

667 + large_gpu_mark(min_gb=80),

668 + # TODO: [ROCm] Fix pickle issue with ROCm spawn and tp>1

669 + pytest.mark.skipif(

670 + current_platform.is_rocm(),

671 + reason=(

672 + "ROCm: Model too large for single GPU; "

673 + "multi-GPU blocked by HF _LazyConfigMapping pickle issue with spawn"

674 + ),

675 + ),

676 + ],

677 ),

678 "molmo": VLMTestInfo(

679 models=["allenai/Molmo-7B-D-0924"],

tests/models/multimodal/generation/test_granite_speech.py

+1 -1

@@ -39,7 +39,7 @@ models = [MODEL_NAME]

39 def granite_speech_attention_config():

40 """Return attention config for Granite Speech tests on ROCm."""

41 if current_platform.is_rocm():

42 - return {"backend": "TRITON_ATTN"}

42 + return {"backend": "ROCM_AITER_FA"}

43 return None

tests/models/multimodal/pooling/conftest.py deleted

-18

@@ -1,18 +0,0 @@

1 -# SPDX-License-Identifier: Apache-2.0

2 -# SPDX-FileCopyrightText: Copyright contributors to the vLLM project

3 -"""Pytest configuration for vLLM pooling tests."""

4 -

5 -import pytest

6 -

7 -from vllm.platforms import current_platform

8 -

9 -

10 -@pytest.fixture

11 -def siglip_attention_config():

12 - """Return attention config for SigLIP tests on ROCm.

13 -

14 - On ROCm, SigLIP tests require FLEX_ATTENTION backend.

15 - """

16 - if current_platform.is_rocm():

17 - return {"backend": "FLEX_ATTENTION"}

18 - return None

vllm/model_executor/models/transformers/multimodal.py

+31 -1

@@ -22,6 +22,7 @@ from typing import TYPE_CHECKING

22 import torch

24 from vllm.config.utils import getattr_iter

25 +from vllm.logger import init_logger

26 from vllm.model_executor.models.interfaces import SupportsMRoPE, SupportsMultiModal

27 from vllm.model_executor.models.utils import WeightsMapper

28 from vllm.multimodal import MultiModalKwargsItems

@@ -36,6 +37,7 @@ from vllm.multimodal.inputs import (

37 from vllm.multimodal.parse import ImageProcessorItems, MultiModalDataItems

38 from vllm.multimodal.processing import BaseMultiModalProcessor, BaseProcessingInfo

39 from vllm.multimodal.profiling import BaseDummyInputsBuilder

40 +from vllm.platforms import current_platform

41 from vllm.sequence import IntermediateTensors

43 if TYPE_CHECKING:

@@ -52,6 +54,8 @@ DYNAMIC_ARG_DIMS = {

54 "inputs_embeds": 0,

55 }

57 +logger = init_logger(__name__)

58 +

60 class MultiModalProcessingInfo(BaseProcessingInfo):

61 def get_supported_mm_limits(self):

@@ -345,8 +349,29 @@ class MultiModalMixin(SupportsMultiModal, SupportsMRoPE):

349

350 num_image_patches = kwargs.pop("num_image_patches")

351 kwargs.pop("token_type_ids", None) # used only in `forward`

352 +

353 if pixel_values is not None:

349 - vision_embeddings = self.model.get_image_features(pixel_values, **kwargs)

354 + # ROCm: Force math SDP backend for vision encoder to avoid accuracy issues

355 + # with flash_sdp and mem_efficient_sdp

356 + if current_platform.is_rocm():

357 + # TODO: [ROCm] Fix accuracy issues with flash backend

358 + logger.debug(

359 + "ROCm platform detected. Forcing math SDP backend "

360 + "for vision encoder. Currently ROCm platform has "

361 + "accuracy issues with `flash_sdp` and"

362 + "`mem_efficient_sdp` backends. See issue: "

363 + "https://github.com/vllm-project/vllm/issues/30167"

364 + )

365 + with torch.nn.attention.sdpa_kernel(

366 + backends=[torch.nn.attention.SDPBackend.MATH]

367 + ):

368 + vision_embeddings = self.model.get_image_features(

369 + pixel_values, **kwargs

370 + )

371 + else:

372 + vision_embeddings = self.model.get_image_features(

373 + pixel_values, **kwargs

374 + )

375

376 if isinstance(vision_embeddings, torch.Tensor):

377 if vision_embeddings.ndim == 2:

@@ -364,6 +389,11 @@ class MultiModalMixin(SupportsMultiModal, SupportsMRoPE):

389 ]

390

391 return vision_embeddings

392 + else:

393 + logger.debug(

394 + "No pixel values or image embeddings provided for multimodal embedding."

395 + )

396 + return None

397

398 def get_mrope_input_positions(

399 self,

vllm/platforms/rocm.py

+28 -14

@@ -8,6 +8,7 @@ from typing import TYPE_CHECKING, Optional

8 import torch

10 import vllm.envs as envs

11 +from vllm.attention.backends.abstract import AttentionType

12 from vllm.attention.backends.registry import AttentionBackendEnum

13 from vllm.logger import init_logger

14 from vllm.utils.torch_utils import cuda_device_count_stateless

@@ -204,7 +205,7 @@ class RocmPlatform(Platform):

205 assert block_size == 1, (

206 "Sparse MLA backend on ROCm only supports block size 1 for now."

207 )

207 - logger.info_once("Using Sparse MLA backend on V1 engine.")

208 + logger.info_once("Using Sparse MLA backend.")

209 return AttentionBackendEnum.ROCM_AITER_MLA_SPARSE.get_path()

210

211 if attn_selector_config.use_mla:

@@ -239,16 +240,16 @@ class RocmPlatform(Platform):

240 return AttentionBackendEnum.FLEX_ATTENTION.get_path()

241

242 if selected_backend == AttentionBackendEnum.TRITON_ATTN:

242 - logger.info("Using Triton Attention backend on V1 engine.")

243 + logger.info("Using Triton Attention backend.")

244 return AttentionBackendEnum.TRITON_ATTN.get_path()

245

246 if selected_backend == AttentionBackendEnum.ROCM_ATTN:

246 - logger.info("Using Rocm Attention backend on V1 engine.")

247 + logger.info("Using Rocm Attention backend.")

248 return AttentionBackendEnum.ROCM_ATTN.get_path()

249

250 if selected_backend == AttentionBackendEnum.ROCM_AITER_FA:

251 if on_gfx9():

251 - logger.info("Using Aiter Flash Attention backend on V1 engine.")

252 + logger.info("Using Aiter Flash Attention backend.")

253 return AttentionBackendEnum.ROCM_AITER_FA.get_path()

254 else:

255 raise ValueError(

@@ -257,25 +258,25 @@ class RocmPlatform(Platform):

258 )

259

260 if selected_backend == AttentionBackendEnum.ROCM_AITER_UNIFIED_ATTN:

260 - logger.info("Using Aiter Unified Attention backend on V1 engine.")

261 + logger.info("Using Aiter Unified Attention backend.")

262 return AttentionBackendEnum.ROCM_AITER_UNIFIED_ATTN.get_path()

263

264 # Handle automatic backend selection based on environment variables

265 if selected_backend is None:

266 # Priority 1: Check for AITER Unified Attention (must check before MHA)

267 if envs.VLLM_ROCM_USE_AITER and envs.VLLM_ROCM_USE_AITER_UNIFIED_ATTENTION:

267 - logger.info("Using Aiter Unified Attention backend on V1 engine.")

268 + logger.info("Using Aiter Unified Attention backend.")

269 return AttentionBackendEnum.ROCM_AITER_UNIFIED_ATTN.get_path()

270

271 # Priority 2: Check for AITER MHA (Flash Attention)

272 # Only use if explicitly enabled (not just VLLM_ROCM_USE_AITER=1)

273 if envs.VLLM_ROCM_USE_AITER and envs.VLLM_ROCM_USE_AITER_MHA and on_gfx9():

273 - logger.info("Using Aiter Flash Attention backend on V1 engine.")

274 + logger.info("Using Aiter Flash Attention backend.")

275 return AttentionBackendEnum.ROCM_AITER_FA.get_path()

276

277 # Priority 3: Check for ROCM_ATTN (prefill-decode split)

278 if envs.VLLM_V1_USE_PREFILL_DECODE_ATTENTION:

278 - logger.info("Using Rocm Attention backend on V1 engine.")

279 + logger.info("Using Rocm Attention backend.")

280 return AttentionBackendEnum.ROCM_ATTN.get_path()

281

282 # Priority 4: Check for AITER enabled without specific flags

@@ -285,11 +286,19 @@ class RocmPlatform(Platform):

286 and on_gfx9()

287 and envs.VLLM_ROCM_USE_AITER_MHA is not False

288 ):

288 - logger.info("Using Aiter Flash Attention backend on V1 engine.")

289 + logger.info("Using Aiter Flash Attention backend.")

290 return AttentionBackendEnum.ROCM_AITER_FA.get_path()

291

292 + # Priority 5: If model is Encoder-only self-attention type

293 + if (

294 + attn_selector_config.attn_type is not None

295 + and attn_selector_config.attn_type == AttentionType.ENCODER_ONLY

296 + ):

297 + logger.info("Using FlexAttention backend.")

298 + return AttentionBackendEnum.FLEX_ATTENTION.get_path()

299 +

300 # Default: Triton Unified Attention

292 - logger.info("Using Triton Attention backend on V1 engine.")

301 + logger.info("Using Triton Attention backend.")

302 return AttentionBackendEnum.TRITON_ATTN.get_path()

303

304 raise RuntimeError(

@@ -324,14 +333,19 @@ class RocmPlatform(Platform):

333

334 from vllm._aiter_ops import rocm_aiter_ops

335

327 - if rocm_aiter_ops.is_mha_enabled():

328 - # Note: AITER FA is only supported for Qwen-VL models.

329 - # TODO: Add support for other VL models in their model class.

336 + if rocm_aiter_ops.is_enabled():

337 + logger.info_once("Using AITER Flash Attention backend for ViT model.")

338 return AttentionBackendEnum.ROCM_AITER_FA

339

332 - if on_gfx9() and find_spec("flash_attn") is not None:

340 + if (

341 + on_gfx9()

342 + and find_spec("flash_attn") is not None

343 + and (dtype == torch.float16 or dtype == torch.bfloat16)

344 + ):

345 + logger.info_once("Using Flash Attention backend for ViT model.")

346 return AttentionBackendEnum.FLASH_ATTN

347

348 + logger.info_once("Using Torch SDPA backend for ViT model.")

349 return AttentionBackendEnum.TORCH_SDPA

350

351 @classmethod