A high-throughput and memory-efficient inference and serving engine for LLMs
Mirrored from https://github.com/tduncan-lum/Nvidia-Optimization-Stack---VLLM.git
Synced Nov 7, 17:45 UTC
[ROCm][CI] Disable skinny GEMMs in multimodal tests to fix non-deterministic results (#35049)
Signed-off-by: Andreas Karatzas <akaratza@amd.com>
Andreas Karatzas committed
Feb 25, 2026 at 10:48 UTC
8fd69754798c50b7b07938451bd97b1e66765927
1 file changed
+18
tests/models/multimodal/conftest.py
+18
@@ -2,6 +2,7 @@
2
# SPDX-FileCopyrightText: Copyright contributors to the vLLM project
3
"""Pytest configuration for vLLM multimodal tests."""
4
5
+import os
6
import warnings
7
8
import torch
@@ -9,6 +10,23 @@ import torch
10
from vllm.platforms import current_platform
11
12
13
+def pytest_configure(config):
14
+ """Early ROCm configuration that must happen before test collection."""
15
+ if not current_platform.is_rocm():
16
+ return
17
+
18
+ # Disable skinny GEMM on ROCm to avoid non-deterministic results
19
+ # from atomic reductions in wvSplitKrc kernel.
20
+ # See: https://github.com/vllm-project/vllm/pull/33493#issuecomment-3906083975
21
+ os.environ["VLLM_ROCM_USE_SKINNY_GEMM"] = "0"
22
+ warnings.warn(
23
+ "ROCm: Set VLLM_ROCM_USE_SKINNY_GEMM=0 to avoid non-deterministic "
24
+ "results from skinny GEMM atomic reductions",
25
+ UserWarning,
26
+ stacklevel=1,
27
+ )
28
+
29
+
30
def pytest_collection_modifyitems(config, items):
31
"""Configure ROCm-specific settings based on collected tests."""
32
if not current_platform.is_rocm():