1ab2801dde · tduncan/Nvidia-Optimization-Stack-VLLM

csrc/libtorch_stable/moe/moe_align_sum_kernels.cu

+3 -1

@@ -532,7 +532,9 @@ __global__ void moe_lora_align_block_size_kernel(

532

533 // Populate the token_mask based on the token-LoRA mapping

534 int num_tokens = numel / topk_num;

535 - if (threadIdx.x == 0) {

535 + // Only the even counting block owns per-LoRA metadata. The odd block only

536 + // initializes sorted_token_ids and must not race the final count write.

537 + if (blockIdx.x % 2 == 0 && threadIdx.x == 0) {

538 total_tokens_post_pad[lora_id] = 0;

539

540 for (int i = 0; i < num_tokens; i++) {

examples/rl/rlhf_async_new_apis.py

-2

@@ -173,8 +173,6 @@ ray_env_vars = {}

173 if current_platform.is_rocm():

174 # Workaround for RCCL bug. See https://github.com/ROCm/rocm-systems/issues/5756

175 ray_env_vars["RAY_EXPERIMENTAL_NOSET_HIP_VISIBLE_DEVICES"] = "1"

176 - # For ROCm, BATCH_INVARIANT vllm is not supported

177 - ray_env_vars["VLLM_ROCM_USE_SKINNY_GEMM"] = "0"

176 else:

177 # Enable batch invariance for deterministic outputs on NVIDIA

178 ray_env_vars["VLLM_BATCH_INVARIANT"] = "1"

tests/entrypoints/multimodal/openai/chat_completion/test_vision.py

+1 -2

@@ -9,7 +9,7 @@ import pytest_asyncio

9 from transformers import AutoProcessor

10

11 from tests.entrypoints.multimodal.conftest import TEST_IMAGE_ASSETS

12 -from tests.utils import ROCM_ENV_OVERRIDES, ROCM_EXTRA_ARGS, RemoteOpenAIServer

12 +from tests.utils import ROCM_EXTRA_ARGS, RemoteOpenAIServer

13 from vllm.multimodal.media import MediaWithBytes

14 from vllm.multimodal.utils import encode_image_url, fetch_image

15 from vllm.platforms import current_platform

@@ -92,7 +92,6 @@ def server():

92 # ROCm: Increase timeouts to handle potential network delays and slower

93 # video processing when downloading multiple videos from external sources

94 env_overrides = {

95 - **ROCM_ENV_OVERRIDES,

95 **(

96 {

97 "VLLM_VIDEO_FETCH_TIMEOUT": "120",

tests/entrypoints/openai/chat_completion/test_completion_with_function_calling.py

+2 -4

@@ -10,7 +10,7 @@ import pytest

10 import pytest_asyncio

11

12 # downloading lora to test lora requests

13 -from tests.utils import ROCM_ENV_OVERRIDES, ROCM_EXTRA_ARGS, RemoteOpenAIServer

13 +from tests.utils import ROCM_EXTRA_ARGS, RemoteOpenAIServer

14

15 # any model with a chat template should work here

16 MODEL_NAME = "Qwen/Qwen3-0.6B"

@@ -144,9 +144,7 @@ def server():

144 "--enforce-eager",

145 ] + ROCM_EXTRA_ARGS

146

147 - with RemoteOpenAIServer(

148 - MODEL_NAME, args, env_dict=ROCM_ENV_OVERRIDES

149 - ) as remote_server:

147 + with RemoteOpenAIServer(MODEL_NAME, args) as remote_server:

148 yield remote_server

149

150

tests/entrypoints/pooling/basic/test_encode.py

-8

@@ -7,7 +7,6 @@ import pytest

7

8 from vllm import LLM, PoolingParams

9 from vllm.distributed import cleanup_dist_env_and_memory

10 -from vllm.platforms import current_platform

10

11 MODEL_NAME = "intfloat/multilingual-e5-small"

12

@@ -30,12 +29,6 @@ TOKEN_IDS = [

29

30 @pytest.fixture(scope="module")

31 def llm():

33 - # ROCm: Use FLEX_ATTENTION backend as it's the only attention backend

34 - # that supports encoder-only models on ROCm.

35 - attention_config = None

36 - if current_platform.is_rocm():

37 - attention_config = {"backend": "FLEX_ATTENTION"}

38 -

32 # pytest caches the fixture so we use weakref.proxy to

33 # enable garbage collection

34 llm = LLM(

@@ -45,7 +38,6 @@ def llm():

38 gpu_memory_utilization=0.75,

39 enforce_eager=True,

40 seed=0,

48 - attention_config=attention_config,

41 )

42

43 yield weakref.proxy(llm)

tests/entrypoints/pooling/basic/test_truncation.py

-5

@@ -7,7 +7,6 @@ import pytest

7 import pytest_asyncio

8

9 from tests.utils import RemoteOpenAIServer

10 -from vllm.platforms import current_platform

10

11 MODEL_NAME = "sentence-transformers/all-MiniLM-L12-v2"

12 max_model_len = 128

@@ -39,10 +38,6 @@ def server():

38 str(max_model_len),

39 ]

40

42 - # ROCm: Use Flex Attention to support encoder-only self-attention.

43 - if current_platform.is_rocm():

44 - args.extend(["--attention-backend", "FLEX_ATTENTION"])

45 -

41 with RemoteOpenAIServer(MODEL_NAME, args) as remote_server:

42 yield remote_server

43

tests/entrypoints/pooling/embed/conftest.py deleted

-28

@@ -1,28 +0,0 @@

1 -# SPDX-License-Identifier: Apache-2.0

2 -# SPDX-FileCopyrightText: Copyright contributors to the vLLM project

3 -"""Pytest configuration for vLLM pooling embed tests."""

4 -

5 -import warnings

6 -

7 -import torch

8 -

9 -from vllm.platforms import current_platform

10 -

11 -

12 -def pytest_collection_modifyitems(config, items):

13 - """Configure ROCm-specific settings based on collected tests."""

14 - if not current_platform.is_rocm():

15 - return

16 -

17 - # Disable Flash/MemEfficient SDP on ROCm to avoid HF Transformers

18 - # accuracy issues: https://github.com/vllm-project/vllm/issues/30167

19 - # TODO: Remove once ROCm SDP accuracy issues are resolved on HuggingFace

20 - torch.backends.cuda.enable_flash_sdp(False)

21 - torch.backends.cuda.enable_mem_efficient_sdp(False)

22 - torch.backends.cuda.enable_math_sdp(True)

23 - warnings.warn(

24 - "ROCm: Disabled flash_sdp and mem_efficient_sdp, enabled math_sdp "

25 - "to avoid HuggingFace Transformers accuracy issues",

26 - UserWarning,

27 - stacklevel=1,

28 - )

tests/entrypoints/pooling/embed/test_correctness_mteb.py

-5

@@ -11,7 +11,6 @@ from tests.models.language.pooling_mteb_test.mteb_embed_utils import (

11 run_mteb_embed_task,

12 )

13 from tests.utils import RemoteOpenAIServer

14 -from vllm.platforms import current_platform

14

15 os.environ["VLLM_LOGGING_LEVEL"] = "WARNING"

16

@@ -23,10 +22,6 @@ MAIN_SCORE = 0.7422994752439667

22 def server():

23 args = ["--runner", "pooling", "--enforce-eager", "--disable-uvicorn-access-log"]

24

26 - # ROCm: Use Flex Attention to support encoder-only self-attention.

27 - if current_platform.is_rocm():

28 - args.extend(["--attention-backend", "FLEX_ATTENTION"])

29 -

25 with RemoteOpenAIServer(MODEL_NAME, args) as remote_server:

26 yield remote_server

27

tests/entrypoints/pooling/embed/test_offline.py

-8

@@ -8,7 +8,6 @@ import torch.nn.functional as F

8

9 from vllm import LLM, EmbeddingRequestOutput, PoolingParams

10 from vllm.distributed import cleanup_dist_env_and_memory

11 -from vllm.platforms import current_platform

11 from vllm.tasks import PoolingTask

12

13 MODEL_NAME = "intfloat/multilingual-e5-small"

@@ -20,12 +19,6 @@ embedding_size = 384

19

20 @pytest.fixture(scope="module")

21 def llm():

23 - # ROCm: Use FLEX_ATTENTION backend as it's the only attention backend

24 - # that supports encoder-only models on ROCm.

25 - attention_config = None

26 - if current_platform.is_rocm():

27 - attention_config = {"backend": "FLEX_ATTENTION"}

28 -

22 # pytest caches the fixture so we use weakref.proxy to

23 # enable garbage collection

24 llm = LLM(

@@ -35,7 +28,6 @@ def llm():

28 gpu_memory_utilization=0.75,

29 enforce_eager=True,

30 seed=0,

38 - attention_config=attention_config,

31 )

32 assert embedding_size == llm.model_config.embedding_size

33

tests/entrypoints/pooling/embed/test_online.py

+9 -11

@@ -50,14 +50,6 @@ input_tokens = [

50 2,

51 ]

52

53 -if current_platform.is_rocm():

54 - # Disable Flash/MemEfficient SDP on ROCm to avoid HF Transformers

55 - # accuracy issues: https://github.com/vllm-project/vllm/issues/30167

56 - # TODO: Remove once ROCm SDP accuracy issues are resolved on HuggingFace

57 - torch.backends.cuda.enable_flash_sdp(False)

58 - torch.backends.cuda.enable_mem_efficient_sdp(False)

59 - torch.backends.cuda.enable_math_sdp(True)

60 -

53 # On ROCm, floating-point reductions in attention and GEMM kernels are

54 # non-associative and sensitive to batch geometry. Force LLM instances

55 # into an identical, deterministic execution mode:

@@ -507,11 +499,17 @@ async def test_base64_embedding(hf_model, client: openai.AsyncOpenAI, model_name

499 "The best thing about vLLM is that it supports many different models",

500 ]

501

502 + def check_outputs(outputs: list[list[float]]) -> None:

503 + # The embeddings endpoint executes list elements as separate scheduler

504 + # requests, so use the same per-request geometry for the HF reference.

505 + for input_text, output in zip(input_texts, outputs, strict=True):

506 + run_embedding_correctness_test(hf_model, [input_text], [output])

507 +

508 responses_float = await client.embeddings.create(

509 input=input_texts, model=model_name, encoding_format="float"

510 )

511 float_data = [d.embedding for d in responses_float.data]

514 - run_embedding_correctness_test(hf_model, input_texts, float_data)

512 + check_outputs(float_data)

513

514 responses_base64 = await client.embeddings.create(

515 input=input_texts, model=model_name, encoding_format="base64"

@@ -522,14 +520,14 @@ async def test_base64_embedding(hf_model, client: openai.AsyncOpenAI, model_name

520 np.frombuffer(base64.b64decode(data.embedding), dtype="float32").tolist()

521 )

522

525 - run_embedding_correctness_test(hf_model, input_texts, base64_data)

523 + check_outputs(base64_data)

524

525 # Default response is float32 decoded from base64 by OpenAI Client

526 responses_default = await client.embeddings.create(

527 input=input_texts, model=model_name

528 )

529 default_data = [d.embedding for d in responses_default.data]

532 - run_embedding_correctness_test(hf_model, input_texts, default_data)

530 + check_outputs(default_data)

531

532

533 @pytest.mark.asyncio

tests/entrypoints/pooling/embed/test_online_dimensions.py

-5

@@ -12,7 +12,6 @@ from tests.models.language.pooling.embed_utils import run_embedding_correctness_

12 from tests.models.utils import EmbedModelInfo

13 from tests.utils import ROCM_EXTRA_ARGS, RemoteOpenAIServer

14 from vllm.entrypoints.pooling.embed.protocol import EmbeddingResponse

15 -from vllm.platforms import current_platform

15

16 MODELS = [

17 EmbedModelInfo("intfloat/multilingual-e5-small", is_matryoshka=False),

@@ -57,10 +56,6 @@ def server(model_info, dtype: str):

56 ["--trust_remote_code", "--hf_overrides", '{"matryoshka_dimensions":[256]}']

57 )

58

60 - # ROCm: Use Flex Attention to support encoder-only self-attention.

61 - if current_platform.is_rocm():

62 - args.extend(["--attention-backend", "FLEX_ATTENTION"])

63 -

59 with RemoteOpenAIServer(model_info.name, args) as remote_server:

60 yield remote_server

61

tests/entrypoints/pooling/embed/test_online_long_text.py

-5

@@ -16,7 +16,6 @@ import pytest_asyncio

16

17 from tests.utils import RemoteOpenAIServer

18 from vllm.entrypoints.pooling.embed.protocol import EmbeddingResponse

19 -from vllm.platforms import current_platform

19

20

21 def _generate_random_text(word_count: int) -> str:

@@ -225,10 +224,6 @@ def server_with_chunked_processing():

224 "0.8",

225 ]

226

228 - # ROCm: Use Flex Attention to support encoder-only self-attention.

229 - if current_platform.is_rocm():

230 - args.extend(["--attention-backend", "FLEX_ATTENTION"])

231 -

227 with RemoteOpenAIServer(MODEL_NAME, args) as remote_server:

228 yield remote_server

229

tests/entrypoints/pooling/scoring/test_bi_encoder_offline.py

-8

@@ -8,7 +8,6 @@ import pytest

8 from tests.entrypoints.pooling.scoring.util import EncoderScoringHfRunner

9 from vllm import LLM

10 from vllm.distributed import cleanup_dist_env_and_memory

11 -from vllm.platforms import current_platform

11

12 MODEL_NAME = "intfloat/multilingual-e5-small"

13 PROMPT = "The chef prepared a delicious meal."

@@ -29,12 +28,6 @@ DTYPE = "half"

28

29 @pytest.fixture(scope="module")

30 def llm():

32 - # ROCm: Use FLEX_ATTENTION backend as it's the only attention backend

33 - # that supports encoder-only models on ROCm.

34 - attention_config = None

35 - if current_platform.is_rocm():

36 - attention_config = {"backend": "FLEX_ATTENTION"}

37 -

31 # pytest caches the fixture so we use weakref.proxy to

32 # enable garbage collection

33 llm = LLM(

@@ -44,7 +37,6 @@ def llm():

37 gpu_memory_utilization=0.75,

38 enforce_eager=True,

39 seed=0,

47 - attention_config=attention_config,

40 )

41

42 yield weakref.proxy(llm)

tests/entrypoints/pooling/scoring/test_bi_encoder_online.py

-5

@@ -8,7 +8,6 @@ from tests.entrypoints.pooling.scoring.util import EncoderScoringHfRunner

8 from tests.utils import RemoteOpenAIServer

9 from vllm.entrypoints.pooling.pooling.protocol import PoolingResponse

10 from vllm.entrypoints.pooling.scoring.protocol import RerankResponse, ScoreResponse

11 -from vllm.platforms import current_platform

11

12 MODEL_NAME = "BAAI/bge-base-en-v1.5"

13 input_text = "This product was excellent and exceeded my expectations"

@@ -30,10 +29,6 @@ TEXTS_2 = [

29 def server():

30 args = ["--enforce-eager", "--max-model-len", "100", "--dtype", DTYPE]

31

33 - # ROCm: Use Flex Attention to support encoder-only self-attention.

34 - if current_platform.is_rocm():

35 - args.extend(["--attention-backend", "FLEX_ATTENTION"])

36 -

32 with RemoteOpenAIServer(MODEL_NAME, args) as remote_server:

33 yield remote_server

34

tests/entrypoints/pooling/scoring/test_cross_encoder_correctness_mteb.py

-5

@@ -13,7 +13,6 @@ from tests.models.language.pooling_mteb_test.mteb_score_utils import (

13 run_mteb_rerank,

14 )

15 from tests.utils import RemoteOpenAIServer

16 -from vllm.platforms import current_platform

16

17 os.environ["VLLM_LOGGING_LEVEL"] = "WARNING"

18

@@ -25,10 +24,6 @@ st_main_score = 0.33457

24 def server():

25 args = ["--runner", "pooling", "--enforce-eager", "--disable-uvicorn-access-log"]

26

28 - # ROCm: Use Flex Attention to support encoder-only self-attention.

29 - if current_platform.is_rocm():

30 - args.extend(["--attention-backend", "FLEX_ATTENTION"])

31 -

27 with RemoteOpenAIServer(MODEL_NAME, args) as remote_server:

28 yield remote_server

29

tests/entrypoints/pooling/scoring/test_cross_encoder_offline.py

-8

@@ -9,7 +9,6 @@ import torch

9 from tests.models.utils import softmax

10 from vllm import LLM, PoolingParams

11 from vllm.distributed import cleanup_dist_env_and_memory

12 -from vllm.platforms import current_platform

12

13 MODEL_NAME = "tomaarsen/Qwen3-Reranker-0.6B-seq-cls"

14 PROMPT = "The chef prepared a delicious meal."

@@ -26,12 +25,6 @@ TEXTS_2 = [

25

26 @pytest.fixture(scope="module")

27 def llm():

29 - # ROCm: Use FLEX_ATTENTION backend as it's the only attention backend

30 - # that supports encoder-only models on ROCm.

31 - attention_config = None

32 - if current_platform.is_rocm():

33 - attention_config = {"backend": "FLEX_ATTENTION"}

34 -

28 # pytest caches the fixture so we use weakref.proxy to

29 # enable garbage collection

30 llm = LLM(

@@ -41,7 +34,6 @@ def llm():

34 gpu_memory_utilization=0.75,

35 enforce_eager=True,

36 seed=0,

44 - attention_config=attention_config,

37 )

38

39 yield weakref.proxy(llm)

tests/entrypoints/pooling/scoring/test_cross_encoder_online.py

-5

@@ -9,7 +9,6 @@ import torch.nn.functional as F

9 from tests.utils import RemoteOpenAIServer

10 from vllm.entrypoints.pooling.pooling.protocol import PoolingResponse

11 from vllm.entrypoints.pooling.scoring.protocol import RerankResponse, ScoreResponse

12 -from vllm.platforms import current_platform

12

13 MODEL_NAME = "BAAI/bge-reranker-base"

14 DTYPE = "half"

@@ -32,10 +31,6 @@ TEXTS_2 = [

31 def server():

32 args = ["--enforce-eager", "--max-model-len", "100", "--dtype", DTYPE]

33

35 - # ROCm: Use Flex Attention to support encoder-only self-attention.

36 - if current_platform.is_rocm():

37 - args.extend(["--attention-backend", "FLEX_ATTENTION"])

38 -

34 with RemoteOpenAIServer(MODEL_NAME, args) as remote_server:

35 yield remote_server

36

tests/entrypoints/pooling/scoring/test_cross_encoder_online_vision.py

-7

@@ -53,12 +53,6 @@ BACKEND_ABS_TOL: dict[str, float] = {

53 "FLEX_ATTENTION": 0.006,

54 }

55

56 -# ROCm: disable skinny GEMM to avoid non-deterministic results from

57 -# atomic reductions in wvSplitKrc kernel.

58 -# See: https://github.com/vllm-project/vllm/pull/33493#issuecomment-3906083975

59 -ROCM_ENV_OVERRIDES = (

60 - {"VLLM_ROCM_USE_SKINNY_GEMM": "0"} if current_platform.is_rocm() else {}

61 -)

56 # ROCm: disable prefix caching and eliminate batch variance to reduce

57 # test flakiness.

58 ROCM_EXTRA_ARGS = (

@@ -132,7 +126,6 @@ def server(request):

126 args += ["--attention-config", json.dumps({"backend": backend})]

127 args += ROCM_EXTRA_ARGS

128

135 - env = dict(ROCM_ENV_OVERRIDES)

129 if backend != "ROCM_AITER_FA":

130 env["VLLM_ROCM_USE_AITER"] = "0"

131

tests/entrypoints/pooling/scoring/test_late_interaction_offline.py

-8

@@ -7,7 +7,6 @@ import pytest

7

8 from vllm import LLM

9 from vllm.distributed import cleanup_dist_env_and_memory

10 -from vllm.platforms import current_platform

10

11 from .util import ColBERTScoringHfRunner

12

@@ -32,12 +31,6 @@ DTYPE = "half"

31

32 @pytest.fixture(scope="module")

33 def llm():

35 - # ROCm: Use FLEX_ATTENTION backend as it's the only attention backend

36 - # that supports encoder-only models on ROCm.

37 - attention_config = None

38 - if current_platform.is_rocm():

39 - attention_config = {"backend": "FLEX_ATTENTION"}

40 -

34 # pytest caches the fixture so we use weakref.proxy to

35 # enable garbage collection

36 llm = LLM(

@@ -47,7 +40,6 @@ def llm():

40 gpu_memory_utilization=0.75,

41 enforce_eager=True,

42 seed=0,

50 - attention_config=attention_config,

43 )

44

45 yield weakref.proxy(llm)

tests/entrypoints/pooling/scoring/test_late_interaction_offline_vision.py

-8

@@ -7,7 +7,6 @@ import pytest

7

8 from vllm import LLM

9 from vllm.distributed import cleanup_dist_env_and_memory

10 -from vllm.platforms import current_platform

10

11 from .util import make_base64_image, make_image_mm_param

12

@@ -16,12 +15,6 @@ MODEL_NAME = "vidore/colpali-v1.3-hf"

15

16 @pytest.fixture(scope="module")

17 def llm():

19 - # ROCm: Use FLEX_ATTENTION backend as it's the only attention backend

20 - # that supports encoder-only models on ROCm.

21 - attention_config = None

22 - if current_platform.is_rocm():

23 - attention_config = {"backend": "FLEX_ATTENTION"}

24 -

18 # pytest caches the fixture so we use weakref.proxy to

19 # enable garbage collection

20 llm = LLM(

@@ -31,7 +24,6 @@ def llm():

24 gpu_memory_utilization=0.75,

25 enforce_eager=True,

26 seed=0,

34 - attention_config=attention_config,

27 )

28

29 yield weakref.proxy(llm)

tests/entrypoints/pooling/token_embed/test_offline.py

-8

@@ -7,7 +7,6 @@ import pytest

7 from vllm import LLM, PoolingRequestOutput

8 from vllm.config import PoolerConfig

9 from vllm.distributed import cleanup_dist_env_and_memory

10 -from vllm.platforms import current_platform

10 from vllm.tasks import PoolingTask

11

12 MODEL_NAME = "intfloat/multilingual-e5-small"

@@ -19,12 +18,6 @@ embedding_size = 384

18

19 @pytest.fixture(scope="module")

20 def llm():

22 - # ROCm: Use FLEX_ATTENTION backend as it's the only attention backend

23 - # that supports encoder-only models on ROCm.

24 - attention_config = None

25 - if current_platform.is_rocm():

26 - attention_config = {"backend": "FLEX_ATTENTION"}

27 -

21 # pytest caches the fixture so we use weakref.proxy to

22 # enable garbage collection

23 llm = LLM(

@@ -35,7 +28,6 @@ def llm():

28 gpu_memory_utilization=0.75,

29 enforce_eager=True,

30 seed=0,

38 - attention_config=attention_config,

31 )

32 assert embedding_size == llm.model_config.embedding_size

33

tests/entrypoints/scale_out/token_in_token_out/test_serving_multimodal_tokens.py

+1 -6

@@ -9,8 +9,6 @@ processor in the pytest parent process to keep os.fork() in sibling

9 tests (e.g. test_weight_transfer_llm.py) deadlock-free.

10 """

11

12 -import os

13 -

12 import httpx

13 import pytest

14 import pytest_asyncio

@@ -41,10 +39,7 @@ def server():

39 "--no-enable-prefix-caching",

40 ]

41

44 - envs = os.environ.copy()

45 - envs["VLLM_ROCM_USE_SKINNY_GEMM"] = "0"

46 -

47 - with RemoteOpenAIServer(MODEL_NAME, args, env_dict=envs) as remote_server:

42 + with RemoteOpenAIServer(MODEL_NAME, args) as remote_server:

43 yield remote_server

44

45

tests/entrypoints/scale_out/token_in_token_out/test_serving_tokens.py

+1 -6

@@ -2,7 +2,6 @@

2 # SPDX-FileCopyrightText: Copyright contributors to the vLLM project

3

4 import json

5 -import os

5

6 import httpx

7 import pytest

@@ -79,11 +78,7 @@ def server(request):

78 else [str(extra_args)]

79 )

80

82 - envs = os.environ.copy()

83 - # See: https://github.com/vllm-project/vllm/pull/33493#issuecomment-3888060787

84 - envs["VLLM_ROCM_USE_SKINNY_GEMM"] = "0"

85 -

86 - with RemoteOpenAIServer(MODEL_NAME, args, env_dict=envs) as remote_server:

81 + with RemoteOpenAIServer(MODEL_NAME, args) as remote_server:

82 yield remote_server

83

84

tests/entrypoints/speech_to_text/conftest.py

-13

@@ -16,19 +16,6 @@ def add_attention_backend(server_args, attention_config):

16 server_args.extend(["--attention-backend", attention_config["backend"]])

17

18

19 -@pytest.fixture(scope="module")

20 -def rocm_aiter_fa_attention():

21 - """Return attention config for transcription/translation tests on ROCm.

22 -

23 - On ROCm, audio tests require ROCM_AITER_FA attention backend.

24 - """

25 - from vllm.platforms import current_platform

26 -

27 - if current_platform.is_rocm():

28 - return {"backend": "ROCM_AITER_FA"}

29 - return None

30 -

31 -

19 @pytest.fixture

20 def mary_had_lamb():

21 path = AudioAsset("mary_had_lamb").get_local_path()

tests/entrypoints/speech_to_text/realtime/test_realtime_validation.py

+5 -21

@@ -10,15 +10,13 @@ import pybase64 as base64

10 import pytest

11 import websockets

12

13 -from tests.entrypoints.speech_to_text.conftest import add_attention_backend

14 -from tests.utils import ROCM_ENV_OVERRIDES, ROCM_EXTRA_ARGS, RemoteOpenAIServer

13 +from tests.utils import ROCM_EXTRA_ARGS, RemoteOpenAIServer

14 from vllm.assets.audio import AudioAsset

15 from vllm.multimodal.media.audio import load_audio

16

17 # Increase engine iteration timeout for ROCm where first-use JIT compilation

18 # can exceed the default 60s, causing a silent deadlock in feed_tokens.

19 REALTIME_ENV_OVERRIDES = {

21 - **ROCM_ENV_OVERRIDES,

20 "VLLM_ENGINE_ITERATION_TIMEOUT_S": "600",

21 }

22

@@ -72,17 +70,13 @@ def mary_had_lamb_audio_chunks() -> list[str]:

70

71 @pytest.mark.asyncio

72 @pytest.mark.parametrize("model_name", [MODEL_NAME])

75 -async def test_multi_chunk_streaming(

76 - model_name, mary_had_lamb_audio_chunks, rocm_aiter_fa_attention

77 -):

73 +async def test_multi_chunk_streaming(model_name, mary_had_lamb_audio_chunks):

74 """Test streaming multiple audio chunks before committing."""

75 server_args = ["--enforce-eager", "--max-model-len", "2048"]

76

77 if model_name.startswith("mistralai"):

78 server_args += MISTRAL_FORMAT_ARGS

79

84 - add_attention_backend(server_args, rocm_aiter_fa_attention)

85 -

80 with RemoteOpenAIServer(

81 model_name, server_args, env_dict=REALTIME_ENV_OVERRIDES

82 ) as remote_server:

@@ -174,7 +168,7 @@ async def test_multi_chunk_streaming(

168 @pytest.mark.asyncio

169 @pytest.mark.parametrize("model_name", [MODEL_NAME])

170 async def test_empty_commit_does_not_crash_engine(

177 - model_name, mary_had_lamb_audio_chunks, rocm_aiter_fa_attention

171 + model_name, mary_had_lamb_audio_chunks

172 ):

173 """Test that committing without audio does not crash the engine.

174

@@ -189,8 +183,6 @@ async def test_empty_commit_does_not_crash_engine(

183 if model_name.startswith("mistralai"):

184 server_args += MISTRAL_FORMAT_ARGS

185

192 - add_attention_backend(server_args, rocm_aiter_fa_attention)

193 -

186 with RemoteOpenAIServer(

187 model_name, server_args, env_dict=REALTIME_ENV_OVERRIDES

188 ) as remote_server:

@@ -273,17 +265,13 @@ async def test_empty_commit_does_not_crash_engine(

265

266 @pytest.mark.asyncio

267 @pytest.mark.parametrize("model_name", [MODEL_NAME])

276 -async def test_session_update_invalid_model_returns_error(

277 - model_name, rocm_aiter_fa_attention

278 -):

268 +async def test_session_update_invalid_model_returns_error(model_name):

269 """Test that session.update with an invalid model returns an error."""

270 server_args = ["--enforce-eager", "--max-model-len", "2048"]

271

272 if model_name.startswith("mistralai"):

273 server_args += MISTRAL_FORMAT_ARGS

274

285 - add_attention_backend(server_args, rocm_aiter_fa_attention)

286 -

275 with RemoteOpenAIServer(

276 model_name, server_args, env_dict=REALTIME_ENV_OVERRIDES

277 ) as remote_server:

@@ -305,9 +293,7 @@ async def test_session_update_invalid_model_returns_error(

293

294 @pytest.mark.asyncio

295 @pytest.mark.parametrize("model_name", [MODEL_NAME])

308 -async def test_commit_without_session_update_returns_error(

309 - model_name, rocm_aiter_fa_attention

310 -):

296 +async def test_commit_without_session_update_returns_error(model_name):

297 """Test that committing before validating the model returns an error

298 and does not fall through to processing."""

299 server_args = ["--enforce-eager", "--max-model-len", "2048"]

@@ -315,8 +301,6 @@ async def test_commit_without_session_update_returns_error(

301 if model_name.startswith("mistralai"):

302 server_args += MISTRAL_FORMAT_ARGS

303

318 - add_attention_backend(server_args, rocm_aiter_fa_attention)

319 -

304 with RemoteOpenAIServer(

305 model_name, server_args, env_dict=REALTIME_ENV_OVERRIDES

306 ) as remote_server:

tests/entrypoints/speech_to_text/transcription/test_transcription_validation.py

+6 -18

@@ -6,8 +6,7 @@ import json

6

7 import pytest

8

9 -from tests.entrypoints.speech_to_text.conftest import add_attention_backend

10 -from tests.utils import ROCM_ENV_OVERRIDES, ROCM_EXTRA_ARGS, RemoteOpenAIServer

9 +from tests.utils import ROCM_EXTRA_ARGS, RemoteOpenAIServer

10

11 MISTRAL_FORMAT_ARGS = [

12 "--tokenizer_mode",

@@ -66,18 +65,14 @@ async def transcribe_and_check(

65 @pytest.mark.parametrize(

66 "model_name", ["mistralai/Voxtral-Mini-3B-2507", "Qwen/Qwen3-ASR-0.6B"]

67 )

69 -async def test_basic_audio(mary_had_lamb, model_name, rocm_aiter_fa_attention):

68 +async def test_basic_audio(mary_had_lamb, model_name):

69 server_args = ["--enforce-eager", *ROCM_EXTRA_ARGS]

70

71 if model_name.startswith("mistralai"):

72 server_args += MISTRAL_FORMAT_ARGS

73

75 - add_attention_backend(server_args, rocm_aiter_fa_attention)

76 -

74 # Based on https://github.com/openai/openai-cookbook/blob/main/examples/Whisper_prompting_guide.ipynb.

78 - with RemoteOpenAIServer(

79 - model_name, server_args, env_dict=ROCM_ENV_OVERRIDES

80 - ) as remote_server:

75 + with RemoteOpenAIServer(model_name, server_args) as remote_server:

76 client = remote_server.get_async_client()

77 await transcribe_and_check(

78 client,

@@ -90,7 +85,7 @@ async def test_basic_audio(mary_had_lamb, model_name, rocm_aiter_fa_attention):

85

86

87 @pytest.mark.asyncio

93 -async def test_basic_audio_with_lora(mary_had_lamb, rocm_aiter_fa_attention):

88 +async def test_basic_audio_with_lora(mary_had_lamb):

89 """Ensure STT (transcribe) requests can pass LoRA through to generate."""

90 # ROCm SPECIFIC CONFIGURATION:

91 # To ensure the test passes on ROCm, we modify the max model length to 512.

@@ -112,12 +107,8 @@ async def test_basic_audio_with_lora(mary_had_lamb, rocm_aiter_fa_attention):

107 "1",

108 ]

109

115 - add_attention_backend(server_args, rocm_aiter_fa_attention)

116 -

110 # Based on https://github.com/openai/openai-cookbook/blob/main/examples/Whisper_prompting_guide.ipynb.

118 - with RemoteOpenAIServer(

119 - model_name, server_args, env_dict=ROCM_ENV_OVERRIDES

120 - ) as remote_server:

111 + with RemoteOpenAIServer(model_name, server_args) as remote_server:

112 client = remote_server.get_async_client()

113 await transcribe_and_check(

114 client,

@@ -133,18 +124,15 @@ async def test_basic_audio_with_lora(mary_had_lamb, rocm_aiter_fa_attention):

124 @pytest.mark.parametrize(

125 "model_name", ["google/gemma-3n-E2B-it", "Qwen/Qwen3-ASR-0.6B"]

126 )

136 -async def test_basic_audio_foscolo(foscolo, rocm_aiter_fa_attention, model_name):

127 +async def test_basic_audio_foscolo(foscolo, model_name):

128 # Gemma accuracy on some of the audio samples we use is particularly bad,

129 # hence we use a different one here. WER is evaluated separately.

130 server_args = ["--enforce-eager", *ROCM_EXTRA_ARGS]

131

141 - add_attention_backend(server_args, rocm_aiter_fa_attention)

142 -

132 with RemoteOpenAIServer(

133 model_name,

134 server_args,

135 max_wait_seconds=480,

147 - env_dict=ROCM_ENV_OVERRIDES,

136 ) as remote_server:

137 client = remote_server.get_async_client()

138 await transcribe_and_check(

tests/entrypoints/speech_to_text/translation/test_translation_validation.py

+18 -12

@@ -28,27 +28,33 @@ def _get_rocm_attention_config(model_name):

28

29 Whisper uses cross-attention (ENCODER_DECODER) which ROCM_AITER_FA does

30 not support. For Whisper we use ROCM_AITER_UNIFIED_ATTN (or TRITON_ATTN

31 - as fallback); other models can use ROCM_AITER_FA.

31 + as fallback). Other models use ROCM_AITER_FA only on supported CDNA3+

32 + hardware and otherwise use the platform default.

33 """

34 from vllm.platforms import current_platform

35

36 if not current_platform.is_rocm():

37 return None

38

39 + try:

40 + from vllm.platforms.rocm import get_cdna_version

41 +

42 + cdna_version = get_cdna_version()

43 + except ImportError:

44 + logger.warning(

45 + "Could not check cdna version from rocm platform; "

46 + "using a compatible fallback attention backend."

47 + )

48 + cdna_version = -1

49 +

50 if "whisper" in model_name.lower():

39 - try:

40 - from vllm.platforms.rocm import get_cdna_version

41 -

42 - if get_cdna_version() > 2:

43 - return {"backend": "ROCM_AITER_UNIFIED_ATTN"}

44 - except ImportError:

45 - logger.warning(

46 - "Could not check cdna version from rocm platform, "

47 - "falling back to TRITON_ATTN for Whisper."

48 - )

51 + if cdna_version > 2:

52 + return {"backend": "ROCM_AITER_UNIFIED_ATTN"}

53 return {"backend": "TRITON_ATTN"}

54

51 - return {"backend": "ROCM_AITER_FA"}

55 + if cdna_version > 2:

56 + return {"backend": "ROCM_AITER_FA"}

57 + return None

58

59

60 def _get_server_args(attention_config):

tests/kernels/attention/test_prefix_prefill.py

+9 -17

@@ -5,12 +5,10 @@ import math

5 import random

6 import time

7 from collections.abc import Callable

8 -from contextlib import nullcontext

8

9 import pytest

10 import torch

11 import torch.nn.functional as F

13 -from torch.nn.attention import SDPBackend, sdpa_kernel

12

13 from vllm.platforms import current_platform

14 from vllm.utils.torch_utils import STR_DTYPE_TO_TORCH_DTYPE, set_random_seed

@@ -895,21 +893,15 @@ def test_contexted_kv_attention_alibi(

893 query_len, seq_len, alibi_slopes, device, dtype

894 )

895

898 - # Compute attention. On ROCm we force use of the Math SDPA backend rather than

899 - # the Flash or Mem-Efficient backends for increased numerical accuracy

900 - if current_platform.is_rocm():

901 - sdpa_context = sdpa_kernel(SDPBackend.MATH)

902 - else:

903 - sdpa_context = nullcontext()

904 - with sdpa_context:

905 - out = F.scaled_dot_product_attention(

906 - q_sdpa,

907 - k_sdpa,

908 - v_sdpa,

909 - attn_mask=alibi_mask,

910 - dropout_p=0.0,

911 - scale=scale,

912 - )

896 + # Compute attention

897 + out = F.scaled_dot_product_attention(

898 + q_sdpa,

899 + k_sdpa,

900 + v_sdpa,

901 + attn_mask=alibi_mask,

902 + dropout_p=0.0,

903 + scale=scale,

904 + )

905

906 # Reshape output back to [query_len, num_heads, head_size]

907 out = out.view(num_heads, query_len, head_size).permute(1, 0, 2)

tests/kernels/quantization/test_rocm_skinny_gemms.py

+1 -1

@@ -140,7 +140,7 @@ def test_rocm_wvsplitkrc_kernel(xnorm, n, k, m, dtype, seed, padded_a, bias_mode

140 GrpsShrB = min(N_p2 // 16, 4)

141 # Given the above, how many CUs would we need?

142 CuNeeded = rndup_cus * GrpsShrB

143 - # candidate for atomic reduce count splitk?

143 + # Deterministic reduction stores one float workspace value per K shard.

144 fits_wvsplitkrc = (N_p2 * m * ((k + 512 - 1) // 512)) <= 128 * 1024 * 12

145 fits_wvsplitkrc &= CuNeeded <= cu_count

146

tests/lora/test_gptoss_tp.py

+20

@@ -50,6 +50,20 @@ def reformat(text: str) -> str:

50 return text

51

52

53 +def _enable_deterministic_lora_shrink(monkeypatch: pytest.MonkeyPatch) -> None:

54 + if not current_platform.is_rocm():

55 + return

56 +

57 + # These tests assert exact greedy outputs. Force the Triton LoRA shrink

58 + # kernel to use SPLIT_K=1 so it stores the complete reduction directly

59 + # instead of accumulating split-K partial results with atomic_add.

60 + monkeypatch.setenv("VLLM_BATCH_INVARIANT", "1")

61 + # The kernel configuration reads VLLM_BATCH_INVARIANT at import time.

62 + # Spawn the engine process so it observes this setting even if the LoRA

63 + # Triton utilities were already imported during test collection.

64 + monkeypatch.setenv("VLLM_WORKER_MULTIPROC_METHOD", "spawn")

65 +

66 +

67 def generate_and_test(llm: vllm.LLM, lora_path: str, lora_id: int) -> None:

68 prompts = [

69 PROMPT_TEMPLATE.format(

@@ -106,7 +120,10 @@ def test_gpt_oss_lora(

120 gptoss20b_lora_files,

121 mxfp4_use_marlin,

122 specialize_active_lora,

123 + monkeypatch: pytest.MonkeyPatch,

124 ):

125 + _enable_deterministic_lora_shrink(monkeypatch)

126 +

127 llm = vllm.LLM(

128 MODEL_PATH,

129 max_model_len=1024,

@@ -146,7 +163,10 @@ def test_gpt_oss_lora_tp2(

163 gptoss20b_lora_files,

164 fully_sharded_loras,

165 mxfp4_use_marlin,

166 + monkeypatch: pytest.MonkeyPatch,

167 ):

168 + _enable_deterministic_lora_shrink(monkeypatch)

169 +

170 llm = vllm.LLM(

171 MODEL_PATH,

172 max_model_len=1024,

tests/lora/test_moe_lora_align_sum.py

+21

@@ -40,6 +40,18 @@ def sample_data(num_experts, max_loras, num_tokens, topk_num):

40 @pytest.mark.parametrize("block_size", [16])

41 def test_moe_lora_align_block_size(

42 num_tokens, topk_num, num_experts, max_loras, block_size

43 +):

44 + _run_moe_lora_align_block_size_case(

45 + num_tokens, topk_num, num_experts, max_loras, block_size

46 + )

47 +

48 +

49 +def test_moe_lora_align_block_size_large_path_counts():

50 + _run_moe_lora_align_block_size_case(100, 12, 64, 4, 16)

51 +

52 +

53 +def _run_moe_lora_align_block_size_case(

54 + num_tokens, topk_num, num_experts, max_loras, block_size

55 ):

56 # sample data

57 random.seed(1)

@@ -94,8 +106,17 @@ def test_moe_lora_align_block_size(

106 # verify values

107 expert_ids = expert_ids.view(max_loras, -1)

108 sorted_token_ids = sorted_token_ids.view(max_loras, -1, block_size)

109 + topk_ids_cpu = topk_ids.cpu()

110 + token_lora_mapping_cpu = token_lora_mapping.cpu()

111

112 for lora_idx in range(max_loras):

113 + expert_counts = torch.bincount(

114 + topk_ids_cpu[token_lora_mapping_cpu == lora_idx].flatten(),

115 + minlength=num_experts,

116 + )

117 + expected_post_pad = round_up(expert_counts, block_size).sum().item()

118 + assert num_tokens_post_pad[lora_idx].item() == expected_post_pad

119 +

120 for token_idx in range(sorted_token_ids.size(1)):

121 block = sorted_token_ids[lora_idx][token_idx]

122 indices = block[block != topk_ids.numel()]

tests/model_executor/layers/test_rocm_unquantized_gemm.py

+111 -1

@@ -42,6 +42,113 @@ def test_rocm_unquantized_gemm_gfx1x_wvsplitk_path(monkeypatch):

42 assert torch.allclose(out, ref, atol=1e-3, rtol=1e-3)

43

44

45 +def test_rocm_unquantized_gemm_makes_skinny_activation_contiguous(monkeypatch):

46 + x = torch.randn(64, 4, dtype=torch.float16).t()

47 + weight = torch.randn(128, 64, dtype=torch.float16)

48 + assert x.shape == (4, 64)

49 + assert x.stride() == (1, 4)

50 +

51 + monkeypatch.setattr(utils, "use_aiter_triton_gemm", lambda *args: False)

52 + monkeypatch.setattr(utils.envs, "VLLM_ROCM_USE_SKINNY_GEMM", True)

53 + monkeypatch.setattr("vllm.platforms.rocm.on_gfx1x", lambda: True)

54 + monkeypatch.setattr("vllm.platforms.rocm.on_gfx9", lambda: False)

55 + monkeypatch.setattr("vllm.platforms.rocm.on_gfx950", lambda: False)

56 + monkeypatch.setattr("vllm.platforms.rocm.on_gfx1250", lambda: False)

57 + monkeypatch.setattr(utils, "num_compute_units", lambda: 120)

58 +

59 + wvsplitk_mock = MagicMock(side_effect=lambda w, x_view, _, __: x_view @ w.t())

60 + monkeypatch.setattr(utils.ops, "wvSplitK", wvsplitk_mock)

61 +

62 + out = utils.rocm_unquantized_gemm_impl(x, weight, None)

63 + ref = torch.nn.functional.linear(x, weight, None)

64 +

65 + wvsplitk_mock.assert_called_once()

66 + x_view = wvsplitk_mock.call_args.args[1]

67 + assert x_view.is_contiguous()

68 + assert torch.allclose(out, ref, atol=1e-3, rtol=1e-3)

69 +

70 +

71 +def test_rocm_unquantized_gemm_makes_llmm1_activation_contiguous(monkeypatch):

72 + x = torch.randn(1, 128, dtype=torch.float16)[:, ::2]

73 + weight = torch.randn(4, 64, dtype=torch.float16)

74 + assert x.shape == (1, 64)

75 + assert x.stride() == (128, 2)

76 +

77 + monkeypatch.setattr(utils, "use_aiter_triton_gemm", lambda *args: False)

78 + monkeypatch.setattr(utils.envs, "VLLM_ROCM_USE_SKINNY_GEMM", True)

79 + monkeypatch.setattr("vllm.platforms.rocm.on_gfx1x", lambda: True)

80 + monkeypatch.setattr("vllm.platforms.rocm.on_gfx9", lambda: False)

81 + monkeypatch.setattr("vllm.platforms.rocm.on_gfx950", lambda: False)

82 + monkeypatch.setattr("vllm.platforms.rocm.on_gfx1250", lambda: False)

83 + monkeypatch.setattr(utils, "num_compute_units", lambda: 120)

84 +

85 + llmm1_mock = MagicMock(side_effect=lambda w, x_view, _: x_view @ w.t())

86 + monkeypatch.setattr(utils.ops, "LLMM1", llmm1_mock)

87 +

88 + out = utils.rocm_unquantized_gemm_impl(x, weight, None)

89 + ref = torch.nn.functional.linear(x, weight, None)

90 +

91 + llmm1_mock.assert_called_once()

92 + x_view = llmm1_mock.call_args.args[1]

93 + assert x_view.is_contiguous()

94 + assert torch.allclose(out, ref, atol=1e-3, rtol=1e-3)

95 +

96 +

97 +@pytest.mark.parametrize("noncontiguous_operand", ["weight", "bias"])

98 +def test_rocm_unquantized_gemm_rejects_unsupported_skinny_layouts(

99 + monkeypatch, noncontiguous_operand

100 +):

101 + x = torch.randn(4, 64, dtype=torch.float16)

102 + weight = torch.randn(128, 64, dtype=torch.float16)

103 + bias = torch.randn(128, dtype=torch.float16)

104 + if noncontiguous_operand == "weight":

105 + weight = torch.randn(64, 128, dtype=torch.float16).t()

106 + assert not weight.is_contiguous()

107 + else:

108 + bias = torch.randn(256, dtype=torch.float16)[::2]

109 + assert not bias.is_contiguous()

110 +

111 + monkeypatch.setattr(utils, "use_aiter_triton_gemm", lambda *args: False)

112 + monkeypatch.setattr(utils.rocm_aiter_ops, "is_tgemm_enabled", lambda: False)

113 + monkeypatch.setattr(utils.envs, "VLLM_ROCM_USE_SKINNY_GEMM", True)

114 + monkeypatch.setattr("vllm.platforms.rocm.on_gfx1x", lambda: True)

115 + monkeypatch.setattr("vllm.platforms.rocm.on_gfx9", lambda: False)

116 + monkeypatch.setattr("vllm.platforms.rocm.on_gfx950", lambda: False)

117 + monkeypatch.setattr("vllm.platforms.rocm.on_gfx1250", lambda: False)

118 + monkeypatch.setattr(utils, "num_compute_units", lambda: 120)

119 +

120 + wvsplitk_mock = MagicMock()

121 + monkeypatch.setattr(utils.ops, "wvSplitK", wvsplitk_mock)

122 + llmm1_mock = MagicMock()

123 + monkeypatch.setattr(utils.ops, "LLMM1", llmm1_mock)

124 +

125 + out = utils.rocm_unquantized_gemm_impl(x, weight, bias)

126 + ref = torch.nn.functional.linear(x, weight, bias)

127 +

128 + wvsplitk_mock.assert_not_called()

129 + llmm1_mock.assert_not_called()

130 + assert torch.allclose(out, ref, atol=1e-3, rtol=1e-3)

131 +

132 +

133 +@pytest.mark.skipif(not current_platform.is_rocm(), reason="ROCm-only kernel test")

134 +@pytest.mark.parametrize("dtype", [torch.float16, torch.bfloat16])

135 +def test_rocm_unquantized_gemm_noncontiguous_activation_real_kernel(monkeypatch, dtype):

136 + x = torch.randn(64, 4, device="cuda", dtype=dtype).t()

137 + weight = torch.randn(128, 64, device="cuda", dtype=dtype)

138 + assert x.stride() == (1, 4)

139 +

140 + monkeypatch.setattr(utils.envs, "VLLM_ROCM_USE_SKINNY_GEMM", True)

141 + original_wvsplitk = utils.ops.wvSplitK

142 + wvsplitk_mock = MagicMock(side_effect=original_wvsplitk)

143 + monkeypatch.setattr(utils.ops, "wvSplitK", wvsplitk_mock)

144 +

145 + out = utils.rocm_unquantized_gemm_impl(x, weight, None)

146 + ref = torch.nn.functional.linear(x, weight, None)

147 +

148 + wvsplitk_mock.assert_called_once()

149 + torch.testing.assert_close(out, ref, atol=1e-2, rtol=1e-2)

150 +

151 +

152 def test_rocm_unquantized_gemm_gfx1x_n_gt_5_falls_back(monkeypatch):

153 # wvSplitK skinny GEMM handles n in [1, 5] (see PR #40687); n > 5 must

154 # fall back to torch.nn.functional.linear.

@@ -70,8 +177,9 @@ def test_rocm_unquantized_gemm_gfx1x_n_gt_5_falls_back(monkeypatch):

177

178

179 def test_rocm_unquantized_gemm_gfx950_wvsplitkrc_path(monkeypatch):

73 - x = torch.randn(16, 1024, dtype=torch.float16)

180 + x = torch.randn(1024, 16, dtype=torch.float16).t()

181 weight = torch.randn(256, 1024, dtype=torch.float16)

182 + assert x.stride() == (1, 16)

183

184 monkeypatch.setattr(utils, "use_aiter_triton_gemm", lambda *args: False)

185 monkeypatch.setattr(utils.envs, "VLLM_ROCM_USE_SKINNY_GEMM", True)

@@ -91,4 +199,6 @@ def test_rocm_unquantized_gemm_gfx950_wvsplitkrc_path(monkeypatch):

199

200 wvsplitkrc_mock.assert_called_once()

201 wvsplitk_mock.assert_not_called()

202 + x_view = wvsplitkrc_mock.call_args.args[0]

203 + assert x_view.is_contiguous()

204 assert torch.allclose(out, ref, atol=1e-3, rtol=1e-3)

tests/models/language/generation/conftest.py deleted

-46

@@ -1,46 +0,0 @@

1 -# SPDX-License-Identifier: Apache-2.0

2 -# SPDX-FileCopyrightText: Copyright contributors to the vLLM project

3 -"""Pytest configuration for vLLM language generation tests."""

4 -

5 -import os

6 -import warnings

7 -

8 -import torch

9 -

10 -from vllm.platforms import current_platform

11 -

12 -

13 -def pytest_configure(config):

14 - """Early ROCm configuration that must happen before test collection."""

15 - if not current_platform.is_rocm():

16 - return

17 -

18 - # Disable skinny GEMM on ROCm to avoid non-deterministic results

19 - # from atomic reductions in wvSplitKrc kernel.

20 - # See: https://github.com/vllm-project/vllm/pull/33493#issuecomment-3906083975

21 - os.environ["VLLM_ROCM_USE_SKINNY_GEMM"] = "0"

22 - warnings.warn(

23 - "ROCm: Set VLLM_ROCM_USE_SKINNY_GEMM=0 to avoid non-deterministic "

24 - "results from skinny GEMM atomic reductions",

25 - UserWarning,

26 - stacklevel=1,

27 - )

28 -

29 -

30 -def pytest_sessionstart(session):

31 - """Configure ROCm-specific settings before test session starts."""

32 - if not current_platform.is_rocm():

33 - return

34 -

35 - # Disable Flash/MemEfficient SDP on ROCm to avoid HF Transformers

36 - # accuracy issues: https://github.com/vllm-project/vllm/issues/30167

37 - # TODO: Remove once ROCm SDP accuracy issues are resolved on HuggingFace

38 - torch.backends.cuda.enable_flash_sdp(False)

39 - torch.backends.cuda.enable_mem_efficient_sdp(False)

40 - torch.backends.cuda.enable_math_sdp(True)

41 - warnings.warn(

42 - "ROCm: Disabled flash_sdp and mem_efficient_sdp, enabled math_sdp "

43 - "to avoid HuggingFace Transformers accuracy issues",

44 - UserWarning,

45 - stacklevel=1,

46 - )

tests/models/language/generation/test_common.py

+6

@@ -126,6 +126,12 @@ def test_models(

126 model_info.check_available_online(on_fail="skip")

127 model_info.check_transformers_version(on_fail="skip")

128

129 + if current_platform.is_rocm() and model == "TitanML/tiny-mixtral":

130 + # Its single-token router selects LLMM1, whose low-precision

131 + # accumulation can change the top-2 experts. Keep the optimized kernel

132 + # enabled generally, but use the reference GEMM for this accuracy test.

133 + monkeypatch.setenv("VLLM_ROCM_USE_SKINNY_GEMM", "0")

134 +

135 if use_rocm_aiter and (model in AITER_MODEL_LIST):

136 monkeypatch.setenv("VLLM_ROCM_USE_AITER", "1")

137 elif use_rocm_aiter and model not in AITER_MODEL_LIST:

tests/models/language/pooling/conftest.py

+2 -18

@@ -2,8 +2,6 @@

2 # SPDX-FileCopyrightText: Copyright contributors to the vLLM project

3 """Pytest configuration for vLLM language generation tests."""

4

5 -import warnings

6 -

5 import torch

6

7 from vllm.platforms import current_platform

@@ -11,19 +9,5 @@ from vllm.platforms import current_platform

9

10 def pytest_sessionstart(session):

11 """Configure ROCm-specific settings before test session starts."""

14 - if not current_platform.is_rocm():

15 - return

16 -

17 - # Disable Flash/MemEfficient SDP on ROCm to avoid HF Transformers

18 - # accuracy issues: https://github.com/vllm-project/vllm/issues/30167

19 - # TODO: Remove once ROCm SDP accuracy issues are resolved on HuggingFace

20 - torch.backends.cuda.enable_flash_sdp(False)

21 - torch.backends.cuda.enable_mem_efficient_sdp(False)

22 - torch.backends.cuda.enable_math_sdp(True)

23 - torch.set_float32_matmul_precision("high")

24 - warnings.warn(

25 - "ROCm: Disabled flash_sdp and mem_efficient_sdp, enabled math_sdp "

26 - "to avoid HuggingFace Transformers accuracy issues",

27 - UserWarning,

28 - stacklevel=1,

29 - )

12 + if current_platform.is_rocm():

13 + torch.set_float32_matmul_precision("high")

tests/models/language/pooling/test_gritlm.py

+2 -2

@@ -7,7 +7,7 @@ from scipy.spatial.distance import cosine

7 from vllm import LLM, SamplingParams

8 from vllm.config import ModelConfig

9

10 -from ....utils import ROCM_ENV_OVERRIDES, ROCM_EXTRA_ARGS, RemoteOpenAIServer

10 +from ....utils import ROCM_EXTRA_ARGS, RemoteOpenAIServer

11 from .embed_utils import run_client_embeddings

12

13 MODEL_NAME = "parasail-ai/GritLM-7B-vllm"

@@ -134,7 +134,7 @@ async def test_gritlm_api_server_embedding():

134 *ROCM_EXTRA_ARGS,

135 ]

136

137 - with RemoteOpenAIServer(MODEL_NAME, args, env_dict=ROCM_ENV_OVERRIDES) as server:

137 + with RemoteOpenAIServer(MODEL_NAME, args) as server:

138 client_embedding = server.get_async_client()

139

140 d_rep = await run_client_embeddings(

tests/models/language/pooling/test_token_classification.py

-33

@@ -48,17 +48,10 @@ def test_bert_model_runner_v2(

48 vllm_model.token_classify(prompts) for prompts in prompt_batches

49 ]

50

51 - # Use eager attention on ROCm to avoid HF Transformers flash attention

52 - # accuracy issues: https://github.com/vllm-project/vllm/issues/30167

53 - hf_model_kwargs = {}

54 - if current_platform.is_rocm():

55 - hf_model_kwargs["attn_implementation"] = "eager"

56 -

51 with hf_runner(

52 model,

53 dtype=dtype,

54 auto_cls=AutoModelForTokenClassification,

61 - model_kwargs=hf_model_kwargs,

55 ) as hf_model:

56 tokenizer = hf_model.tokenizer

57 hf_outputs = []

@@ -100,17 +93,10 @@ def test_modernbert_models(

93 with vllm_runner(model, max_model_len=None, dtype=dtype) as vllm_model:

94 vllm_outputs = vllm_model.token_classify(example_prompts)

95

103 - # Use eager attention on ROCm to avoid HF Transformers flash attention

104 - # accuracy issues: https://github.com/vllm-project/vllm/issues/30167

105 - hf_model_kwargs = {}

106 - if current_platform.is_rocm():

107 - hf_model_kwargs["attn_implementation"] = "eager"

108 -

96 with hf_runner(

97 model,

98 dtype=dtype,

99 auto_cls=AutoModelForTokenClassification,

113 - model_kwargs=hf_model_kwargs,

100 ) as hf_model:

101 tokenizer = hf_model.tokenizer

102 hf_outputs = []

@@ -140,17 +126,10 @@ def test_xlm_roberta_models(

126 with vllm_runner(model, max_model_len=None, dtype=dtype) as vllm_model:

127 vllm_outputs = vllm_model.token_classify(example_prompts)

128

143 - # Use eager attention on ROCm to avoid HF Transformers flash attention

144 - # accuracy issues: https://github.com/vllm-project/vllm/issues/30167

145 - hf_model_kwargs = {}

146 - if current_platform.is_rocm():

147 - hf_model_kwargs["attn_implementation"] = "eager"

148 -

129 with hf_runner(

130 model,

131 dtype=dtype,

132 auto_cls=AutoModelForTokenClassification,

153 - model_kwargs=hf_model_kwargs,

133 ) as hf_model:

134 tokenizer = hf_model.tokenizer

135 hf_outputs = []

@@ -197,15 +176,10 @@ def test_openai_privacy_filter(

176 with vllm_runner(model, max_model_len=None, dtype=dtype) as vllm_model:

177 vllm_outputs = vllm_model.token_classify(PRIVACY_FILTER_PROMPTS)

178

200 - hf_model_kwargs = {}

201 - if current_platform.is_rocm():

202 - hf_model_kwargs["attn_implementation"] = "eager"

203 -

179 with hf_runner(

180 model,

181 dtype=dtype,

182 auto_cls=AutoModelForTokenClassification,

208 - model_kwargs=hf_model_kwargs,

183 ) as hf_model:

184 tokenizer = hf_model.tokenizer

185 hf_outputs = []

@@ -276,12 +250,6 @@ def test_bert_for_masked_lm(

250 with vllm_runner(model, max_model_len=None, dtype=dtype) as vllm_model:

251 vllm_outputs = vllm_model.token_classify(example_prompts)

252

279 - # Use eager attention on ROCm to avoid HF Transformers flash attention

280 - # accuracy issues: https://github.com/vllm-project/vllm/issues/30167

281 - hf_model_kwargs = {}

282 - if current_platform.is_rocm():

283 - hf_model_kwargs["attn_implementation"] = "eager"

284 -

253 # Run hf_runner reference with "highest" fp32 precision to match

254 # default behvior of vLLM. This is needed on ROCm since the

255 # pooling tests set matmul precision to "high" in conftest.py

@@ -291,7 +259,6 @@ def test_bert_for_masked_lm(

259 model,

260 dtype=dtype,

261 auto_cls=AutoModelForMaskedLM,

294 - model_kwargs=hf_model_kwargs,

262 ) as hf_model:

263 tokenizer = hf_model.tokenizer

264 hf_outputs = []

tests/models/multimodal/conftest.py

-65

@@ -2,14 +2,8 @@

2 # SPDX-FileCopyrightText: Copyright contributors to the vLLM project

3 """Pytest configuration for vLLM multimodal tests."""

4

5 -import os

6 -import warnings

7 -

8 -import torch

5 import torch.nn.functional as F

6

11 -from vllm.platforms import current_platform

12 -

7 # openbmb/MiniCPM-Llama3-V-2_5's remote ``resampler.py`` annotates with ``List``

8 # without importing it, relying on the alias leaking from

9 # ``from torch.nn.functional import *``. Recent torch uses builtin generics and no

@@ -17,62 +11,3 @@ from vllm.platforms import current_platform

11 # TODO: Remove once the checkpoint fixes its imports.

12 if not hasattr(F, "List"):

13 F.List = list

20 -

21 -

22 -def pytest_configure(config):

23 - """Early ROCm configuration that must happen before test collection."""

24 - if not current_platform.is_rocm():

25 - return

26 -

27 - # Disable skinny GEMM on ROCm to avoid non-deterministic results

28 - # from atomic reductions in wvSplitKrc kernel.

29 - # See: https://github.com/vllm-project/vllm/pull/33493#issuecomment-3906083975

30 - os.environ["VLLM_ROCM_USE_SKINNY_GEMM"] = "0"

31 - warnings.warn(

32 - "ROCm: Set VLLM_ROCM_USE_SKINNY_GEMM=0 to avoid non-deterministic "

33 - "results from skinny GEMM atomic reductions",

34 - UserWarning,

35 - stacklevel=1,

36 - )

37 -

38 -

39 -def pytest_collection_modifyitems(config, items):

40 - """Configure ROCm-specific settings based on collected tests."""

41 - if not current_platform.is_rocm():

42 - return

43 -

44 - skip_patterns = ["test_granite_speech.py"]

45 - if any(pattern in str(arg) for arg in config.args for pattern in skip_patterns):

46 - return

47 -

48 - # Disable Flash/MemEfficient SDP on ROCm to avoid HF Transformers

49 - # accuracy issues: https://github.com/vllm-project/vllm/issues/30167

50 - # TODO: Remove once ROCm SDP accuracy issues are resolved on HuggingFace

51 - torch.backends.cuda.enable_flash_sdp(False)

52 - torch.backends.cuda.enable_mem_efficient_sdp(False)

53 - torch.backends.cuda.enable_math_sdp(True)

54 - warnings.warn(

55 - "ROCm: Disabled flash_sdp and mem_efficient_sdp, enabled math_sdp "

56 - "to avoid HuggingFace Transformers accuracy issues",

57 - UserWarning,

58 - stacklevel=1,

59 - )

60 -

61 -

62 -def patch_hf_vision_attn_for_rocm(model):

63 - """Force SDPA for HF vision encoders on ROCm.

64 -

65 - HF's flash_attention_2 has accuracy issues on ROCm that bypass

66 - torch.backends.cuda settings. This forces SDPA which then uses

67 - math_sdp via the pytest_collection_modifyitems settings.

68 - """

69 - if not current_platform.is_rocm():

70 - return

71 -

72 - inner = getattr(model, "model", model)

73 -

74 - if hasattr(inner, "vision_embedding"):

75 - vit = inner.vision_embedding[0]

76 - for layer in vit.encoder.layers:

77 - if hasattr(layer, "self_attn"):

78 - layer.self_attn.vision_config._attn_implementation = "sdpa"

tests/models/multimodal/generation/test_common.py

+2 -16

@@ -249,10 +249,7 @@ VLM_TEST_SETTINGS = {

249 vllm_runner_kwargs={

250 "model_impl": "transformers",

251 },

252 - marks=[

253 - pytest.mark.core_model,

254 - *([large_gpu_mark(min_gb=80)] if current_platform.is_rocm() else []),

255 - ],

252 + marks=[pytest.mark.core_model],

253 ),

254 "idefics3-transformers": VLMTestInfo(

255 models=["HuggingFaceTB/SmolVLM-256M-Instruct"],

@@ -282,19 +279,8 @@ VLM_TEST_SETTINGS = {

279 image_size_factors=[(0.25, 0.2, 0.15)],

280 vllm_runner_kwargs={

281 "model_impl": "transformers",

285 - # TODO: [ROCm] Revert this once issue #30167 is resolved

286 - **(

287 - {

288 - "mm_processor_kwargs": {

289 - "min_pixels": 256 * 28 * 28,

290 - "max_pixels": 1280 * 28 * 28,

291 - },

292 - }

293 - if current_platform.is_rocm()

294 - else {}

295 - ),

282 },

297 - marks=[large_gpu_mark(min_gb=80 if current_platform.is_rocm() else 32)],

283 + marks=[large_gpu_mark(min_gb=32)],

284 ),

285 #### Extended model tests

286 "aria": VLMTestInfo(

tests/models/multimodal/generation/test_whisper.py

-4

@@ -129,7 +129,6 @@ def check_model_available(model: str) -> None:

129 @pytest.mark.parametrize("max_tokens", [64])

130 @pytest.mark.parametrize("beam_width", [1, 2])

131 def test_beam_search_encoder_decoder(

132 - monkeypatch,

132 hf_runner,

133 vllm_runner,

134 dtype: str,

@@ -138,9 +137,6 @@ def test_beam_search_encoder_decoder(

137 resampled_assets,

138 ) -> None:

139 """Test beam search with encoder-decoder models (Whisper)."""

141 - if current_platform.is_rocm():

142 - monkeypatch.setenv("VLLM_ROCM_USE_SKINNY_GEMM", "0")

143 -

140 model = "openai/whisper-large-v3-turbo"

141 check_model_available(model)

142

tests/models/multimodal/generation/vlm_utils/model_utils.py

-30

@@ -5,9 +5,7 @@ for manipulating the input / output of HF & vLLM test runners, which are

5 typically specific to a small subset of models.

6 """

7

8 -import logging

8 import types

10 -import warnings

9 from pathlib import PosixPath

10

11 import numpy as np

@@ -34,8 +32,6 @@ from vllm.utils.collection_utils import is_list_of

32 from .....conftest import HfRunner, ImageAsset, ImageTestAssets

33 from .types import RunnerOutput

34

37 -logger = logging.getLogger(__name__)

38 -

35

36 ####### vLLM output processors functions

37 def blip2_vllm_to_hf_output(vllm_output: RunnerOutput, model: str) -> RunnerOutput:

@@ -559,32 +555,6 @@ def isaac_patch_hf_runner(hf_model: HfRunner) -> HfRunner:

555 # ----------------------------

556 isaac_model = hf_model.model.model

557

562 - # [ROCm] Disable Flash/MemEfficient SDP on ROCm to avoid HF Transformers

563 - # accuracy issues: https://github.com/vllm-project/vllm/issues/30167

564 - # TODO: Remove once ROCm SDP accuracy issues are resolved on HuggingFace

565 - # ----------------------------

566 - from ...conftest import patch_hf_vision_attn_for_rocm

567 -

568 - try:

569 - patch_hf_vision_attn_for_rocm(hf_model.model)

570 - except AttributeError as e:

571 - if "vision_config" in str(e):

572 - warnings.warn(

573 - f"Skipping ROCm vision attention patch for Isaac model: {e}. "

574 - "This is expected for models without vision_config in "

575 - "attention layers (e.g., Siglip2VariableLengthAttention).",

576 - stacklevel=2,

577 - )

578 - else:

579 - logger.error(

580 - "Unexpected AttributeError during ROCm vision attention patch: %s. "

581 - "Model type: %s. Inner model type: %s.",

582 - e,

583 - type(hf_model.model).__name__,

584 - type(getattr(hf_model.model, "model", None)).__name__,

585 - )

586 - raise

587 -

558 def patched_forward(

559 self,

560 input_ids=None,

tests/models/multimodal/pooling/conftest.py deleted

-18

@@ -1,18 +0,0 @@

1 -# SPDX-License-Identifier: Apache-2.0

2 -# SPDX-FileCopyrightText: Copyright contributors to the vLLM project

3 -"""Pytest configuration for vLLM pooling tests."""

4 -

5 -import pytest

6 -

7 -from vllm.platforms import current_platform

8 -

9 -

10 -@pytest.fixture

11 -def siglip_attention_config():

12 - """Return attention config for SigLIP tests on ROCm.

13 -

14 - On ROCm, SigLIP tests require FLEX_ATTENTION backend.

15 - """

16 - if current_platform.is_rocm():

17 - return {"backend": "FLEX_ATTENTION"}

18 - return None

tests/models/multimodal/pooling/test_siglip.py

-8

@@ -39,7 +39,6 @@ def _run_test(

39 *,

40 dtype: str,

41 tokenization_kwargs: dict[str, Any] | None = None,

42 - attention_config: dict[str, Any] | None = None,

42 ) -> None:

43 if tokenization_kwargs is None:

44 tokenization_kwargs = {}

@@ -51,7 +50,6 @@ def _run_test(

50 enforce_eager=True,

51 max_model_len=64,

52 gpu_memory_utilization=0.7,

54 - attention_config=attention_config,

53 ) as vllm_model:

54 vllm_outputs = vllm_model.embed(

55 input_texts, images=input_images, tokenization_kwargs=tokenization_kwargs

@@ -96,7 +94,6 @@ def test_models_text(

94 hf_runner,

95 vllm_runner,

96 image_assets,

99 - siglip_attention_config,

97 model: str,

98 dtype: str,

99 ) -> None:

@@ -115,7 +112,6 @@ def test_models_text(

112 "padding": "max_length",

113 "max_length": 64,

114 }, # siglip2 was trained with this padding setting.

118 - attention_config=siglip_attention_config,

115 )

116

117

@@ -125,7 +121,6 @@ def test_models_image(

121 hf_runner,

122 vllm_runner,

123 image_assets,

128 - siglip_attention_config,

124 model: str,

125 dtype: str,

126 ) -> None:

@@ -142,7 +137,6 @@ def test_models_image(

137 input_images,

138 model,

139 dtype=dtype,

145 - attention_config=siglip_attention_config,

140 )

141

142

@@ -151,7 +145,6 @@ def test_models_image(

145 def test_models_text_image_no_crash(

146 vllm_runner,

147 image_assets,

154 - siglip_attention_config,

148 model: str,

149 dtype: str,

150 ) -> None:

@@ -165,7 +158,6 @@ def test_models_text_image_no_crash(

158 enforce_eager=True,

159 max_model_len=64,

160 gpu_memory_utilization=0.7,

168 - attention_config=siglip_attention_config,

161 ) as vllm_model:

162 with pytest.raises(ValueError, match="not both"):

163 vllm_model.embed(texts, images=images)

tests/samplers/test_beam_search.py

-12

@@ -50,7 +50,6 @@ MODELS = ["TinyLlama/TinyLlama-1.1B-Chat-v1.0"]

50 @pytest.mark.parametrize("max_tokens", MAX_TOKENS)

51 @pytest.mark.parametrize("beam_width", BEAM_WIDTHS)

52 def test_beam_search_single_input(

53 - monkeypatch,

53 hf_runner,

54 vllm_runner,

55 example_prompts,

@@ -59,9 +58,6 @@ def test_beam_search_single_input(

58 max_tokens: int,

59 beam_width: int,

60 ) -> None:

62 - if current_platform.is_rocm():

63 - monkeypatch.setenv("VLLM_ROCM_USE_SKINNY_GEMM", "0")

64 -

61 example_prompts = example_prompts[:1]

62 with hf_runner(model, dtype=dtype) as hf_model:

63 hf_outputs = hf_model.generate_beam_search(

@@ -95,7 +91,6 @@ def test_beam_search_single_input(

91 @pytest.mark.parametrize("max_tokens", MAX_TOKENS)

92 @pytest.mark.parametrize("beam_width", BEAM_WIDTHS)

93 def test_beam_search_with_concurrency_limit(

98 - monkeypatch,

94 hf_runner,

95 vllm_runner,

96 example_prompts,

@@ -104,9 +99,6 @@ def test_beam_search_with_concurrency_limit(

99 max_tokens: int,

100 beam_width: int,

101 ) -> None:

107 - if current_platform.is_rocm():

108 - monkeypatch.setenv("VLLM_ROCM_USE_SKINNY_GEMM", "0")

109 -

102 # example_prompts[1]&[3]&[7] fails due to unknown reason even without

103 # concurrency limit. skip them for now.

104 example_prompts = example_prompts[:8]

@@ -156,7 +148,6 @@ def test_beam_search_with_concurrency_limit(

148 @pytest.mark.parametrize("max_tokens", MAX_TOKENS)

149 @pytest.mark.parametrize("beam_width", MM_BEAM_WIDTHS)

150 def test_beam_search_passes_multimodal_data(

159 - monkeypatch,

151 hf_runner,

152 vllm_runner,

153 dtype: str,

@@ -164,9 +155,6 @@ def test_beam_search_passes_multimodal_data(

155 beam_width: int,

156 ) -> None:

157 """Ensure that beam search passes multimodal data through correctly."""

167 - if current_platform.is_rocm():

168 - monkeypatch.setenv("VLLM_ROCM_USE_SKINNY_GEMM", "0")

169 -

158 # NOTE - this test is primarily to check that mm data is passed to beams

159 # correctly. As such, we just need to check one extra modality to make

160 # sure things pass through properly.

tests/utils.py

-6

@@ -131,12 +131,6 @@ else:

131 VLLM_PATH = Path(__file__).parent.parent

132 """Path to root of the vLLM repository."""

133

134 -# ROCm: disable skinny GEMM to avoid non-deterministic results from

135 -# atomic reductions in wvSplitKrc kernel.

136 -# See: https://github.com/vllm-project/vllm/pull/33493#issuecomment-3906083975

137 -ROCM_ENV_OVERRIDES = (

138 - {"VLLM_ROCM_USE_SKINNY_GEMM": "0"} if current_platform.is_rocm() else {}

139 -)

134 # ROCm: disable prefix caching and eliminate batch variance to reduce

135 # test flakiness.

136 ROCM_EXTRA_ARGS = (

tests/v1/distributed/test_internal_lb_dp.py

+1 -4

@@ -12,7 +12,7 @@ import pytest

12 import pytest_asyncio

13 import requests

14

15 -from tests.utils import ROCM_ENV_OVERRIDES, RemoteOpenAIServer

15 +from tests.utils import RemoteOpenAIServer

16 from tests.v1.utils import check_request_balancing

17 from vllm.platforms import current_platform

18

@@ -186,7 +186,6 @@ class MultinodeInternalLBServerManager:

186 auto_port=False,

187 env_dict={

188 "VLLM_SERVER_DEV_MODE": "1",

189 - **ROCM_ENV_OVERRIDES,

189 current_platform.device_control_env_var: ",".join(

190 str(current_platform.device_id_to_physical_device_id(i))

191 for i in range(r, r + gpus_per_node)

@@ -308,7 +307,6 @@ class APIOnlyServerManager:

307 auto_port=False,

308 env_dict={

309 "VLLM_SERVER_DEV_MODE": "1",

311 - **ROCM_ENV_OVERRIDES,

310 # No GPUs needed for API-only server

311 },

312 )

@@ -329,7 +327,6 @@ class APIOnlyServerManager:

327 engines_server_args,

328 auto_port=False,

329 env_dict={

332 - **ROCM_ENV_OVERRIDES,

330 current_platform.device_control_env_var: ",".join(

331 str(current_platform.device_id_to_physical_device_id(i))

332 for i in range(self.dp_size * self.tp_size)

tests/v1/kv_connector/nixl_integration/test_accuracy.py

+17 -4

@@ -38,6 +38,19 @@ SIMPLE_PROMPT = (

38 MODEL_NAME = os.environ.get("TEST_MODEL", "Qwen/Qwen3-0.6B")

39

40

41 +def _meets_accuracy_threshold(measured_value: float, expected_value: float) -> bool:

42 + return measured_value >= expected_value - RTOL

43 +

44 +

45 +def test_accuracy_threshold_is_one_sided():

46 + expected_value = 0.77

47 + minimum_value = expected_value - RTOL

48 +

49 + assert _meets_accuracy_threshold(minimum_value, expected_value)

50 + assert _meets_accuracy_threshold(expected_value + RTOL + 0.01, expected_value)

51 + assert not _meets_accuracy_threshold(minimum_value - 0.01, expected_value)

52 +

53 +

54 def run_simple_prompt():

55 client = openai.OpenAI(api_key="EMPTY", base_url=BASE_URL)

56 completion = client.completions.create(model=MODEL_NAME, prompt=SIMPLE_PROMPT)

@@ -93,7 +106,7 @@ def test_accuracy():

106 )

107 return

108

96 - assert (

97 - measured_value - RTOL < expected_value

98 - and measured_value + RTOL > expected_value

99 - ), f"Expected: {expected_value} | Measured: {measured_value}"

109 + minimum_value = expected_value - RTOL

110 + assert _meets_accuracy_threshold(measured_value, expected_value), (

111 + f"Expected at least: {minimum_value} | Measured: {measured_value}"

112 + )

tests/v1/kv_offload/cpu/test_gpu_worker.py

+70

@@ -18,6 +18,7 @@ from vllm.v1.kv_offload.base import (

18 CanonicalKVCaches,

19 CanonicalKVCacheTensor,

20 GPULoadStoreSpec,

21 + TransferResult,

22 )

23 from vllm.v1.kv_offload.cpu import gpu_worker

24 from vllm.v1.kv_offload.cpu.common import CPULoadStoreSpec

@@ -612,3 +613,72 @@ def test_transfer_multi_group(

613 )

614

615 worker.shutdown()

616 +

617 +

618 +@pytest.mark.skipif(

619 + not current_platform.is_cuda_alike(),

620 + reason="stream ordering test requires a CUDA-like platform",

621 +)

622 +@torch.inference_mode()

623 +def test_load_waits_for_pending_compute_stream_writes(default_vllm_config) -> None:

624 + """A CPU load must land after pending writes to its GPU destination."""

625 + device = DEVICES[0]

626 + page_size_bytes = 128 * 1024

627 + num_blocks = 64

628 + loaded_blocks = list(range(32))

629 + sentinel = 0x5A

630 +

631 + gpu_tensor = torch.zeros(

632 + (num_blocks, page_size_bytes), dtype=torch.int8, device=device

633 + )

634 + loaded_block_ids = torch.tensor(loaded_blocks, dtype=torch.long, device=device)

635 + worker = CPUOffloadingWorker(

636 + kv_caches=CanonicalKVCaches(

637 + tensors=[

638 + CanonicalKVCacheTensor(

639 + tensor=gpu_tensor, page_size_bytes=page_size_bytes

640 + )

641 + ],

642 + group_data_refs=[

643 + [CanonicalKVCacheRef(tensor_idx=0, page_size_bytes=page_size_bytes)]

644 + ],

645 + ),

646 + blocks_per_chunk=1,

647 + num_cpu_blocks=num_blocks,

648 + )

649 + worker._load_handler.src_tensors[0].fill_(sentinel)

650 + expected = torch.full((page_size_bytes,), sentinel, dtype=torch.int8)

651 +

652 + try:

653 + for trial in range(3):

654 + gpu_tensor.fill_(0x11)

655 + torch.accelerator.synchronize()

656 +

657 + # Model a delayed zero of a freshly allocated KV block. Without a

658 + # compute-stream dependency, the DMA can finish during the sleep

659 + # and this later fill wipes out the loaded cache contents.

660 + torch.cuda._sleep(50_000_000)

661 + gpu_tensor.index_fill_(0, loaded_block_ids, 0)

662 +

663 + assert worker.submit_load(

664 + trial + 1,

665 + CPULoadStoreSpec(loaded_blocks),

666 + GPULoadStoreSpec(

667 + loaded_blocks,

668 + group_sizes=(len(loaded_blocks),),

669 + block_indices=(0,),

670 + ),

671 + )

672 + deadline = time.time() + 10

673 + finished: list[TransferResult] = []

674 + while time.time() < deadline and not finished:

675 + finished = worker.get_finished()

676 + if not finished:

677 + time.sleep(0.001)

678 + assert finished and finished[0].success, f"load {trial} did not finish"

679 +

680 + torch.accelerator.synchronize()

681 + for block_id in loaded_blocks:

682 + torch.testing.assert_close(gpu_tensor[block_id].cpu(), expected)

683 + finally:

684 + worker.shutdown()

tests/v1/sample/test_logprobs.py

-12

@@ -1114,7 +1114,6 @@ def test_correct_decoded_token_preserves_valid_tokens():

1114 def test_spec_decode_logprobs(

1115 logprobs_mode: LogprobsMode,

1116 model_setup: tuple[str, str, dict, int],

1117 - monkeypatch,

1117 ):

1118 """Spec decode logprobs should match those of the base model.

1119

@@ -1127,20 +1126,9 @@ def test_spec_decode_logprobs(

1126 logprobs_mode: logprobs mode.

1127 model_setup: Tuple of (method, base model name,

1128 speculative_config dict, top_logprobs).

1130 - monkeypatch: pytest fixture for setting env vars.

1129 """

1130 from vllm import LLM

1131

1134 - # The ROCm skinny GEMM kernels (gemm_kernels.cu) are

1135 - # non-deterministic across LLM instantiations due to persistent

1136 - # workgroup scheduling and wave-level shuffle reductions, which

1137 - # causes logprob differences that get misattributed to spec decode.

1138 - # Disable them so this test isolates spec decode correctness only.

1139 - # TODO(akaratza): Remove this workaround once the follow-up to

1140 - # https://github.com/vllm-project/vllm/pull/33493#issuecomment-3906083975

1141 - # lands with a determinism fix for wvSplitK kernels.

1142 - monkeypatch.setenv("VLLM_ROCM_USE_SKINNY_GEMM", "0")

1143 -

1132 method, model_name, spec_config, top_logprobs = model_setup

1133

1134 prompt = "Hello world " * 50

tests/v1/worker/test_gpu_model_runner.py

+60

@@ -1,6 +1,7 @@

1 # SPDX-License-Identifier: Apache-2.0

2 # SPDX-FileCopyrightText: Copyright contributors to the vLLM project

3

4 +import gc

5 from types import SimpleNamespace

6 from unittest.mock import Mock

7

@@ -126,6 +127,34 @@ def get_vllm_config():

127 return vllm_config

128

129

130 +@pytest.mark.parametrize("gc_initially_enabled", [True, False])

131 +def test_freeze_gc_disables_and_restores_automatic_gc(

132 + monkeypatch: pytest.MonkeyPatch,

133 + gc_initially_enabled: bool,

134 +):

135 + monkeypatch.setenv("VLLM_ENABLE_CUDAGRAPH_GC", "0")

136 + original_gc_state = gc.isenabled()

137 + try:

138 + if gc_initially_enabled:

139 + gc.enable()

140 + else:

141 + gc.disable()

142 +

143 + with (

144 + pytest.raises(RuntimeError, match="capture failed"),

145 + GPUModelRunner._freeze_gc(),

146 + ):

147 + assert not gc.isenabled()

148 + raise RuntimeError("capture failed")

149 +

150 + assert gc.isenabled() is gc_initially_enabled

151 + finally:

152 + if original_gc_state:

153 + gc.enable()

154 + else:

155 + gc.disable()

156 +

157 +

158 @pytest.fixture

159 def model_runner():

160 vllm_config = get_vllm_config()

@@ -896,6 +925,37 @@ def test_sample_passes_reordered_draft_probs_to_rejection_sampler():

925 assert torch.equal(passed_draft_probs, expected_draft_probs)

926

927

928 +def test_dummy_sampler_run_warms_all_greedy_rejection_sampler(monkeypatch):

929 + runner = object.__new__(GPUModelRunner)

930 + runner.device = torch.device("cpu")

931 + runner.vllm_config = SimpleNamespace(

932 + model_config=SimpleNamespace(multimodal_config=None)

933 + )

934 + runner.model = SimpleNamespace(compute_logits=Mock(return_value=torch.randn(3, 8)))

935 + runner.sampler = Mock(return_value="sampler_output")

936 + runner.sampler.logprobs_mode = "processed_logprobs"

937 + runner.speculative_config = SimpleNamespace(

938 + rejection_sample_method="standard",

939 + draft_sample_method="greedy",

940 + )

941 + runner.rejection_sampler = Mock()

942 + synchronize = Mock()

943 + monkeypatch.setattr(torch.accelerator, "synchronize", synchronize)

944 +

945 + output = GPUModelRunner._dummy_sampler_run(runner, torch.randn(3, 4))

946 +

947 + assert output == "sampler_output"

948 + assert runner.rejection_sampler.call_count == 2

949 + mixed_metadata = runner.rejection_sampler.call_args_list[0].args[3]

950 + all_greedy_metadata = runner.rejection_sampler.call_args_list[1].args[3]

951 + assert not mixed_metadata.all_greedy

952 + assert mixed_metadata.temperature is not None

953 + assert all_greedy_metadata.all_greedy

954 + assert not all_greedy_metadata.all_random

955 + assert all_greedy_metadata.temperature is None

956 + synchronize.assert_called_once_with()

957 +

958 +

959 def test_invalid_draft_suffixes_remain_rejected_in_metadata():

960 runner = object.__new__(GPUModelRunner)

961 runner.device = torch.device("cpu")

vllm/_aiter_ops.py

+82

@@ -1,12 +1,15 @@

1 # SPDX-License-Identifier: Apache-2.0

2 # SPDX-FileCopyrightText: Copyright contributors to the vLLM project

3 +import ctypes

4 import functools

5 +import os

6 from collections.abc import Callable

7

8 import torch

9 from torch._ops import OpOverload

10

11 import vllm.envs as envs

12 +from vllm.logger import init_logger

13 from vllm.platforms import current_platform

14 from vllm.utils.import_utils import PlaceholderModule

15 from vllm.utils.torch_utils import direct_register_custom_op

@@ -15,6 +18,8 @@ from vllm.v1.attention.ops.rocm_aiter_mla_sparse import (

18 rocm_aiter_sparse_attn_indexer_fake,

19 )

20

21 +logger = init_logger(__name__)

22 +

23 try:

24 import pandas as pd

25 except ImportError:

@@ -49,6 +54,83 @@ def is_aiter_found() -> bool:

54 IS_AITER_FOUND = is_aiter_found()

55

56

57 +class _DlInfo(ctypes.Structure):

58 + _fields_ = [

59 + ("dli_fname", ctypes.c_char_p),

60 + ("dli_fbase", ctypes.c_void_p),

61 + ("dli_sname", ctypes.c_char_p),

62 + ("dli_saddr", ctypes.c_void_p),

63 + ]

64 +

65 +

66 +_HIP_RUNTIME_HANDLE: ctypes.CDLL | None = None

67 +

68 +

69 +def _find_torch_hip_runtime_path() -> str | None:

70 + """Find the HIP runtime already selected by PyTorch's extension graph."""

71 + try:

72 + torch_extension = ctypes.CDLL(torch._C.__file__)

73 + hip_symbol = torch_extension.hipGetDeviceCount

74 + dladdr = ctypes.CDLL(None).dladdr

75 + dladdr.argtypes = [ctypes.c_void_p, ctypes.POINTER(_DlInfo)]

76 + dladdr.restype = ctypes.c_int

77 + info = _DlInfo()

78 + if not dladdr(ctypes.cast(hip_symbol, ctypes.c_void_p), ctypes.byref(info)):

79 + return None

80 + except (AttributeError, OSError, TypeError, ctypes.ArgumentError):

81 + return None

82 + if info.dli_fname is None:

83 + return None

84 +

85 + runtime_path = os.fsdecode(info.dli_fname)

86 + if "libamdhip64.so" not in os.path.basename(runtime_path):

87 + return None

88 + return runtime_path

89 +

90 +

91 +def _promote_torch_hip_runtime() -> None:

92 + """Make PyTorch's HIP runtime symbols visible to AITER JIT extensions.

93 +

94 + PyTorch loads its native extension with ``RTLD_LOCAL``. Some AITER JIT

95 + extensions rely on HIP registration and launch symbols being available in

96 + the process-global lookup scope. Reopening the exact runtime that provides

97 + PyTorch's ``hipGetDeviceCount`` symbol promotes the existing DSO instead of

98 + selecting or initializing a second HIP runtime.

99 + """

100 + global _HIP_RUNTIME_HANDLE

101 + if _HIP_RUNTIME_HANDLE is not None:

102 + return

103 +

104 + runtime_path = _find_torch_hip_runtime_path()

105 + if runtime_path is None:

106 + logger.warning_once(

107 + "Could not locate PyTorch's HIP runtime; AITER JIT extensions may "

108 + "fail to resolve HIP symbols."

109 + )

110 + return

111 +

112 + try:

113 + # Retain the handle for the process lifetime. Besides preventing a

114 + # future dlclose, this documents that the RTLD_GLOBAL promotion is

115 + # intentional and must outlive every lazily imported AITER extension.

116 + _HIP_RUNTIME_HANDLE = ctypes.CDLL(runtime_path, mode=ctypes.RTLD_GLOBAL)

117 + except OSError as err:

118 + logger.warning_once(

119 + "Could not promote PyTorch's HIP runtime at %s; AITER JIT "

120 + "extensions may fail to resolve HIP symbols: %s",

121 + runtime_path,

122 + err,

123 + )

124 +

125 +

126 +def _maybe_promote_torch_hip_runtime() -> None:

127 + if current_platform.is_rocm() and IS_AITER_FOUND:

128 + _promote_torch_hip_runtime()

129 +

130 +

131 +_maybe_promote_torch_hip_runtime()

132 +

133 +

134 def is_aiter_found_and_supported() -> bool:

135 """Check if AITER library is available and platform supports it.

136

vllm/model_executor/layers/utils.py

+12 -4

@@ -134,12 +134,16 @@ def rocm_unquantized_gemm_impl(

134 GrpsShrB = min(N_p2 // 16, 4)

135 # Given the above, how many CUs would we need?

136 CuNeeded = rndup_cus * GrpsShrB

137 - # candidate for atomic reduce count splitk?

137 + # Deterministic reduction stores one float workspace value per K shard.

138 fits_wvsplitkrc = (

139 N_p2 * m * ((k + 512 - 1) // 512)

140 ) <= 128 * 1024 * 12 # deterministic

141 fits_wvsplitkrc &= CuNeeded <= cu_count

142

143 + skinny_operands_compatible = weight.is_contiguous() and (

144 + bias is None or bias.is_contiguous()

145 + )

146 +

147 use_skinny_reduce_counting = (

148 envs.VLLM_ROCM_USE_SKINNY_GEMM

149 and on_gfx950()

@@ -151,12 +155,13 @@ def rocm_unquantized_gemm_impl(

155 and k > 512

156 and m % 16 == 0

157 and fits_wvsplitkrc

154 - and weight.is_contiguous()

158 + and skinny_operands_compatible

159 )

160 )

161

162 if use_skinny_reduce_counting:

159 - return ops.wvSplitKrc(x, weight, cu_count, bias)

163 + x_view = x.reshape(-1, x.size(-1)).contiguous()

164 + return ops.wvSplitKrc(x_view, weight, cu_count, bias)

165

166 # gfx1250's aiter gemm_a16w16 uses the gluon backend, which requires

167 # K % 256 == 0 (it walks K with fixed-size descriptors and won't pad a

@@ -174,10 +179,13 @@ def rocm_unquantized_gemm_impl(

179 # TODO GFX1250: Include once skinny GEMM is supported on gfx1250

180 and x.dtype in [torch.float16, torch.bfloat16]

181 and k % 8 == 0

182 + and skinny_operands_compatible

183 )

184

185 if use_skinny:

180 - x_view = x.reshape(-1, x.size(-1))

186 + # The skinny kernels assume contiguous K elements. A shape-preserving

187 + # reshape can retain a transposed activation's non-contiguous strides.

188 + x_view = x.reshape(-1, x.size(-1)).contiguous()

189 if m > 8 and 0 < n <= 5:

190 cu_count = num_compute_units()

191 out = ops.wvSplitK(weight, x_view, cu_count, bias)

vllm/model_executor/models/transformers/multimodal.py

+5 -28

@@ -18,7 +18,7 @@

18

19 from collections import defaultdict

20 from collections.abc import Mapping, Sequence

21 -from contextlib import ExitStack, contextmanager, nullcontext

21 +from contextlib import ExitStack, contextmanager

22 from typing import TYPE_CHECKING, Any

23

24 import torch

@@ -53,7 +53,6 @@ from vllm.multimodal.processing import (

53 PromptUpdate,

54 TimingContext,

55 )

56 -from vllm.platforms import current_platform

56 from vllm.sequence import IntermediateTensors

57

58 if TYPE_CHECKING:

@@ -727,15 +726,9 @@ class MultiModalMixin(SupportsMultiModal, SupportsMRoPE):

726 kwargs.pop("token_type_ids", None)

727 kwargs.pop("mm_token_type_ids", None)

728

730 - context = nullcontext()

731 - if current_platform.is_rocm():

732 - context = torch.nn.attention.sdpa_kernel(

733 - backends=[torch.nn.attention.SDPBackend.MATH]

734 - )

735 - with context:

736 - audio_output = self.model.get_audio_features(

737 - input_features, return_dict=True, **kwargs

738 - )

729 + audio_output = self.model.get_audio_features(

730 + input_features, return_dict=True, **kwargs

731 + )

732 audio_embeddings = audio_output.pooler_output

733

734 split_sizes = num_audio_tokens.flatten().tolist()

@@ -756,23 +749,7 @@ class MultiModalMixin(SupportsMultiModal, SupportsMRoPE):

749

750 num_image_patches = kwargs.pop("num_image_patches")

751

759 - context = nullcontext()

760 - if current_platform.is_rocm():

761 - # ROCm: Force math SDP backend for vision encoder to avoid accuracy issues

762 - # with flash_sdp and mem_efficient_sdp

763 - # TODO: [ROCm] Fix accuracy issues with flash backend

764 - logger.debug(

765 - "ROCm platform detected. Forcing math SDP backend "

766 - "for vision encoder. Currently ROCm platform has "

767 - "accuracy issues with `flash_sdp` and"

768 - "`mem_efficient_sdp` backends. See issue: "

769 - "https://github.com/vllm-project/vllm/issues/30167"

770 - )

771 - context = torch.nn.attention.sdpa_kernel(

772 - backends=[torch.nn.attention.SDPBackend.MATH]

773 - )

774 - with context:

775 - vision_embeddings = self.model.get_image_features(pixel_values, **kwargs)

752 + vision_embeddings = self.model.get_image_features(pixel_values, **kwargs)

753

754 # Transformers `v5`, `self.get_image_features` returns a tuple

755 # containing the features and optionally attentions/hidden_states

vllm/v1/core/kv_cache_utils.py

+1 -2

@@ -619,8 +619,7 @@ def resolve_kv_cache_block_sizes(

619 ``cache_config.prefix_match_unit`` override if set, else the GCD of

620 group block sizes; every group's block size must be divisible by it.

621 Returns the scheduler block size (i.e. disables finer hashing) if block

622 - hashing is inactive or a mamba group's block size diverges from the

623 - cache block size (mamba_cache_mode != "align").

622 + hashing is inactive or a mamba group is not using cache mode "align".

623 """

624 cache_config = vllm_config.cache_config

625 dcp = vllm_config.parallel_config.decode_context_parallel_size

vllm/v1/kv_offload/cpu/gpu_worker.py

+5 -3

@@ -634,9 +634,11 @@ class SingleDirectionOffloadingHandler:

634 else torch.Event(enable_timing=True)

635 )

636

637 - if self.gpu_to_cpu:

638 - # wait for model computation to finish before offloading

639 - stream.wait_stream(current_platform.current_stream())

637 + # Stores must wait for the model to finish writing the KV they read.

638 + # Loads must wait for pending writes (including zeroing) to their

639 + # destination blocks; otherwise an earlier transfer can be overwritten

640 + # by compute-stream work that was already queued when the load began.

641 + stream.wait_stream(current_platform.current_stream())

642 if self._transfers:

643 last_transfer: Transfer = self._transfers[-1]

644 last_event = last_transfer.end_event

vllm/v1/worker/gpu_model_runner.py

+27 -2

@@ -6420,6 +6420,21 @@ class GPUModelRunner(

6420 logits,

6421 dummy_metadata,

6422 )

6423 + # All-greedy is a separate Triton specialization from the

6424 + # mixed-sampling path above. Compile and exercise it before serving.

6425 + all_greedy_metadata = replace(

6426 + dummy_metadata,

6427 + temperature=None,

6428 + all_greedy=True,

6429 + all_random=False,

6430 + )

6431 + self.rejection_sampler(

6432 + dummy_spec_decode_metadata,

6433 + draft_probs,

6434 + logits,

6435 + all_greedy_metadata,

6436 + )

6437 + torch.accelerator.synchronize()

6438 return sampler_output

6439

6440 def _dummy_pooler_run_task(

@@ -6615,16 +6630,26 @@ class GPUModelRunner(

6630 @staticmethod

6631 @contextmanager

6632 def _freeze_gc():

6633 + gc_was_enabled = gc.isenabled()

6634 gc.collect()

6635 should_freeze = not envs.VLLM_ENABLE_CUDAGRAPH_GC

6636 if should_freeze:

6637 gc.freeze()

6638 + # A Triton kernel finalized during stream capture unloads its

6639 + # module and invalidates the captured graph.

6640 + gc.disable()

6641 try:

6642 yield

6643 finally:

6644 if should_freeze:

6626 - gc.unfreeze()

6627 - gc.collect()

6645 + try:

6646 + gc.unfreeze()

6647 + gc.collect()

6648 + finally:

6649 + if gc_was_enabled:

6650 + gc.enable()

6651 + else:

6652 + gc.disable()

6653

6654 def shutdown(self) -> None:

6655 """Release GPU tensors (model weights, KV caches, workspace) so that

[ROCm] Remove stale SDPA and skinny GEMM workarounds (#50907)