[ROCm][CI] Skip ModernBERT FP8 MTEB test when no FP8 ScaledMM kernel exists (#53589)
Signed-off-by: Djordje Ramic <djoramic@amd.com>
djramic committed
Aug 25, 2026 at 10:07 UTC
d9fbe526c0787eb5e6dd1e3e4d9b88848d21bc6b
2 files changed
+33
-14
.buildkite/test-amd.yaml
+13
-12
@@ -320,18 +320,6 @@ steps:
320
321
#----------------------------------------------------- mi250 · models / language -----------------------------------------------------#
322
323
-- label: ":amd: (MI250) Language Models (MTEB)" # TBD
324
- timeout_in_minutes: 180
325
- mirror_hardwares: [amdexperimental, amdproduction, amdgfx90anightly, amdmi250]
326
- agent_pool: mi250_1
327
- optional: true
328
- working_dir: "/vllm-workspace/tests"
329
- source_file_dependencies:
330
- - vllm/
331
- - tests/models/language/pooling_mteb_test
332
- commands:
333
- - pytest -v -s models/language/pooling_mteb_test
334
-
323
- label: ":amd: (MI250) Language Models (PPL)" # TBD
324
timeout_in_minutes: 180
325
mirror_hardwares: [amdexperimental, amdproduction, amdgfx90anightly, amdmi250]
@@ -1961,6 +1949,19 @@ steps:
1949
1950
#----------------------------------------------------- mi300 · models / language -----------------------------------------------------#
1951
1952
+- label: ":amd: (MI300) Language Models (MTEB)" # TBD
1953
+ timeout_in_minutes: 180
1954
+ mirror_hardwares: [amdexperimental, amdproduction, amdgfx942nightly, amdmi300]
1955
+ dind: false
1956
+ agent_pool: mi300_1
1957
+ optional: true
1958
+ working_dir: "/vllm-workspace/tests"
1959
+ source_file_dependencies:
1960
+ - vllm/
1961
+ - tests/models/language/pooling_mteb_test
1962
+ commands:
1963
+ - pytest -v -s models/language/pooling_mteb_test
1964
+
1965
- label: ":amd: (MI300) Language Models (Extended Pooling)" # TBD
1966
timeout_in_minutes: 180
1967
mirror_hardwares: [amdexperimental, amdproduction, amdgfx942nightly, amdmi300]
tests/models/language/pooling_mteb_test/test_modernbert_fp8.py
+20
-2
@@ -5,9 +5,27 @@ import pytest
5
6
from tests.models.utils import EmbedModelInfo
7
from tests.quantization.utils import is_quant_method_supported
8
+from vllm.platforms import current_platform
9
10
from .mteb_embed_utils import mteb_test_embed_models
11
12
+
13
+def _fp8_scaled_mm_unsupported() -> bool:
14
+ """Whether the ModernBERT online FP8 test should be skipped.
15
+
16
+ ``is_quant_method_supported("fp8")`` returns True on MI250 (gfx90a), but
17
+ serving a per-tensor FP8 linear layer on ROCm requires CDNA3+ or RDNA4,
18
+ so treat it as unsupported here.
19
+ """
20
+ if not is_quant_method_supported("fp8"):
21
+ return True
22
+ if current_platform.is_rocm():
23
+ from vllm.platforms.rocm import get_cdna_version, on_gfx12x
24
+
25
+ return get_cdna_version() <= 2 and not on_gfx12x()
26
+ return False
27
+
28
+
29
MODEL_INFO = EmbedModelInfo(
30
"Alibaba-NLP/gte-modernbert-base",
31
mteb_score=0.748193353,
@@ -39,8 +57,8 @@ def _assert_modernbert_online_fp8(model) -> None:
57
58
59
@pytest.mark.skipif(
42
- not is_quant_method_supported("fp8"),
43
- reason="FP8 is not supported on this GPU type.",
60
+ _fp8_scaled_mm_unsupported(),
61
+ reason="No FP8 ScaledMM kernel is available on this GPU type.",
62
)
63
def test_modernbert_online_fp8_mteb(hf_runner, vllm_runner, monkeypatch) -> None:
64
monkeypatch.setenv("VLLM_ALLOW_INSECURE_SERIALIZATION", "1")