A high-throughput and memory-efficient inference and serving engine for LLMs
Mirrored from https://github.com/tduncan-lum/Nvidia-Optimization-Stack---VLLM.git
Synced Nov 10, 20:15 UTC
[ROCm][CI] Fix ModernBERT token classification test (#31612)
Signed-off-by: Andreas Karatzas <akaratza@amd.com>
Andreas Karatzas committed
Jan 1, 2026 at 22:19 UTC
013b54088c2be9a485ad7d3df2925c983b56ac9b
1 file changed
+18
-7
tests/models/language/pooling/test_token_classification.py
+18
-7
@@ -34,8 +34,8 @@ def test_bert_models(
34
35
# check logits difference
36
for hf_output, vllm_output in zip(hf_outputs, vllm_outputs):
37
- hf_output = torch.tensor(hf_output).cpu().float()
38
- vllm_output = torch.tensor(vllm_output).cpu().float()
37
+ hf_output = hf_output.detach().clone().cpu().float()
38
+ vllm_output = vllm_output.detach().clone().cpu().float()
39
assert torch.allclose(hf_output, vllm_output, 1e-2)
40
41
@@ -49,11 +49,22 @@ def test_modernbert_models(
49
model: str,
50
dtype: str,
51
) -> None:
52
+ from vllm.platforms import current_platform
53
+
54
with vllm_runner(model, max_model_len=None, dtype=dtype) as vllm_model:
55
vllm_outputs = vllm_model.token_classify(example_prompts)
56
57
+ # Use eager attention on ROCm to avoid HF Transformers flash attention
58
+ # accuracy issues: https://github.com/vllm-project/vllm/issues/30167
59
+ hf_model_kwargs = {}
60
+ if current_platform.is_rocm():
61
+ hf_model_kwargs["attn_implementation"] = "eager"
62
+
63
with hf_runner(
56
- model, dtype=dtype, auto_cls=AutoModelForTokenClassification
64
+ model,
65
+ dtype=dtype,
66
+ auto_cls=AutoModelForTokenClassification,
67
+ model_kwargs=hf_model_kwargs,
68
) as hf_model:
69
tokenizer = hf_model.tokenizer
70
hf_outputs = []
@@ -65,8 +76,8 @@ def test_modernbert_models(
76
77
# check logits difference
78
for hf_output, vllm_output in zip(hf_outputs, vllm_outputs):
68
- hf_output = torch.tensor(hf_output).cpu().float()
69
- vllm_output = torch.tensor(vllm_output).cpu().float()
79
+ hf_output = hf_output.detach().clone().cpu().float()
80
+ vllm_output = vllm_output.detach().clone().cpu().float()
81
assert torch.allclose(hf_output, vllm_output, atol=1e-2)
82
83
@@ -96,6 +107,6 @@ def test_auto_conversion(
107
108
# check logits difference
109
for hf_output, vllm_output in zip(hf_outputs, vllm_outputs):
99
- hf_output = torch.tensor(hf_output).cpu().float()
100
- vllm_output = torch.tensor(vllm_output).cpu().float()
110
+ hf_output = hf_output.detach().clone().cpu().float()
111
+ vllm_output = vllm_output.detach().clone().cpu().float()
112
assert torch.allclose(hf_output, vllm_output, atol=1e-2)