A high-throughput and memory-efficient inference and serving engine for LLMs
Mirrored from https://github.com/tduncan-lum/Nvidia-Optimization-Stack---VLLM.git
Synced Nov 10, 06:30 UTC
[ROCm][CI] Fix HuggingFace flash_attention_2 accuracy issue in Isaac vision encoder (#32233)
Signed-off-by: Andreas Karatzas <akaratza@amd.com>
Andreas Karatzas committed
Jan 13, 2026 at 00:33 UTC
5e714f7ff4166d62b5f923766a0268a1758f2a61
2 files changed
+27
tests/models/multimodal/conftest.py
+19
@@ -30,3 +30,22 @@ def pytest_collection_modifyitems(config, items):
30
UserWarning,
31
stacklevel=1,
32
)
33
+
34
+
35
+def patch_hf_vision_attn_for_rocm(model):
36
+ """Force SDPA for HF vision encoders on ROCm.
37
+
38
+ HF's flash_attention_2 has accuracy issues on ROCm that bypass
39
+ torch.backends.cuda settings. This forces SDPA which then uses
40
+ math_sdp via the pytest_collection_modifyitems settings.
41
+ """
42
+ if not current_platform.is_rocm():
43
+ return
44
+
45
+ inner = getattr(model, "model", model)
46
+
47
+ if hasattr(inner, "vision_embedding"):
48
+ vit = inner.vision_embedding[0]
49
+ for layer in vit.encoder.layers:
50
+ if hasattr(layer, "self_attn"):
51
+ layer.self_attn.vision_config._attn_implementation = "sdpa"
tests/models/multimodal/generation/vlm_utils/model_utils.py
+8
@@ -576,6 +576,14 @@ def isaac_patch_hf_runner(hf_model: HfRunner) -> HfRunner:
576
# ----------------------------
577
isaac_model = hf_model.model.model
578
579
+ # [ROCm] Disable Flash/MemEfficient SDP on ROCm to avoid HF Transformers
580
+ # accuracy issues: https://github.com/vllm-project/vllm/issues/30167
581
+ # TODO: Remove once ROCm SDP accuracy issues are resolved on HuggingFace
582
+ # ----------------------------
583
+ from ...conftest import patch_hf_vision_attn_for_rocm
584
+
585
+ patch_hf_vision_attn_for_rocm(hf_model.model)
586
+
587
def patched_forward(
588
self,
589
input_ids=None,