A high-throughput and memory-efficient inference and serving engine for LLMs
Mirrored from https://github.com/tduncan-lum/Nvidia-Optimization-Stack---VLLM.git
Synced Nov 11, 08:45 UTC
[ROCm] [Bugfix] Fix torch sdpa hallucination (#30789)
Signed-off-by: tjtanaa <tunjian.tan@embeddedllm.com>
TJian committed
Dec 17, 2025 at 07:32 UTC
2410132bb1f9faa5b252fad3f2b83dc926946b08
1 file changed
+8
vllm/attention/ops/vit_attn_wrappers.py
+8
@@ -16,6 +16,7 @@ import einops
16
import torch
17
import torch.nn.functional as F
18
19
+from vllm.platforms import current_platform
20
from vllm.utils.torch_utils import direct_register_custom_op
21
22
@@ -89,6 +90,13 @@ def torch_sdpa_wrapper(
90
v: torch.Tensor,
91
cu_seqlens: torch.Tensor,
92
) -> torch.Tensor:
93
+ # Never remove the contiguous logic for ROCm
94
+ # Without it, hallucinations occur with the backend
95
+ if current_platform.is_rocm():
96
+ q = q.contiguous()
97
+ k = k.contiguous()
98
+ v = v.contiguous()
99
+
100
outputs = []
101
102
lens = (cu_seqlens[1:] - cu_seqlens[:-1]).tolist()