A high-throughput and memory-efficient inference and serving engine for LLMs
Mirrored from https://github.com/tduncan-lum/Nvidia-Optimization-Stack---VLLM.git
Synced Nov 13, 20:45 UTC
[Bugfix][ROCm] Fix ViT rotary embeddings for torch.compile compatibility on ROCm (#27748)
Signed-off-by: vllmellm <vllm.ellm@embeddedllm.com>
vllmellm committed
Nov 4, 2025 at 09:12 UTC
b13a44754674a0056d7c8113deb33ea858f6ef1c
2 files changed
+8
-5
vllm/model_executor/layers/rotary_embedding/common.py
+7
-4
@@ -77,7 +77,11 @@ def dispatch_rotary_emb_function(
77
if current_platform.is_cuda():
78
return apply_rotary_emb
79
80
- if current_platform.is_rocm():
80
+ # if torch compile is not enabled
81
+ # use rotary embedding function from flash_attn package
82
+ # otherwise use the naive pytorch embedding implementation
83
+ # is faster when torch compile is enabled.
84
+ if current_platform.is_rocm() and not torch.compiler.is_compiling():
85
if find_spec("flash_attn") is not None:
86
from flash_attn.ops.triton.rotary import apply_rotary
87
@@ -87,11 +91,10 @@ def dispatch_rotary_emb_function(
91
"flash_attn is not installed. Falling back to PyTorch "
92
"implementation for rotary embeddings."
93
)
90
-
94
if default is not None:
95
return default
93
- else:
94
- return apply_rotary_emb_torch
96
+
97
+ return apply_rotary_emb_torch
98
99
100
# yarn functions
vllm/model_executor/models/glm4_1v.py
+1
-1
@@ -370,7 +370,7 @@ class Glm4vVisionAttention(nn.Module):
370
cu_seqlens_k=cu_seqlens,
371
max_seqlen_q=max_seqlen,
372
max_seqlen_k=max_seqlen,
373
- dropout_p=0,
373
+ dropout_p=0.0,
374
causal=False,
375
)
376