A high-throughput and memory-efficient inference and serving engine for LLMs
Mirrored from https://github.com/tduncan-lum/Nvidia-Optimization-Stack---VLLM.git
Synced Nov 13, 10:45 UTC
[BugFix] Add fallback path in `apply_rotary_pos_emb_flashattn` for non-cuda platforms (#28447)
Signed-off-by: Lin, Fanli <fanli.lin@intel.com>
Fanli Lin committed
Nov 12, 2025 at 11:13 UTC
b9ce9a301341a4f128f434c874f452fb7767e94b
1 file changed
+7
vllm/model_executor/models/keye.py
+7
@@ -346,6 +346,13 @@ def apply_rotary_pos_emb_flashatt(
346
from vllm.vllm_flash_attn.layers.rotary import apply_rotary_emb
347
elif current_platform.is_rocm():
348
from flash_attn.ops.triton.rotary import apply_rotary as apply_rotary_emb
349
+ else:
350
+ # For other platforms, use PyTorch fallback
351
+ from vllm.model_executor.layers.rotary_embedding.common import (
352
+ apply_rotary_emb_torch,
353
+ )
354
+
355
+ apply_rotary_emb = partial(apply_rotary_emb_torch, is_neox_style=True)
356
357
q_embed = apply_rotary_emb(q.float(), cos.float(), sin.float()).type_as(q)
358
k_embed = apply_rotary_emb(k.float(), cos.float(), sin.float()).type_as(k)