A high-throughput and memory-efficient inference and serving engine for LLMs
Mirrored from https://github.com/tduncan-lum/Nvidia-Optimization-Stack---VLLM.git
Synced Nov 14, 05:30 UTC
[Llama4] [multimodal] Fix misplaced dtype cast of `cos_sin_cache` in `Llama4VisionRotaryEmbedding` (#25889)
Signed-off-by: cjackal <44624812+cjackal@users.noreply.github.com>
cjackal committed
Oct 1, 2025 at 05:35 UTC
43b752c325d5225c7eaa2d96219009cbd8488638
1 file changed
+3
-1
vllm/model_executor/layers/rotary_embedding/llama4_vision_rope.py
+3
-1
@@ -59,7 +59,9 @@ class Llama4VisionRotaryEmbedding(RotaryEmbedding):
59
key: Optional[torch.Tensor] = None,
60
) -> tuple[torch.Tensor, Optional[torch.Tensor]]:
61
assert key is not None
62
- self._match_cos_sin_cache_dtype(query)
62
+ # self.cos_sin_cache here is complex tensor so we cannot cast into
63
+ # query's dtype directly with self._match_cos_sin_cache_dtype
64
+ self.cos_sin_cache: torch.Tensor = self.cos_sin_cache.to(query.device)
65
query_ = torch.view_as_complex(query.float().reshape(
66
*query.shape[:-1], -1, 2))
67
key_ = torch.view_as_complex(key.float().reshape(