A high-throughput and memory-efficient inference and serving engine for LLMs
Mirrored from https://github.com/tduncan-lum/Nvidia-Optimization-Stack---VLLM.git
Synced Nov 17, 08:00 UTC
faster startup of vLLM (#982)
* update --------- Co-authored-by: Robert Irvine <robert@seamlessml.com>
Robert Irvine committed
Sep 8, 2023 at 06:48 UTC
4b5bcf89065e0d9247cb1e12635b0d57f67a3a6a
1 file changed
+3
-2
vllm/model_executor/layers/attention.py
+3
-2
@@ -259,8 +259,9 @@ class PagedAttentionWithRoPE(PagedAttention):
259
self.is_neox_style = is_neox_style
260
261
# Create the cos and sin cache.
262
- inv_freq = 1.0 / (base**(torch.arange(0, rotary_dim, 2) / rotary_dim))
263
- t = torch.arange(max_position).float()
262
+ inv_freq = 1.0 / (base**(
263
+ torch.arange(0, rotary_dim, 2, device="cuda") / rotary_dim))
264
+ t = torch.arange(max_position, device="cuda").float()
265
freqs = torch.einsum("i,j -> ij", t, inv_freq.float())
266
cos = freqs.cos()
267
sin = freqs.sin()