@tduncan / Nvidia-Optimization-Stack-VLLM Mirror · read-only / commits / 4b5bcf8906
Fork 0

A high-throughput and memory-efficient inference and serving engine for LLMs

Mirrored from https://github.com/tduncan-lum/Nvidia-Optimization-Stack---VLLM.git

Synced Nov 17, 08:00 UTC

faster startup of vLLM (#982)

* update --------- Co-authored-by: Robert Irvine <robert@seamlessml.com>

Robert Irvine committed Sep 8, 2023 at 06:48 UTC 4b5bcf89065e0d9247cb1e12635b0d57f67a3a6a
1 file changed +3 -2
vllm/model_executor/layers/attention.py
+3 -2
@@ -259,8 +259,9 @@ class PagedAttentionWithRoPE(PagedAttention):
259 self.is_neox_style = is_neox_style
260
261 # Create the cos and sin cache.
262 - inv_freq = 1.0 / (base**(torch.arange(0, rotary_dim, 2) / rotary_dim))
263 - t = torch.arange(max_position).float()
262 + inv_freq = 1.0 / (base**(
263 + torch.arange(0, rotary_dim, 2, device="cuda") / rotary_dim))
264 + t = torch.arange(max_position, device="cuda").float()
265 freqs = torch.einsum("i,j -> ij", t, inv_freq.float())
266 cos = freqs.cos()
267 sin = freqs.sin()