@tduncan / Nvidia-Optimization-Stack-VLLM Mirror · read-only / commits / e67b4f2c2a
Fork 0

A high-throughput and memory-efficient inference and serving engine for LLMs

Mirrored from https://github.com/tduncan-lum/Nvidia-Optimization-Stack---VLLM.git

Synced Nov 17, 08:00 UTC

Use FP32 in RoPE initialization (#1004)

Co-authored-by: One <imone@tuta.io>

Woosuk Kwon committed Sep 11, 2023 at 00:26 UTC e67b4f2c2a216ff12d4f607caa3ba3409ae3f572
2 files changed +7 -6
tests/kernels/test_pos_encoding.py
+3 -2
@@ -133,9 +133,10 @@ def test_rotary_embedding(
133 device="cuda")
134
135 # Create the rotary embedding.
136 - inv_freq = 1.0 / (base**(torch.arange(0, rotary_dim, 2) / rotary_dim))
136 + inv_freq = 1.0 / (base**(
137 + torch.arange(0, rotary_dim, 2, dtype=torch.float) / rotary_dim))
138 t = torch.arange(max_position).float()
138 - freqs = torch.einsum("i,j -> ij", t, inv_freq.float())
139 + freqs = torch.einsum("i,j -> ij", t, inv_freq)
140 cos = freqs.cos()
141 sin = freqs.sin()
142 cos_sin_cache = torch.cat((cos, sin), dim=-1)
vllm/model_executor/layers/attention.py
+4 -4
@@ -264,10 +264,10 @@ class PagedAttentionWithRoPE(PagedAttention):
264 self.is_neox_style = is_neox_style
265
266 # Create the cos and sin cache.
267 - inv_freq = 1.0 / (base**(
268 - torch.arange(0, rotary_dim, 2, device="cuda") / rotary_dim))
269 - t = torch.arange(max_position, device="cuda").float()
270 - freqs = torch.einsum("i,j -> ij", t, inv_freq.float())
267 + inv_freq = 1.0 / (base**(torch.arange(
268 + 0, rotary_dim, 2, dtype=torch.float, device="cuda") / rotary_dim))
269 + t = torch.arange(max_position, dtype=torch.float, device="cuda")
270 + freqs = torch.einsum("i,j -> ij", t, inv_freq)
271 cos = freqs.cos()
272 sin = freqs.sin()
273 cache = torch.cat((cos, sin), dim=-1)