A high-throughput and memory-efficient inference and serving engine for LLMs
Mirrored from https://github.com/tduncan-lum/Nvidia-Optimization-Stack---VLLM.git
Synced Nov 17, 08:00 UTC
Use FP32 in RoPE initialization (#1004)
Co-authored-by: One <imone@tuta.io>
Woosuk Kwon committed
Sep 11, 2023 at 00:26 UTC
e67b4f2c2a216ff12d4f607caa3ba3409ae3f572
2 files changed
+7
-6
tests/kernels/test_pos_encoding.py
+3
-2
@@ -133,9 +133,10 @@ def test_rotary_embedding(
133
device="cuda")
134
135
# Create the rotary embedding.
136
- inv_freq = 1.0 / (base**(torch.arange(0, rotary_dim, 2) / rotary_dim))
136
+ inv_freq = 1.0 / (base**(
137
+ torch.arange(0, rotary_dim, 2, dtype=torch.float) / rotary_dim))
138
t = torch.arange(max_position).float()
138
- freqs = torch.einsum("i,j -> ij", t, inv_freq.float())
139
+ freqs = torch.einsum("i,j -> ij", t, inv_freq)
140
cos = freqs.cos()
141
sin = freqs.sin()
142
cos_sin_cache = torch.cat((cos, sin), dim=-1)
vllm/model_executor/layers/attention.py
+4
-4
@@ -264,10 +264,10 @@ class PagedAttentionWithRoPE(PagedAttention):
264
self.is_neox_style = is_neox_style
265
266
# Create the cos and sin cache.
267
- inv_freq = 1.0 / (base**(
268
- torch.arange(0, rotary_dim, 2, device="cuda") / rotary_dim))
269
- t = torch.arange(max_position, device="cuda").float()
270
- freqs = torch.einsum("i,j -> ij", t, inv_freq.float())
267
+ inv_freq = 1.0 / (base**(torch.arange(
268
+ 0, rotary_dim, 2, dtype=torch.float, device="cuda") / rotary_dim))
269
+ t = torch.arange(max_position, dtype=torch.float, device="cuda")
270
+ freqs = torch.einsum("i,j -> ij", t, inv_freq)
271
cos = freqs.cos()
272
sin = freqs.sin()
273
cache = torch.cat((cos, sin), dim=-1)