A high-throughput and memory-efficient inference and serving engine for LLMs
Mirrored from https://github.com/tduncan-lum/Nvidia-Optimization-Stack---VLLM.git
Synced Nov 17, 06:45 UTC
Add comments on RoPE initialization (#1176)
Woosuk Kwon committed
Sep 26, 2023 at 10:48 UTC
03ffd0a02251e10c1aa14fca8cb0ab1e4e40b886
1 file changed
+9
-1
vllm/model_executor/layers/attention.py
+9
-1
@@ -264,6 +264,15 @@ class PagedAttentionWithRoPE(PagedAttention):
264
self.is_neox_style = is_neox_style
265
266
# Create the cos and sin cache.
267
+ # NOTE(woosuk): The HF implementation uses `torch.arange(...).float()`.
268
+ # However, we use `torch.arange(..., dtype=torch.float)` instead to
269
+ # avoid numerical issues with large base values (e.g., 10000000).
270
+ # This may cause a slight numerical difference between the HF
271
+ # implementation and ours.
272
+ # NOTE(woosuk): To exactly match the HF implementation, we need to
273
+ # use CPU to compute the cache and then move it to GPU. However, we
274
+ # create the cache on GPU for faster initialization. This may cause
275
+ # a slight numerical difference between the HF implementation and ours.
276
inv_freq = 1.0 / (base**(torch.arange(
277
0, rotary_dim, 2, dtype=torch.float, device="cuda") / rotary_dim))
278
t = torch.arange(max_position, dtype=torch.float, device="cuda")
@@ -274,7 +283,6 @@ class PagedAttentionWithRoPE(PagedAttention):
283
284
# FIXME(woosuk): This assumes that we configure the default dtype when
285
# initializing the model.
277
- # TODO(woosuk): Make it more robust.
286
torch_dtype = torch.get_default_dtype()
287
cache = cache.to(torch_dtype)
288
# Embedding size: [max_position, rotary_dim]