@tduncan / Nvidia-Optimization-Stack-VLLM Mirror · read-only / commits / 03ffd0a022
Fork 0

A high-throughput and memory-efficient inference and serving engine for LLMs

Mirrored from https://github.com/tduncan-lum/Nvidia-Optimization-Stack---VLLM.git

Synced Nov 17, 06:45 UTC

Add comments on RoPE initialization (#1176)

Woosuk Kwon committed Sep 26, 2023 at 10:48 UTC 03ffd0a02251e10c1aa14fca8cb0ab1e4e40b886
1 file changed +9 -1
vllm/model_executor/layers/attention.py
+9 -1
@@ -264,6 +264,15 @@ class PagedAttentionWithRoPE(PagedAttention):
264 self.is_neox_style = is_neox_style
265
266 # Create the cos and sin cache.
267 + # NOTE(woosuk): The HF implementation uses `torch.arange(...).float()`.
268 + # However, we use `torch.arange(..., dtype=torch.float)` instead to
269 + # avoid numerical issues with large base values (e.g., 10000000).
270 + # This may cause a slight numerical difference between the HF
271 + # implementation and ours.
272 + # NOTE(woosuk): To exactly match the HF implementation, we need to
273 + # use CPU to compute the cache and then move it to GPU. However, we
274 + # create the cache on GPU for faster initialization. This may cause
275 + # a slight numerical difference between the HF implementation and ours.
276 inv_freq = 1.0 / (base**(torch.arange(
277 0, rotary_dim, 2, dtype=torch.float, device="cuda") / rotary_dim))
278 t = torch.arange(max_position, dtype=torch.float, device="cuda")
@@ -274,7 +283,6 @@ class PagedAttentionWithRoPE(PagedAttention):
283
284 # FIXME(woosuk): This assumes that we configure the default dtype when
285 # initializing the model.
277 - # TODO(woosuk): Make it more robust.
286 torch_dtype = torch.get_default_dtype()
287 cache = cache.to(torch_dtype)
288 # Embedding size: [max_position, rotary_dim]