@tduncan / Nvidia-Optimization-Stack-VLLM Mirror · read-only / commits / c530e2cfe3
Fork 0

A high-throughput and memory-efficient inference and serving engine for LLMs

Mirrored from https://github.com/tduncan-lum/Nvidia-Optimization-Stack---VLLM.git

Synced Nov 17, 04:15 UTC

[FIX] Fix a bug in initializing Yarn RoPE (#2983)

44670 committed Feb 22, 2024 at 17:40 UTC c530e2cfe3b3d7e60130ff817cee7f3a395af232
1 file changed +2 -4
vllm/model_executor/layers/rotary_embedding.py
+2 -4
@@ -245,13 +245,11 @@ def _yarn_find_correction_range(low_rot: int,
245
246
247 def _yarn_linear_ramp_mask(low: float, high: float, dim: int,
248 - dtype: torch.dtype,
249 - device: torch.device) -> torch.Tensor:
248 + dtype: torch.dtype) -> torch.Tensor:
249 if low == high:
250 high += 0.001 # Prevent singularity
251
253 - linear_func = (torch.arange(dim, dtype=dtype, device=device) -
254 - low) / (high - low)
252 + linear_func = (torch.arange(dim, dtype=dtype) - low) / (high - low)
253 ramp_func = torch.clamp(linear_func, 0, 1)
254 return ramp_func
255