A high-throughput and memory-efficient inference and serving engine for LLMs
Mirrored from https://github.com/tduncan-lum/Nvidia-Optimization-Stack---VLLM.git
Synced Nov 17, 04:15 UTC
[FIX] Fix a bug in initializing Yarn RoPE (#2983)
44670 committed
Feb 22, 2024 at 17:40 UTC
c530e2cfe3b3d7e60130ff817cee7f3a395af232
1 file changed
+2
-4
vllm/model_executor/layers/rotary_embedding.py
+2
-4
@@ -245,13 +245,11 @@ def _yarn_find_correction_range(low_rot: int,
245
246
247
def _yarn_linear_ramp_mask(low: float, high: float, dim: int,
248
- dtype: torch.dtype,
249
- device: torch.device) -> torch.Tensor:
248
+ dtype: torch.dtype) -> torch.Tensor:
249
if low == high:
250
high += 0.001 # Prevent singularity
251
253
- linear_func = (torch.arange(dim, dtype=dtype, device=device) -
254
- low) / (high - low)
252
+ linear_func = (torch.arange(dim, dtype=dtype) - low) / (high - low)
253
ramp_func = torch.clamp(linear_func, 0, 1)
254
return ramp_func
255