A high-throughput and memory-efficient inference and serving engine for LLMs
Mirrored from https://github.com/tduncan-lum/Nvidia-Optimization-Stack---VLLM.git
Synced Nov 17, 01:45 UTC
[Bugfix] fix rope error when load models with different dtypes (#4835)
Jinzhen Lin committed
May 17, 2024 at 17:43 UTC
33e0823de583819f39e88c39ea3f7dd4e07c3990
2 files changed
+64
-13
tests/kernels/test_pos_encoding.py
+43
-1
@@ -1,4 +1,4 @@
1
-from itertools import accumulate
1
+from itertools import accumulate, product
2
from typing import List, Optional
3
4
import pytest
@@ -207,3 +207,45 @@ def test_batched_rotary_embedding_multi_lora(
207
ref_key,
208
atol=get_default_atol(out_key),
209
rtol=get_default_rtol(out_key))
210
+
211
+
212
+@torch.inference_mode()
213
+def test_rope_module_cache():
214
+ MAX_POSITIONS = [123, 1234]
215
+ BASES = [10000, 1000000]
216
+ ROPE_SCALINGS = [
217
+ None, {
218
+ "type": "linear",
219
+ "factor": (1, )
220
+ }, {
221
+ "type": "dynamic",
222
+ "factor": 1
223
+ }
224
+ ]
225
+ settings = [
226
+ HEAD_SIZES, ROTARY_DIMS, MAX_POSITIONS, BASES, IS_NEOX_STYLE,
227
+ ROPE_SCALINGS, DTYPES
228
+ ]
229
+ rope_setting_id_map = {}
230
+ for setting in product(*settings):
231
+ head_size, rotary_dim, max_position, base, \
232
+ is_neox_stype, rope_scaling, dtype = setting
233
+ if rotary_dim is None:
234
+ rotary_dim = head_size
235
+ rope = get_rope(head_size, rotary_dim, max_position, base,
236
+ is_neox_stype, rope_scaling, dtype)
237
+ # different settings cannot share the same rope module
238
+ assert id(rope) not in rope_setting_id_map.values()
239
+ assert all(x.dtype == dtype for x in rope.buffers())
240
+ assert all(x.dtype == dtype for x in rope.parameters())
241
+ rope_setting_id_map[str(setting)] = id(rope)
242
+
243
+ for setting in product(*settings):
244
+ head_size, rotary_dim, max_position, base, \
245
+ is_neox_stype, rope_scaling, dtype = setting
246
+ if rotary_dim is None:
247
+ rotary_dim = head_size
248
+ rope = get_rope(head_size, rotary_dim, max_position, base,
249
+ is_neox_stype, rope_scaling, dtype)
250
+ # check if cache take effect
251
+ assert id(rope) == rope_setting_id_map[str(setting)]
vllm/model_executor/layers/rotary_embedding.py
+21
-12
@@ -53,6 +53,7 @@ class RotaryEmbedding(nn.Module):
53
max_position_embeddings: int,
54
base: int,
55
is_neox_style: bool,
56
+ dtype: torch.dtype,
57
) -> None:
58
super().__init__()
59
self.head_size = head_size
@@ -62,7 +63,7 @@ class RotaryEmbedding(nn.Module):
63
self.is_neox_style = is_neox_style
64
65
cache = self._compute_cos_sin_cache()
65
- cache = cache.to(torch.get_default_dtype())
66
+ cache = cache.to(dtype)
67
self.register_buffer("cos_sin_cache", cache, persistent=False)
68
69
def _compute_inv_freq(self, base: Union[int, float]) -> torch.Tensor:
@@ -178,12 +179,13 @@ class LinearScalingRotaryEmbedding(RotaryEmbedding):
179
base: int,
180
is_neox_style: bool,
181
scaling_factors: Union[List[float], float],
182
+ dtype: torch.dtype,
183
) -> None:
184
if isinstance(scaling_factors, float):
185
scaling_factors = [scaling_factors]
186
self.scaling_factors = scaling_factors
187
super().__init__(head_size, rotary_dim, max_position_embeddings, base,
186
- is_neox_style)
188
+ is_neox_style, dtype)
189
190
def _compute_cos_sin_cache(self) -> torch.Tensor:
191
inv_freq = self._compute_inv_freq(self.base)
@@ -219,10 +221,11 @@ class DynamicNTKScalingRotaryEmbedding(RotaryEmbedding):
221
base: int,
222
is_neox_style: bool,
223
scaling_factor: float,
224
+ dtype: torch.dtype,
225
) -> None:
226
self.scaling_factor = scaling_factor
227
super().__init__(head_size, rotary_dim, max_position_embeddings, base,
225
- is_neox_style)
228
+ is_neox_style, dtype)
229
230
def _compute_cos_sin_cache(self) -> torch.Tensor:
231
# NOTE(woosuk): self.max_position_embeddings is the original
@@ -299,6 +302,7 @@ class YaRNScalingRotaryEmbedding(RotaryEmbedding):
302
base: int,
303
is_neox_style: bool,
304
scaling_factor: float,
305
+ dtype: torch.dtype,
306
*,
307
extrapolation_factor: float = 1,
308
attn_factor: float = 1,
@@ -314,7 +318,7 @@ class YaRNScalingRotaryEmbedding(RotaryEmbedding):
318
self.mscale = float(
319
_yarn_get_mscale(self.scaling_factor) * attn_factor)
320
super().__init__(head_size, rotary_dim, max_position_embeddings, base,
317
- is_neox_style)
321
+ is_neox_style, dtype)
322
323
def _compute_inv_freq(self, scaling_factor: float) -> torch.Tensor:
324
pos_freqs = self.base**(
@@ -359,6 +363,7 @@ class Phi3SuScaledRotaryEmbedding(nn.Module):
363
original_max_position_embeddings: int,
364
base: int,
365
is_neox_style: bool,
366
+ dtype: torch.dtype,
367
short_factor: List[float],
368
long_factor: List[float],
369
short_mscale: float = 1.1,
@@ -385,14 +390,14 @@ class Phi3SuScaledRotaryEmbedding(nn.Module):
390
391
short_cache = self._compute_cos_sin_cache(
392
original_max_position_embeddings, short_factor, short_mscale)
388
- short_cache = short_cache.to(torch.get_default_dtype())
393
+ short_cache = short_cache.to(dtype)
394
self.register_buffer("short_cos_sin_cache",
395
short_cache,
396
persistent=False)
397
398
long_cache = self._compute_cos_sin_cache(max_position_embeddings,
399
long_factor, long_mscale)
395
- long_cache = long_cache.to(torch.get_default_dtype())
400
+ long_cache = long_cache.to(dtype)
401
self.register_buffer("long_cos_sin_cache",
402
long_cache,
403
persistent=False)
@@ -463,7 +468,10 @@ def get_rope(
468
base: int,
469
is_neox_style: bool = True,
470
rope_scaling: Optional[Dict[str, Any]] = None,
471
+ dtype: Optional[torch.dtype] = None,
472
) -> RotaryEmbedding:
473
+ if dtype is None:
474
+ dtype = torch.get_default_dtype()
475
if rope_scaling is not None:
476
# Transforms every value that is a list into a tuple for caching calls
477
rope_scaling_tuple = {
@@ -474,12 +482,12 @@ def get_rope(
482
else:
483
rope_scaling_args = None
484
key = (head_size, rotary_dim, max_position, base, is_neox_style,
477
- rope_scaling_args)
485
+ rope_scaling_args, dtype)
486
if key in _ROPE_DICT:
487
return _ROPE_DICT[key]
488
if rope_scaling is None:
489
rotary_emb = RotaryEmbedding(head_size, rotary_dim, max_position, base,
482
- is_neox_style)
490
+ is_neox_style, dtype)
491
else:
492
scaling_type = rope_scaling["type"]
493
if scaling_type != "su":
@@ -488,11 +496,11 @@ def get_rope(
496
rotary_emb = LinearScalingRotaryEmbedding(head_size, rotary_dim,
497
max_position, base,
498
is_neox_style,
491
- scaling_factor)
499
+ scaling_factor, dtype)
500
elif scaling_type == "dynamic":
501
rotary_emb = DynamicNTKScalingRotaryEmbedding(
502
head_size, rotary_dim, max_position, base, is_neox_style,
495
- scaling_factor)
503
+ scaling_factor, dtype)
504
elif scaling_type == "yarn":
505
original_max_position = rope_scaling[
506
"original_max_position_embeddings"]
@@ -505,7 +513,7 @@ def get_rope(
513
rotary_emb = YaRNScalingRotaryEmbedding(head_size, rotary_dim,
514
original_max_position,
515
base, is_neox_style,
508
- scaling_factor,
516
+ scaling_factor, dtype,
517
**extra_kwargs)
518
elif scaling_type == "su":
519
short_factor = rope_scaling["short_factor"]
@@ -519,7 +527,8 @@ def get_rope(
527
}
528
rotary_emb = Phi3SuScaledRotaryEmbedding(
529
head_size, rotary_dim, max_position, original_max_position,
522
- base, is_neox_style, short_factor, long_factor, **extra_kwargs)
530
+ base, is_neox_style, dtype, short_factor, long_factor,
531
+ **extra_kwargs)
532
else:
533
raise ValueError(f"Unknown RoPE scaling type {scaling_type}")
534
_ROPE_DICT[key] = rotary_emb