@tduncan / Nvidia-Optimization-Stack-VLLM Mirror · read-only / commits / 33e0823de5
Fork 0

A high-throughput and memory-efficient inference and serving engine for LLMs

Mirrored from https://github.com/tduncan-lum/Nvidia-Optimization-Stack---VLLM.git

Synced Nov 17, 01:45 UTC

[Bugfix] fix rope error when load models with different dtypes (#4835)

Jinzhen Lin committed May 17, 2024 at 17:43 UTC 33e0823de583819f39e88c39ea3f7dd4e07c3990
2 files changed +64 -13
tests/kernels/test_pos_encoding.py
+43 -1
@@ -1,4 +1,4 @@
1 -from itertools import accumulate
1 +from itertools import accumulate, product
2 from typing import List, Optional
3
4 import pytest
@@ -207,3 +207,45 @@ def test_batched_rotary_embedding_multi_lora(
207 ref_key,
208 atol=get_default_atol(out_key),
209 rtol=get_default_rtol(out_key))
210 +
211 +
212 +@torch.inference_mode()
213 +def test_rope_module_cache():
214 + MAX_POSITIONS = [123, 1234]
215 + BASES = [10000, 1000000]
216 + ROPE_SCALINGS = [
217 + None, {
218 + "type": "linear",
219 + "factor": (1, )
220 + }, {
221 + "type": "dynamic",
222 + "factor": 1
223 + }
224 + ]
225 + settings = [
226 + HEAD_SIZES, ROTARY_DIMS, MAX_POSITIONS, BASES, IS_NEOX_STYLE,
227 + ROPE_SCALINGS, DTYPES
228 + ]
229 + rope_setting_id_map = {}
230 + for setting in product(*settings):
231 + head_size, rotary_dim, max_position, base, \
232 + is_neox_stype, rope_scaling, dtype = setting
233 + if rotary_dim is None:
234 + rotary_dim = head_size
235 + rope = get_rope(head_size, rotary_dim, max_position, base,
236 + is_neox_stype, rope_scaling, dtype)
237 + # different settings cannot share the same rope module
238 + assert id(rope) not in rope_setting_id_map.values()
239 + assert all(x.dtype == dtype for x in rope.buffers())
240 + assert all(x.dtype == dtype for x in rope.parameters())
241 + rope_setting_id_map[str(setting)] = id(rope)
242 +
243 + for setting in product(*settings):
244 + head_size, rotary_dim, max_position, base, \
245 + is_neox_stype, rope_scaling, dtype = setting
246 + if rotary_dim is None:
247 + rotary_dim = head_size
248 + rope = get_rope(head_size, rotary_dim, max_position, base,
249 + is_neox_stype, rope_scaling, dtype)
250 + # check if cache take effect
251 + assert id(rope) == rope_setting_id_map[str(setting)]
vllm/model_executor/layers/rotary_embedding.py
+21 -12
@@ -53,6 +53,7 @@ class RotaryEmbedding(nn.Module):
53 max_position_embeddings: int,
54 base: int,
55 is_neox_style: bool,
56 + dtype: torch.dtype,
57 ) -> None:
58 super().__init__()
59 self.head_size = head_size
@@ -62,7 +63,7 @@ class RotaryEmbedding(nn.Module):
63 self.is_neox_style = is_neox_style
64
65 cache = self._compute_cos_sin_cache()
65 - cache = cache.to(torch.get_default_dtype())
66 + cache = cache.to(dtype)
67 self.register_buffer("cos_sin_cache", cache, persistent=False)
68
69 def _compute_inv_freq(self, base: Union[int, float]) -> torch.Tensor:
@@ -178,12 +179,13 @@ class LinearScalingRotaryEmbedding(RotaryEmbedding):
179 base: int,
180 is_neox_style: bool,
181 scaling_factors: Union[List[float], float],
182 + dtype: torch.dtype,
183 ) -> None:
184 if isinstance(scaling_factors, float):
185 scaling_factors = [scaling_factors]
186 self.scaling_factors = scaling_factors
187 super().__init__(head_size, rotary_dim, max_position_embeddings, base,
186 - is_neox_style)
188 + is_neox_style, dtype)
189
190 def _compute_cos_sin_cache(self) -> torch.Tensor:
191 inv_freq = self._compute_inv_freq(self.base)
@@ -219,10 +221,11 @@ class DynamicNTKScalingRotaryEmbedding(RotaryEmbedding):
221 base: int,
222 is_neox_style: bool,
223 scaling_factor: float,
224 + dtype: torch.dtype,
225 ) -> None:
226 self.scaling_factor = scaling_factor
227 super().__init__(head_size, rotary_dim, max_position_embeddings, base,
225 - is_neox_style)
228 + is_neox_style, dtype)
229
230 def _compute_cos_sin_cache(self) -> torch.Tensor:
231 # NOTE(woosuk): self.max_position_embeddings is the original
@@ -299,6 +302,7 @@ class YaRNScalingRotaryEmbedding(RotaryEmbedding):
302 base: int,
303 is_neox_style: bool,
304 scaling_factor: float,
305 + dtype: torch.dtype,
306 *,
307 extrapolation_factor: float = 1,
308 attn_factor: float = 1,
@@ -314,7 +318,7 @@ class YaRNScalingRotaryEmbedding(RotaryEmbedding):
318 self.mscale = float(
319 _yarn_get_mscale(self.scaling_factor) * attn_factor)
320 super().__init__(head_size, rotary_dim, max_position_embeddings, base,
317 - is_neox_style)
321 + is_neox_style, dtype)
322
323 def _compute_inv_freq(self, scaling_factor: float) -> torch.Tensor:
324 pos_freqs = self.base**(
@@ -359,6 +363,7 @@ class Phi3SuScaledRotaryEmbedding(nn.Module):
363 original_max_position_embeddings: int,
364 base: int,
365 is_neox_style: bool,
366 + dtype: torch.dtype,
367 short_factor: List[float],
368 long_factor: List[float],
369 short_mscale: float = 1.1,
@@ -385,14 +390,14 @@ class Phi3SuScaledRotaryEmbedding(nn.Module):
390
391 short_cache = self._compute_cos_sin_cache(
392 original_max_position_embeddings, short_factor, short_mscale)
388 - short_cache = short_cache.to(torch.get_default_dtype())
393 + short_cache = short_cache.to(dtype)
394 self.register_buffer("short_cos_sin_cache",
395 short_cache,
396 persistent=False)
397
398 long_cache = self._compute_cos_sin_cache(max_position_embeddings,
399 long_factor, long_mscale)
395 - long_cache = long_cache.to(torch.get_default_dtype())
400 + long_cache = long_cache.to(dtype)
401 self.register_buffer("long_cos_sin_cache",
402 long_cache,
403 persistent=False)
@@ -463,7 +468,10 @@ def get_rope(
468 base: int,
469 is_neox_style: bool = True,
470 rope_scaling: Optional[Dict[str, Any]] = None,
471 + dtype: Optional[torch.dtype] = None,
472 ) -> RotaryEmbedding:
473 + if dtype is None:
474 + dtype = torch.get_default_dtype()
475 if rope_scaling is not None:
476 # Transforms every value that is a list into a tuple for caching calls
477 rope_scaling_tuple = {
@@ -474,12 +482,12 @@ def get_rope(
482 else:
483 rope_scaling_args = None
484 key = (head_size, rotary_dim, max_position, base, is_neox_style,
477 - rope_scaling_args)
485 + rope_scaling_args, dtype)
486 if key in _ROPE_DICT:
487 return _ROPE_DICT[key]
488 if rope_scaling is None:
489 rotary_emb = RotaryEmbedding(head_size, rotary_dim, max_position, base,
482 - is_neox_style)
490 + is_neox_style, dtype)
491 else:
492 scaling_type = rope_scaling["type"]
493 if scaling_type != "su":
@@ -488,11 +496,11 @@ def get_rope(
496 rotary_emb = LinearScalingRotaryEmbedding(head_size, rotary_dim,
497 max_position, base,
498 is_neox_style,
491 - scaling_factor)
499 + scaling_factor, dtype)
500 elif scaling_type == "dynamic":
501 rotary_emb = DynamicNTKScalingRotaryEmbedding(
502 head_size, rotary_dim, max_position, base, is_neox_style,
495 - scaling_factor)
503 + scaling_factor, dtype)
504 elif scaling_type == "yarn":
505 original_max_position = rope_scaling[
506 "original_max_position_embeddings"]
@@ -505,7 +513,7 @@ def get_rope(
513 rotary_emb = YaRNScalingRotaryEmbedding(head_size, rotary_dim,
514 original_max_position,
515 base, is_neox_style,
508 - scaling_factor,
516 + scaling_factor, dtype,
517 **extra_kwargs)
518 elif scaling_type == "su":
519 short_factor = rope_scaling["short_factor"]
@@ -519,7 +527,8 @@ def get_rope(
527 }
528 rotary_emb = Phi3SuScaledRotaryEmbedding(
529 head_size, rotary_dim, max_position, original_max_position,
522 - base, is_neox_style, short_factor, long_factor, **extra_kwargs)
530 + base, is_neox_style, dtype, short_factor, long_factor,
531 + **extra_kwargs)
532 else:
533 raise ValueError(f"Unknown RoPE scaling type {scaling_type}")
534 _ROPE_DICT[key] = rotary_emb