A high-throughput and memory-efficient inference and serving engine for LLMs
Mirrored from https://github.com/tduncan-lum/Nvidia-Optimization-Stack---VLLM.git
Synced Nov 16, 20:45 UTC
[BugFix] Fix RoPE error in Llama 3.1 (#6693)
Woosuk Kwon committed
Jul 23, 2024 at 09:46 UTC
a112a84aadad40a69b85a07a5eb3f329498ea20e
2 files changed
+30
-30
vllm/config.py
+26
-27
@@ -154,15 +154,6 @@ class ModelConfig:
154
self.hf_text_config = get_hf_text_config(self.hf_config)
155
self.dtype = _get_and_verify_dtype(self.hf_text_config, dtype)
156
157
- if (getattr(self.hf_config, "max_position_embeddings", 0) == 131072
158
- and getattr(self.hf_config, "rope_scaling", None) is None):
159
- # Note(simon): this is a special case for a model that doesn't
160
- # supply rope_scaling. We should remove this once the model is
161
- # updated.
162
- self.hf_config.update({"rope_scaling": {
163
- "type": "extended",
164
- }})
165
-
157
if (not self.disable_sliding_window
158
and self.hf_text_config.model_type == "gemma2"
159
and self.hf_text_config.sliding_window is not None):
@@ -1492,24 +1483,32 @@ def _get_and_verify_max_len(
1483
derived_max_model_len = default_max_len
1484
1485
rope_scaling = getattr(hf_config, "rope_scaling", None)
1495
- # The correct one should be "longrope", kept "su" here
1496
- # to be backward compatible
1497
- if rope_scaling is not None and rope_scaling["type"] not in {
1498
- "su", "longrope", "extended"
1499
- }:
1500
- if disable_sliding_window:
1501
- # TODO(robertgshaw): Find a model that supports rope_scaling
1502
- # with sliding window to see if this case should be allowed.
1503
- raise NotImplementedError(
1504
- "Disabling sliding window is not supported for models "
1505
- "with rope_scaling. Please raise an issue so we can "
1506
- "investigate.")
1507
- assert "factor" in rope_scaling
1508
- scaling_factor = rope_scaling["factor"]
1509
- if rope_scaling["type"] == "yarn":
1510
- derived_max_model_len = rope_scaling[
1511
- "original_max_position_embeddings"]
1512
- derived_max_model_len *= scaling_factor
1486
+ if rope_scaling is not None:
1487
+ if "type" in rope_scaling:
1488
+ rope_type = rope_scaling["type"]
1489
+ elif "rope_type" in rope_scaling:
1490
+ rope_type = rope_scaling["rope_type"]
1491
+ else:
1492
+ raise ValueError(
1493
+ "rope_scaling must have a 'type' or 'rope_type' key.")
1494
+
1495
+ # The correct one should be "longrope", kept "su" here
1496
+ # to be backward compatible
1497
+ if rope_type not in ("su", "longrope", "llama3"):
1498
+ if disable_sliding_window:
1499
+ # TODO(robertgshaw): Find a model that supports rope_scaling
1500
+ # with sliding window to see if this case should be allowed.
1501
+ raise NotImplementedError(
1502
+ "Disabling sliding window is not supported for models "
1503
+ "with rope_scaling. Please raise an issue so we can "
1504
+ "investigate.")
1505
+
1506
+ assert "factor" in rope_scaling
1507
+ scaling_factor = rope_scaling["factor"]
1508
+ if rope_type == "yarn":
1509
+ derived_max_model_len = rope_scaling[
1510
+ "original_max_position_embeddings"]
1511
+ derived_max_model_len *= scaling_factor
1512
1513
# If the user specified a max length, make sure it is smaller than the
1514
# derived length from the HF model config.
vllm/model_executor/layers/rotary_embedding.py
+4
-3
@@ -794,12 +794,13 @@ def get_rope(
794
rotary_emb = RotaryEmbedding(head_size, rotary_dim, max_position, base,
795
is_neox_style, dtype)
796
else:
797
- scaling_type = rope_scaling["type"]
797
+ scaling_type = rope_scaling[
798
+ "type"] if "type" in rope_scaling else rope_scaling["rope_type"]
799
# The correct one should be "longrope" but keep "su" here
800
# for backward compatible
800
- if scaling_type not in {"su", "longrope", "extended"}:
801
+ if scaling_type not in {"su", "longrope", "llama3"}:
802
scaling_factor = rope_scaling["factor"]
802
- if scaling_type == "extended":
803
+ if scaling_type == "llama3":
804
rotary_emb = ExtendedRotaryEmbedding(head_size, rotary_dim,
805
max_position, base,
806
is_neox_style, dtype)