A high-throughput and memory-efficient inference and serving engine for LLMs
Mirrored from https://github.com/tduncan-lum/Nvidia-Optimization-Stack---VLLM.git
Synced Nov 16, 23:15 UTC
[Frontend] Customizable RoPE theta (#5197)
sasha0552 committed
Jun 11, 2024 at 17:42 UTC
dcbf4286afbff55d836b1c69bd2b4705f0082ddb
5 files changed
+27
-8
tests/test_config.py
+6
-1
@@ -63,8 +63,9 @@ def test_get_sliding_window():
63
assert mistral_model_config.get_sliding_window() == TEST_SLIDING_WINDOW
64
65
66
-def test_rope_scaling():
66
+def test_rope_customization():
67
TEST_ROPE_SCALING = {"type": "dynamic", "factor": 2.0}
68
+ TEST_ROPE_THETA = 16_000_000.0
69
LONGCHAT_ROPE_SCALING = {"type": "linear", "factor": 8.0}
70
71
llama_model_config = ModelConfig(
@@ -76,6 +77,7 @@ def test_rope_scaling():
77
seed=0,
78
)
79
assert getattr(llama_model_config.hf_config, "rope_scaling", None) is None
80
+ assert getattr(llama_model_config.hf_config, "rope_theta", None) == 500_000
81
assert llama_model_config.max_model_len == 8192
82
83
llama_model_config = ModelConfig(
@@ -86,9 +88,12 @@ def test_rope_scaling():
88
dtype="float16",
89
seed=0,
90
rope_scaling=TEST_ROPE_SCALING,
91
+ rope_theta=TEST_ROPE_THETA,
92
)
93
assert getattr(llama_model_config.hf_config, "rope_scaling",
94
None) == TEST_ROPE_SCALING
95
+ assert getattr(llama_model_config.hf_config, "rope_theta",
96
+ None) == TEST_ROPE_THETA
97
assert llama_model_config.max_model_len == 16384
98
99
longchat_model_config = ModelConfig(
vllm/config.py
+3
-1
@@ -93,6 +93,7 @@ class ModelConfig:
93
revision: Optional[str] = None,
94
code_revision: Optional[str] = None,
95
rope_scaling: Optional[dict] = None,
96
+ rope_theta: Optional[float] = None,
97
tokenizer_revision: Optional[str] = None,
98
max_model_len: Optional[int] = None,
99
quantization: Optional[str] = None,
@@ -113,6 +114,7 @@ class ModelConfig:
114
self.revision = revision
115
self.code_revision = code_revision
116
self.rope_scaling = rope_scaling
117
+ self.rope_theta = rope_theta
118
# The tokenizer version is consistent with the model version by default.
119
if tokenizer_revision is None:
120
self.tokenizer_revision = revision
@@ -132,7 +134,7 @@ class ModelConfig:
134
self.skip_tokenizer_init = skip_tokenizer_init
135
136
self.hf_config = get_config(self.model, trust_remote_code, revision,
135
- code_revision, rope_scaling)
137
+ code_revision, rope_scaling, rope_theta)
138
self.hf_text_config = get_hf_text_config(self.hf_config)
139
self.dtype = _get_and_verify_dtype(self.hf_text_config, dtype)
140
self.max_model_len = _get_and_verify_max_len(
vllm/engine/arg_utils.py
+8
@@ -53,6 +53,7 @@ class EngineArgs:
53
revision: Optional[str] = None
54
code_revision: Optional[str] = None
55
rope_scaling: Optional[dict] = None
56
+ rope_theta: Optional[float] = None
57
tokenizer_revision: Optional[str] = None
58
quantization: Optional[str] = None
59
enforce_eager: bool = False
@@ -400,6 +401,12 @@ class EngineArgs:
401
type=json.loads,
402
help='RoPE scaling configuration in JSON format. '
403
'For example, {"type":"dynamic","factor":2.0}')
404
+ parser.add_argument('--rope-theta',
405
+ default=None,
406
+ type=float,
407
+ help='RoPE theta. Use with `rope_scaling`. In '
408
+ 'some cases, changing the RoPE theta improves the '
409
+ 'performance of the scaled model.')
410
parser.add_argument('--enforce-eager',
411
action='store_true',
412
help='Always use eager-mode PyTorch. If False, '
@@ -630,6 +637,7 @@ class EngineArgs:
637
revision=self.revision,
638
code_revision=self.code_revision,
639
rope_scaling=self.rope_scaling,
640
+ rope_theta=self.rope_theta,
641
tokenizer_revision=self.tokenizer_revision,
642
max_model_len=self.max_model_len,
643
quantization=self.quantization,
vllm/engine/llm_engine.py
+2
-1
@@ -162,7 +162,7 @@ class LLMEngine:
162
"Initializing an LLM engine (v%s) with config: "
163
"model=%r, speculative_config=%r, tokenizer=%r, "
164
"skip_tokenizer_init=%s, tokenizer_mode=%s, revision=%s, "
165
- "rope_scaling=%r, tokenizer_revision=%s, "
165
+ "rope_scaling=%r, rope_theta=%r, tokenizer_revision=%s, "
166
"trust_remote_code=%s, dtype=%s, max_seq_len=%d, "
167
"download_dir=%r, load_format=%s, tensor_parallel_size=%d, "
168
"disable_custom_all_reduce=%s, quantization=%s, "
@@ -177,6 +177,7 @@ class LLMEngine:
177
model_config.tokenizer_mode,
178
model_config.revision,
179
model_config.rope_scaling,
180
+ model_config.rope_theta,
181
model_config.tokenizer_revision,
182
model_config.trust_remote_code,
183
model_config.dtype,
vllm/transformers_utils/config.py
+8
-5
@@ -23,7 +23,8 @@ def get_config(model: str,
23
trust_remote_code: bool,
24
revision: Optional[str] = None,
25
code_revision: Optional[str] = None,
26
- rope_scaling: Optional[dict] = None) -> PretrainedConfig:
26
+ rope_scaling: Optional[dict] = None,
27
+ rope_theta: Optional[float] = None) -> PretrainedConfig:
28
try:
29
if VLLM_USE_MODELSCOPE:
30
from modelscope import AutoConfig
@@ -50,10 +51,12 @@ def get_config(model: str,
51
config = config_class.from_pretrained(model,
52
revision=revision,
53
code_revision=code_revision)
53
- if rope_scaling is not None:
54
- logger.info("Updating rope_scaling from %r to %r",
55
- getattr(config, "rope_scaling", None), rope_scaling)
56
- config.update({"rope_scaling": rope_scaling})
54
+ for key, value in [("rope_scaling", rope_scaling),
55
+ ("rope_theta", rope_theta)]:
56
+ if value is not None:
57
+ logger.info("Updating %s from %r to %r", key,
58
+ getattr(config, key, None), value)
59
+ config.update({key: value})
60
return config
61
62