@tduncan / Nvidia-Optimization-Stack-VLLM Mirror · read-only / commits / dcbf4286af
Fork 0

A high-throughput and memory-efficient inference and serving engine for LLMs

Mirrored from https://github.com/tduncan-lum/Nvidia-Optimization-Stack---VLLM.git

Synced Nov 16, 23:15 UTC

[Frontend] Customizable RoPE theta (#5197)

sasha0552 committed Jun 11, 2024 at 17:42 UTC dcbf4286afbff55d836b1c69bd2b4705f0082ddb
5 files changed +27 -8
tests/test_config.py
+6 -1
@@ -63,8 +63,9 @@ def test_get_sliding_window():
63 assert mistral_model_config.get_sliding_window() == TEST_SLIDING_WINDOW
64
65
66 -def test_rope_scaling():
66 +def test_rope_customization():
67 TEST_ROPE_SCALING = {"type": "dynamic", "factor": 2.0}
68 + TEST_ROPE_THETA = 16_000_000.0
69 LONGCHAT_ROPE_SCALING = {"type": "linear", "factor": 8.0}
70
71 llama_model_config = ModelConfig(
@@ -76,6 +77,7 @@ def test_rope_scaling():
77 seed=0,
78 )
79 assert getattr(llama_model_config.hf_config, "rope_scaling", None) is None
80 + assert getattr(llama_model_config.hf_config, "rope_theta", None) == 500_000
81 assert llama_model_config.max_model_len == 8192
82
83 llama_model_config = ModelConfig(
@@ -86,9 +88,12 @@ def test_rope_scaling():
88 dtype="float16",
89 seed=0,
90 rope_scaling=TEST_ROPE_SCALING,
91 + rope_theta=TEST_ROPE_THETA,
92 )
93 assert getattr(llama_model_config.hf_config, "rope_scaling",
94 None) == TEST_ROPE_SCALING
95 + assert getattr(llama_model_config.hf_config, "rope_theta",
96 + None) == TEST_ROPE_THETA
97 assert llama_model_config.max_model_len == 16384
98
99 longchat_model_config = ModelConfig(
vllm/config.py
+3 -1
@@ -93,6 +93,7 @@ class ModelConfig:
93 revision: Optional[str] = None,
94 code_revision: Optional[str] = None,
95 rope_scaling: Optional[dict] = None,
96 + rope_theta: Optional[float] = None,
97 tokenizer_revision: Optional[str] = None,
98 max_model_len: Optional[int] = None,
99 quantization: Optional[str] = None,
@@ -113,6 +114,7 @@ class ModelConfig:
114 self.revision = revision
115 self.code_revision = code_revision
116 self.rope_scaling = rope_scaling
117 + self.rope_theta = rope_theta
118 # The tokenizer version is consistent with the model version by default.
119 if tokenizer_revision is None:
120 self.tokenizer_revision = revision
@@ -132,7 +134,7 @@ class ModelConfig:
134 self.skip_tokenizer_init = skip_tokenizer_init
135
136 self.hf_config = get_config(self.model, trust_remote_code, revision,
135 - code_revision, rope_scaling)
137 + code_revision, rope_scaling, rope_theta)
138 self.hf_text_config = get_hf_text_config(self.hf_config)
139 self.dtype = _get_and_verify_dtype(self.hf_text_config, dtype)
140 self.max_model_len = _get_and_verify_max_len(
vllm/engine/arg_utils.py
+8
@@ -53,6 +53,7 @@ class EngineArgs:
53 revision: Optional[str] = None
54 code_revision: Optional[str] = None
55 rope_scaling: Optional[dict] = None
56 + rope_theta: Optional[float] = None
57 tokenizer_revision: Optional[str] = None
58 quantization: Optional[str] = None
59 enforce_eager: bool = False
@@ -400,6 +401,12 @@ class EngineArgs:
401 type=json.loads,
402 help='RoPE scaling configuration in JSON format. '
403 'For example, {"type":"dynamic","factor":2.0}')
404 + parser.add_argument('--rope-theta',
405 + default=None,
406 + type=float,
407 + help='RoPE theta. Use with `rope_scaling`. In '
408 + 'some cases, changing the RoPE theta improves the '
409 + 'performance of the scaled model.')
410 parser.add_argument('--enforce-eager',
411 action='store_true',
412 help='Always use eager-mode PyTorch. If False, '
@@ -630,6 +637,7 @@ class EngineArgs:
637 revision=self.revision,
638 code_revision=self.code_revision,
639 rope_scaling=self.rope_scaling,
640 + rope_theta=self.rope_theta,
641 tokenizer_revision=self.tokenizer_revision,
642 max_model_len=self.max_model_len,
643 quantization=self.quantization,
vllm/engine/llm_engine.py
+2 -1
@@ -162,7 +162,7 @@ class LLMEngine:
162 "Initializing an LLM engine (v%s) with config: "
163 "model=%r, speculative_config=%r, tokenizer=%r, "
164 "skip_tokenizer_init=%s, tokenizer_mode=%s, revision=%s, "
165 - "rope_scaling=%r, tokenizer_revision=%s, "
165 + "rope_scaling=%r, rope_theta=%r, tokenizer_revision=%s, "
166 "trust_remote_code=%s, dtype=%s, max_seq_len=%d, "
167 "download_dir=%r, load_format=%s, tensor_parallel_size=%d, "
168 "disable_custom_all_reduce=%s, quantization=%s, "
@@ -177,6 +177,7 @@ class LLMEngine:
177 model_config.tokenizer_mode,
178 model_config.revision,
179 model_config.rope_scaling,
180 + model_config.rope_theta,
181 model_config.tokenizer_revision,
182 model_config.trust_remote_code,
183 model_config.dtype,
vllm/transformers_utils/config.py
+8 -5
@@ -23,7 +23,8 @@ def get_config(model: str,
23 trust_remote_code: bool,
24 revision: Optional[str] = None,
25 code_revision: Optional[str] = None,
26 - rope_scaling: Optional[dict] = None) -> PretrainedConfig:
26 + rope_scaling: Optional[dict] = None,
27 + rope_theta: Optional[float] = None) -> PretrainedConfig:
28 try:
29 if VLLM_USE_MODELSCOPE:
30 from modelscope import AutoConfig
@@ -50,10 +51,12 @@ def get_config(model: str,
51 config = config_class.from_pretrained(model,
52 revision=revision,
53 code_revision=code_revision)
53 - if rope_scaling is not None:
54 - logger.info("Updating rope_scaling from %r to %r",
55 - getattr(config, "rope_scaling", None), rope_scaling)
56 - config.update({"rope_scaling": rope_scaling})
54 + for key, value in [("rope_scaling", rope_scaling),
55 + ("rope_theta", rope_theta)]:
56 + if value is not None:
57 + logger.info("Updating %s from %r to %r", key,
58 + getattr(config, key, None), value)
59 + config.update({key: value})
60 return config
61
62