A high-throughput and memory-efficient inference and serving engine for LLMs
Mirrored from https://github.com/tduncan-lum/Nvidia-Optimization-Stack---VLLM.git
Synced Nov 16, 08:15 UTC
[Frontend][Core] Override HF `config.json` via CLI (#5836)
Signed-off-by: DarkLight1337 <tlleungac@connect.ust.hk> Co-authored-by: DarkLight1337 <tlleungac@connect.ust.hk>
Krishna Mandal committed
Nov 9, 2024 at 08:19 UTC
b09895a61843e654088773851a2b1acae4cdf184
7 files changed
+73
-53
tests/test_config.py
+7
-3
@@ -200,8 +200,10 @@ def test_rope_customization():
200
trust_remote_code=False,
201
dtype="float16",
202
seed=0,
203
- rope_scaling=TEST_ROPE_SCALING,
204
- rope_theta=TEST_ROPE_THETA,
203
+ hf_overrides={
204
+ "rope_scaling": TEST_ROPE_SCALING,
205
+ "rope_theta": TEST_ROPE_THETA,
206
+ },
207
)
208
assert getattr(llama_model_config.hf_config, "rope_scaling",
209
None) == TEST_ROPE_SCALING
@@ -232,7 +234,9 @@ def test_rope_customization():
234
trust_remote_code=False,
235
dtype="float16",
236
seed=0,
235
- rope_scaling=TEST_ROPE_SCALING,
237
+ hf_overrides={
238
+ "rope_scaling": TEST_ROPE_SCALING,
239
+ },
240
)
241
assert getattr(longchat_model_config.hf_config, "rope_scaling",
242
None) == TEST_ROPE_SCALING
vllm/config.py
+22
-8
@@ -1,5 +1,6 @@
1
import enum
2
import json
3
+import warnings
4
from dataclasses import dataclass, field
5
from typing import (TYPE_CHECKING, Any, ClassVar, Dict, Final, List, Literal,
6
Mapping, Optional, Set, Tuple, Type, Union)
@@ -74,9 +75,6 @@ class ModelConfig:
75
code_revision: The specific revision to use for the model code on
76
Hugging Face Hub. It can be a branch name, a tag name, or a
77
commit id. If unspecified, will use the default version.
77
- rope_scaling: Dictionary containing the scaling configuration for the
78
- RoPE embeddings. When using this flag, don't update
79
- `max_position_embeddings` to the expected new maximum.
78
tokenizer_revision: The specific tokenizer version to use. It can be a
79
branch name, a tag name, or a commit id. If unspecified, will use
80
the default version.
@@ -116,6 +114,7 @@ class ModelConfig:
114
can not be gathered from the vllm arguments.
115
config_format: The config format which shall be loaded.
116
Defaults to 'auto' which defaults to 'hf'.
117
+ hf_overrides: Arguments to be forwarded to the HuggingFace config.
118
mm_processor_kwargs: Arguments to be forwarded to the model's processor
119
for multi-modal data, e.g., image processor.
120
pooling_type: Used to configure the pooling method in the embedding
@@ -146,7 +145,7 @@ class ModelConfig:
145
allowed_local_media_path: str = "",
146
revision: Optional[str] = None,
147
code_revision: Optional[str] = None,
149
- rope_scaling: Optional[dict] = None,
148
+ rope_scaling: Optional[Dict[str, Any]] = None,
149
rope_theta: Optional[float] = None,
150
tokenizer_revision: Optional[str] = None,
151
max_model_len: Optional[int] = None,
@@ -164,6 +163,7 @@ class ModelConfig:
163
override_neuron_config: Optional[Dict[str, Any]] = None,
164
config_format: ConfigFormat = ConfigFormat.AUTO,
165
chat_template_text_format: str = "string",
166
+ hf_overrides: Optional[Dict[str, Any]] = None,
167
mm_processor_kwargs: Optional[Dict[str, Any]] = None,
168
pooling_type: Optional[str] = None,
169
pooling_norm: Optional[bool] = None,
@@ -178,8 +178,22 @@ class ModelConfig:
178
self.seed = seed
179
self.revision = revision
180
self.code_revision = code_revision
181
- self.rope_scaling = rope_scaling
182
- self.rope_theta = rope_theta
181
+
182
+ if hf_overrides is None:
183
+ hf_overrides = {}
184
+ if rope_scaling is not None:
185
+ hf_override: Dict[str, Any] = {"rope_scaling": rope_scaling}
186
+ hf_overrides.update(hf_override)
187
+ msg = ("`--rope-scaling` will be removed in a future release. "
188
+ f"'Please instead use `--hf-overrides '{hf_override!r}'`")
189
+ warnings.warn(DeprecationWarning(msg), stacklevel=2)
190
+ if rope_theta is not None:
191
+ hf_override = {"rope_theta": rope_theta}
192
+ hf_overrides.update(hf_override)
193
+ msg = ("`--rope-theta` will be removed in a future release. "
194
+ f"'Please instead use `--hf-overrides '{hf_override!r}'`")
195
+ warnings.warn(DeprecationWarning(msg), stacklevel=2)
196
+
197
# The tokenizer version is consistent with the model version by default.
198
if tokenizer_revision is None:
199
self.tokenizer_revision = revision
@@ -193,8 +207,8 @@ class ModelConfig:
207
self.disable_sliding_window = disable_sliding_window
208
self.skip_tokenizer_init = skip_tokenizer_init
209
self.hf_config = get_config(self.model, trust_remote_code, revision,
196
- code_revision, rope_scaling, rope_theta,
197
- config_format)
210
+ code_revision, config_format,
211
+ **hf_overrides)
212
self.hf_text_config = get_hf_text_config(self.hf_config)
213
self.encoder_config = self._get_encoder_config()
214
self.hf_image_processor_config = get_hf_image_processor_config(
vllm/engine/arg_utils.py
+11
-3
@@ -128,8 +128,9 @@ class EngineArgs:
128
disable_log_stats: bool = False
129
revision: Optional[str] = None
130
code_revision: Optional[str] = None
131
- rope_scaling: Optional[dict] = None
131
+ rope_scaling: Optional[Dict[str, Any]] = None
132
rope_theta: Optional[float] = None
133
+ hf_overrides: Optional[Dict[str, Any]] = None
134
tokenizer_revision: Optional[str] = None
135
quantization: Optional[str] = None
136
enforce_eager: Optional[bool] = None
@@ -140,8 +141,9 @@ class EngineArgs:
141
# is intended for expert use only. The API may change without
142
# notice.
143
tokenizer_pool_type: Union[str, Type["BaseTokenizerGroup"]] = "ray"
143
- tokenizer_pool_extra_config: Optional[dict] = None
144
+ tokenizer_pool_extra_config: Optional[Dict[str, Any]] = None
145
limit_mm_per_prompt: Optional[Mapping[str, int]] = None
146
+ mm_processor_kwargs: Optional[Dict[str, Any]] = None
147
enable_lora: bool = False
148
max_loras: int = 1
149
max_lora_rank: int = 16
@@ -187,7 +189,6 @@ class EngineArgs:
189
collect_detailed_traces: Optional[str] = None
190
disable_async_output_proc: bool = False
191
override_neuron_config: Optional[Dict[str, Any]] = None
190
- mm_processor_kwargs: Optional[Dict[str, Any]] = None
192
scheduling_policy: Literal["fcfs", "priority"] = "fcfs"
193
194
# Pooling configuration.
@@ -512,6 +513,12 @@ class EngineArgs:
513
help='RoPE theta. Use with `rope_scaling`. In '
514
'some cases, changing the RoPE theta improves the '
515
'performance of the scaled model.')
516
+ parser.add_argument('--hf-overrides',
517
+ type=json.loads,
518
+ default=EngineArgs.hf_overrides,
519
+ help='Extra arguments for the HuggingFace config.'
520
+ 'This should be a JSON string that will be '
521
+ 'parsed into a dictionary.')
522
parser.add_argument('--enforce-eager',
523
action='store_true',
524
help='Always use eager-mode PyTorch. If False, '
@@ -940,6 +947,7 @@ class EngineArgs:
947
code_revision=self.code_revision,
948
rope_scaling=self.rope_scaling,
949
rope_theta=self.rope_theta,
950
+ hf_overrides=self.hf_overrides,
951
tokenizer_revision=self.tokenizer_revision,
952
max_model_len=self.max_model_len,
953
quantization=self.quantization,
vllm/engine/llm_engine.py
+1
-4
@@ -248,8 +248,7 @@ class LLMEngine:
248
"Initializing an LLM engine (v%s) with config: "
249
"model=%r, speculative_config=%r, tokenizer=%r, "
250
"skip_tokenizer_init=%s, tokenizer_mode=%s, revision=%s, "
251
- "override_neuron_config=%s, "
252
- "rope_scaling=%r, rope_theta=%r, tokenizer_revision=%s, "
251
+ "override_neuron_config=%s, tokenizer_revision=%s, "
252
"trust_remote_code=%s, dtype=%s, max_seq_len=%d, "
253
"download_dir=%r, load_format=%s, tensor_parallel_size=%d, "
254
"pipeline_parallel_size=%d, "
@@ -271,8 +270,6 @@ class LLMEngine:
270
model_config.tokenizer_mode,
271
model_config.revision,
272
model_config.override_neuron_config,
274
- model_config.rope_scaling,
275
- model_config.rope_theta,
273
model_config.tokenizer_revision,
274
model_config.trust_remote_code,
275
model_config.dtype,
vllm/entrypoints/llm.py
+6
-1
@@ -98,7 +98,10 @@ class LLM:
98
to eager mode. Additionally for encoder-decoder models, if the
99
sequence length of the encoder input is larger than this, we fall
100
back to the eager mode.
101
- disable_custom_all_reduce: See ParallelConfig
101
+ disable_custom_all_reduce: See :class:`~vllm.config.ParallelConfig`
102
+ disable_async_output_proc: Disable async output processing.
103
+ This may result in lower performance.
104
+ hf_overrides: Arguments to be forwarded to the HuggingFace config.
105
**kwargs: Arguments for :class:`~vllm.EngineArgs`. (See
106
:ref:`engine_args`)
107
@@ -153,6 +156,7 @@ class LLM:
156
max_seq_len_to_capture: int = 8192,
157
disable_custom_all_reduce: bool = False,
158
disable_async_output_proc: bool = False,
159
+ hf_overrides: Optional[dict] = None,
160
mm_processor_kwargs: Optional[Dict[str, Any]] = None,
161
# After positional args are removed, move this right below `model`
162
task: TaskOption = "auto",
@@ -194,6 +198,7 @@ class LLM:
198
max_seq_len_to_capture=max_seq_len_to_capture,
199
disable_custom_all_reduce=disable_custom_all_reduce,
200
disable_async_output_proc=disable_async_output_proc,
201
+ hf_overrides=hf_overrides,
202
mm_processor_kwargs=mm_processor_kwargs,
203
pooling_type=pooling_type,
204
pooling_norm=pooling_norm,
vllm/transformers_utils/config.py
+25
-30
@@ -146,9 +146,8 @@ def get_config(
146
trust_remote_code: bool,
147
revision: Optional[str] = None,
148
code_revision: Optional[str] = None,
149
- rope_scaling: Optional[dict] = None,
150
- rope_theta: Optional[float] = None,
149
config_format: ConfigFormat = ConfigFormat.AUTO,
150
+ token: Optional[str] = None,
151
**kwargs,
152
) -> PretrainedConfig:
153
# Separate model folder from file path for GGUF models
@@ -159,39 +158,43 @@ def get_config(
158
model = Path(model).parent
159
160
if config_format == ConfigFormat.AUTO:
162
- if is_gguf or file_or_path_exists(model,
163
- HF_CONFIG_NAME,
164
- revision=revision,
165
- token=kwargs.get("token")):
161
+ if is_gguf or file_or_path_exists(
162
+ model, HF_CONFIG_NAME, revision=revision, token=token):
163
config_format = ConfigFormat.HF
164
elif file_or_path_exists(model,
165
MISTRAL_CONFIG_NAME,
166
revision=revision,
170
- token=kwargs.get("token")):
167
+ token=token):
168
config_format = ConfigFormat.MISTRAL
169
else:
170
# If we're in offline mode and found no valid config format, then
171
# raise an offline mode error to indicate to the user that they
172
# don't have files cached and may need to go online.
173
# This is conveniently triggered by calling file_exists().
177
- file_exists(model,
178
- HF_CONFIG_NAME,
179
- revision=revision,
180
- token=kwargs.get("token"))
174
+ file_exists(model, HF_CONFIG_NAME, revision=revision, token=token)
175
176
raise ValueError(f"No supported config format found in {model}")
177
178
if config_format == ConfigFormat.HF:
179
config_dict, _ = PretrainedConfig.get_config_dict(
186
- model, revision=revision, code_revision=code_revision, **kwargs)
180
+ model,
181
+ revision=revision,
182
+ code_revision=code_revision,
183
+ token=token,
184
+ **kwargs,
185
+ )
186
187
# Use custom model class if it's in our registry
188
model_type = config_dict.get("model_type")
189
if model_type in _CONFIG_REGISTRY:
190
config_class = _CONFIG_REGISTRY[model_type]
192
- config = config_class.from_pretrained(model,
193
- revision=revision,
194
- code_revision=code_revision)
191
+ config = config_class.from_pretrained(
192
+ model,
193
+ revision=revision,
194
+ code_revision=code_revision,
195
+ token=token,
196
+ **kwargs,
197
+ )
198
else:
199
try:
200
config = AutoConfig.from_pretrained(
@@ -199,6 +202,7 @@ def get_config(
202
trust_remote_code=trust_remote_code,
203
revision=revision,
204
code_revision=code_revision,
205
+ token=token,
206
**kwargs,
207
)
208
except ValueError as e:
@@ -216,7 +220,7 @@ def get_config(
220
raise e
221
222
elif config_format == ConfigFormat.MISTRAL:
219
- config = load_params_config(model, revision, token=kwargs.get("token"))
223
+ config = load_params_config(model, revision, token=token, **kwargs)
224
else:
225
raise ValueError(f"Unsupported config format: {config_format}")
226
@@ -228,19 +232,6 @@ def get_config(
232
model_type = MODEL_FOR_CAUSAL_LM_MAPPING_NAMES[config.model_type]
233
config.update({"architectures": [model_type]})
234
231
- for key, value in [
232
- ("rope_scaling", rope_scaling),
233
- ("rope_theta", rope_theta),
234
- ]:
235
- if value is not None:
236
- logger.info(
237
- "Updating %s from %r to %r",
238
- key,
239
- getattr(config, key, None),
240
- value,
241
- )
242
- config.update({key: value})
243
-
235
patch_rope_scaling(config)
236
237
return config
@@ -462,13 +453,15 @@ def maybe_register_config_serialize_by_value(trust_remote_code: bool) -> None:
453
454
def load_params_config(model: Union[str, Path],
455
revision: Optional[str],
465
- token: Optional[str] = None) -> PretrainedConfig:
456
+ token: Optional[str] = None,
457
+ **kwargs) -> PretrainedConfig:
458
# This function loads a params.json config which
459
# should be used when loading models in mistral format
460
461
config_file_name = "params.json"
462
463
config_dict = get_hf_file_to_dict(config_file_name, model, revision, token)
464
+ assert isinstance(config_dict, dict)
465
466
config_mapping = {
467
"dim": "hidden_size",
@@ -512,6 +505,8 @@ def load_params_config(model: Union[str, Path],
505
config_dict["architectures"] = ["PixtralForConditionalGeneration"]
506
config_dict["model_type"] = "pixtral"
507
508
+ config_dict.update(kwargs)
509
+
510
config = recurse_elems(config_dict)
511
return config
512
vllm/v1/engine/llm_engine.py
+1
-4
@@ -74,8 +74,7 @@ class LLMEngine:
74
"Initializing an LLM engine (v%s) with config: "
75
"model=%r, speculative_config=%r, tokenizer=%r, "
76
"skip_tokenizer_init=%s, tokenizer_mode=%s, revision=%s, "
77
- "override_neuron_config=%s, "
78
- "rope_scaling=%r, rope_theta=%r, tokenizer_revision=%s, "
77
+ "override_neuron_config=%s, tokenizer_revision=%s, "
78
"trust_remote_code=%s, dtype=%s, max_seq_len=%d, "
79
"download_dir=%r, load_format=%s, tensor_parallel_size=%d, "
80
"pipeline_parallel_size=%d, "
@@ -94,8 +93,6 @@ class LLMEngine:
93
model_config.tokenizer_mode,
94
model_config.revision,
95
model_config.override_neuron_config,
97
- model_config.rope_scaling,
98
- model_config.rope_theta,
96
model_config.tokenizer_revision,
97
model_config.trust_remote_code,
98
model_config.dtype,