@tduncan / Nvidia-Optimization-Stack-VLLM Mirror · read-only / commits / b09895a618
Fork 0

A high-throughput and memory-efficient inference and serving engine for LLMs

Mirrored from https://github.com/tduncan-lum/Nvidia-Optimization-Stack---VLLM.git

Synced Nov 16, 08:15 UTC

[Frontend][Core] Override HF `config.json` via CLI (#5836)

Signed-off-by: DarkLight1337 <tlleungac@connect.ust.hk> Co-authored-by: DarkLight1337 <tlleungac@connect.ust.hk>

Krishna Mandal committed Nov 9, 2024 at 08:19 UTC b09895a61843e654088773851a2b1acae4cdf184
7 files changed +73 -53
tests/test_config.py
+7 -3
@@ -200,8 +200,10 @@ def test_rope_customization():
200 trust_remote_code=False,
201 dtype="float16",
202 seed=0,
203 - rope_scaling=TEST_ROPE_SCALING,
204 - rope_theta=TEST_ROPE_THETA,
203 + hf_overrides={
204 + "rope_scaling": TEST_ROPE_SCALING,
205 + "rope_theta": TEST_ROPE_THETA,
206 + },
207 )
208 assert getattr(llama_model_config.hf_config, "rope_scaling",
209 None) == TEST_ROPE_SCALING
@@ -232,7 +234,9 @@ def test_rope_customization():
234 trust_remote_code=False,
235 dtype="float16",
236 seed=0,
235 - rope_scaling=TEST_ROPE_SCALING,
237 + hf_overrides={
238 + "rope_scaling": TEST_ROPE_SCALING,
239 + },
240 )
241 assert getattr(longchat_model_config.hf_config, "rope_scaling",
242 None) == TEST_ROPE_SCALING
vllm/config.py
+22 -8
@@ -1,5 +1,6 @@
1 import enum
2 import json
3 +import warnings
4 from dataclasses import dataclass, field
5 from typing import (TYPE_CHECKING, Any, ClassVar, Dict, Final, List, Literal,
6 Mapping, Optional, Set, Tuple, Type, Union)
@@ -74,9 +75,6 @@ class ModelConfig:
75 code_revision: The specific revision to use for the model code on
76 Hugging Face Hub. It can be a branch name, a tag name, or a
77 commit id. If unspecified, will use the default version.
77 - rope_scaling: Dictionary containing the scaling configuration for the
78 - RoPE embeddings. When using this flag, don't update
79 - `max_position_embeddings` to the expected new maximum.
78 tokenizer_revision: The specific tokenizer version to use. It can be a
79 branch name, a tag name, or a commit id. If unspecified, will use
80 the default version.
@@ -116,6 +114,7 @@ class ModelConfig:
114 can not be gathered from the vllm arguments.
115 config_format: The config format which shall be loaded.
116 Defaults to 'auto' which defaults to 'hf'.
117 + hf_overrides: Arguments to be forwarded to the HuggingFace config.
118 mm_processor_kwargs: Arguments to be forwarded to the model's processor
119 for multi-modal data, e.g., image processor.
120 pooling_type: Used to configure the pooling method in the embedding
@@ -146,7 +145,7 @@ class ModelConfig:
145 allowed_local_media_path: str = "",
146 revision: Optional[str] = None,
147 code_revision: Optional[str] = None,
149 - rope_scaling: Optional[dict] = None,
148 + rope_scaling: Optional[Dict[str, Any]] = None,
149 rope_theta: Optional[float] = None,
150 tokenizer_revision: Optional[str] = None,
151 max_model_len: Optional[int] = None,
@@ -164,6 +163,7 @@ class ModelConfig:
163 override_neuron_config: Optional[Dict[str, Any]] = None,
164 config_format: ConfigFormat = ConfigFormat.AUTO,
165 chat_template_text_format: str = "string",
166 + hf_overrides: Optional[Dict[str, Any]] = None,
167 mm_processor_kwargs: Optional[Dict[str, Any]] = None,
168 pooling_type: Optional[str] = None,
169 pooling_norm: Optional[bool] = None,
@@ -178,8 +178,22 @@ class ModelConfig:
178 self.seed = seed
179 self.revision = revision
180 self.code_revision = code_revision
181 - self.rope_scaling = rope_scaling
182 - self.rope_theta = rope_theta
181 +
182 + if hf_overrides is None:
183 + hf_overrides = {}
184 + if rope_scaling is not None:
185 + hf_override: Dict[str, Any] = {"rope_scaling": rope_scaling}
186 + hf_overrides.update(hf_override)
187 + msg = ("`--rope-scaling` will be removed in a future release. "
188 + f"'Please instead use `--hf-overrides '{hf_override!r}'`")
189 + warnings.warn(DeprecationWarning(msg), stacklevel=2)
190 + if rope_theta is not None:
191 + hf_override = {"rope_theta": rope_theta}
192 + hf_overrides.update(hf_override)
193 + msg = ("`--rope-theta` will be removed in a future release. "
194 + f"'Please instead use `--hf-overrides '{hf_override!r}'`")
195 + warnings.warn(DeprecationWarning(msg), stacklevel=2)
196 +
197 # The tokenizer version is consistent with the model version by default.
198 if tokenizer_revision is None:
199 self.tokenizer_revision = revision
@@ -193,8 +207,8 @@ class ModelConfig:
207 self.disable_sliding_window = disable_sliding_window
208 self.skip_tokenizer_init = skip_tokenizer_init
209 self.hf_config = get_config(self.model, trust_remote_code, revision,
196 - code_revision, rope_scaling, rope_theta,
197 - config_format)
210 + code_revision, config_format,
211 + **hf_overrides)
212 self.hf_text_config = get_hf_text_config(self.hf_config)
213 self.encoder_config = self._get_encoder_config()
214 self.hf_image_processor_config = get_hf_image_processor_config(
vllm/engine/arg_utils.py
+11 -3
@@ -128,8 +128,9 @@ class EngineArgs:
128 disable_log_stats: bool = False
129 revision: Optional[str] = None
130 code_revision: Optional[str] = None
131 - rope_scaling: Optional[dict] = None
131 + rope_scaling: Optional[Dict[str, Any]] = None
132 rope_theta: Optional[float] = None
133 + hf_overrides: Optional[Dict[str, Any]] = None
134 tokenizer_revision: Optional[str] = None
135 quantization: Optional[str] = None
136 enforce_eager: Optional[bool] = None
@@ -140,8 +141,9 @@ class EngineArgs:
141 # is intended for expert use only. The API may change without
142 # notice.
143 tokenizer_pool_type: Union[str, Type["BaseTokenizerGroup"]] = "ray"
143 - tokenizer_pool_extra_config: Optional[dict] = None
144 + tokenizer_pool_extra_config: Optional[Dict[str, Any]] = None
145 limit_mm_per_prompt: Optional[Mapping[str, int]] = None
146 + mm_processor_kwargs: Optional[Dict[str, Any]] = None
147 enable_lora: bool = False
148 max_loras: int = 1
149 max_lora_rank: int = 16
@@ -187,7 +189,6 @@ class EngineArgs:
189 collect_detailed_traces: Optional[str] = None
190 disable_async_output_proc: bool = False
191 override_neuron_config: Optional[Dict[str, Any]] = None
190 - mm_processor_kwargs: Optional[Dict[str, Any]] = None
192 scheduling_policy: Literal["fcfs", "priority"] = "fcfs"
193
194 # Pooling configuration.
@@ -512,6 +513,12 @@ class EngineArgs:
513 help='RoPE theta. Use with `rope_scaling`. In '
514 'some cases, changing the RoPE theta improves the '
515 'performance of the scaled model.')
516 + parser.add_argument('--hf-overrides',
517 + type=json.loads,
518 + default=EngineArgs.hf_overrides,
519 + help='Extra arguments for the HuggingFace config.'
520 + 'This should be a JSON string that will be '
521 + 'parsed into a dictionary.')
522 parser.add_argument('--enforce-eager',
523 action='store_true',
524 help='Always use eager-mode PyTorch. If False, '
@@ -940,6 +947,7 @@ class EngineArgs:
947 code_revision=self.code_revision,
948 rope_scaling=self.rope_scaling,
949 rope_theta=self.rope_theta,
950 + hf_overrides=self.hf_overrides,
951 tokenizer_revision=self.tokenizer_revision,
952 max_model_len=self.max_model_len,
953 quantization=self.quantization,
vllm/engine/llm_engine.py
+1 -4
@@ -248,8 +248,7 @@ class LLMEngine:
248 "Initializing an LLM engine (v%s) with config: "
249 "model=%r, speculative_config=%r, tokenizer=%r, "
250 "skip_tokenizer_init=%s, tokenizer_mode=%s, revision=%s, "
251 - "override_neuron_config=%s, "
252 - "rope_scaling=%r, rope_theta=%r, tokenizer_revision=%s, "
251 + "override_neuron_config=%s, tokenizer_revision=%s, "
252 "trust_remote_code=%s, dtype=%s, max_seq_len=%d, "
253 "download_dir=%r, load_format=%s, tensor_parallel_size=%d, "
254 "pipeline_parallel_size=%d, "
@@ -271,8 +270,6 @@ class LLMEngine:
270 model_config.tokenizer_mode,
271 model_config.revision,
272 model_config.override_neuron_config,
274 - model_config.rope_scaling,
275 - model_config.rope_theta,
273 model_config.tokenizer_revision,
274 model_config.trust_remote_code,
275 model_config.dtype,
vllm/entrypoints/llm.py
+6 -1
@@ -98,7 +98,10 @@ class LLM:
98 to eager mode. Additionally for encoder-decoder models, if the
99 sequence length of the encoder input is larger than this, we fall
100 back to the eager mode.
101 - disable_custom_all_reduce: See ParallelConfig
101 + disable_custom_all_reduce: See :class:`~vllm.config.ParallelConfig`
102 + disable_async_output_proc: Disable async output processing.
103 + This may result in lower performance.
104 + hf_overrides: Arguments to be forwarded to the HuggingFace config.
105 **kwargs: Arguments for :class:`~vllm.EngineArgs`. (See
106 :ref:`engine_args`)
107
@@ -153,6 +156,7 @@ class LLM:
156 max_seq_len_to_capture: int = 8192,
157 disable_custom_all_reduce: bool = False,
158 disable_async_output_proc: bool = False,
159 + hf_overrides: Optional[dict] = None,
160 mm_processor_kwargs: Optional[Dict[str, Any]] = None,
161 # After positional args are removed, move this right below `model`
162 task: TaskOption = "auto",
@@ -194,6 +198,7 @@ class LLM:
198 max_seq_len_to_capture=max_seq_len_to_capture,
199 disable_custom_all_reduce=disable_custom_all_reduce,
200 disable_async_output_proc=disable_async_output_proc,
201 + hf_overrides=hf_overrides,
202 mm_processor_kwargs=mm_processor_kwargs,
203 pooling_type=pooling_type,
204 pooling_norm=pooling_norm,
vllm/transformers_utils/config.py
+25 -30
@@ -146,9 +146,8 @@ def get_config(
146 trust_remote_code: bool,
147 revision: Optional[str] = None,
148 code_revision: Optional[str] = None,
149 - rope_scaling: Optional[dict] = None,
150 - rope_theta: Optional[float] = None,
149 config_format: ConfigFormat = ConfigFormat.AUTO,
150 + token: Optional[str] = None,
151 **kwargs,
152 ) -> PretrainedConfig:
153 # Separate model folder from file path for GGUF models
@@ -159,39 +158,43 @@ def get_config(
158 model = Path(model).parent
159
160 if config_format == ConfigFormat.AUTO:
162 - if is_gguf or file_or_path_exists(model,
163 - HF_CONFIG_NAME,
164 - revision=revision,
165 - token=kwargs.get("token")):
161 + if is_gguf or file_or_path_exists(
162 + model, HF_CONFIG_NAME, revision=revision, token=token):
163 config_format = ConfigFormat.HF
164 elif file_or_path_exists(model,
165 MISTRAL_CONFIG_NAME,
166 revision=revision,
170 - token=kwargs.get("token")):
167 + token=token):
168 config_format = ConfigFormat.MISTRAL
169 else:
170 # If we're in offline mode and found no valid config format, then
171 # raise an offline mode error to indicate to the user that they
172 # don't have files cached and may need to go online.
173 # This is conveniently triggered by calling file_exists().
177 - file_exists(model,
178 - HF_CONFIG_NAME,
179 - revision=revision,
180 - token=kwargs.get("token"))
174 + file_exists(model, HF_CONFIG_NAME, revision=revision, token=token)
175
176 raise ValueError(f"No supported config format found in {model}")
177
178 if config_format == ConfigFormat.HF:
179 config_dict, _ = PretrainedConfig.get_config_dict(
186 - model, revision=revision, code_revision=code_revision, **kwargs)
180 + model,
181 + revision=revision,
182 + code_revision=code_revision,
183 + token=token,
184 + **kwargs,
185 + )
186
187 # Use custom model class if it's in our registry
188 model_type = config_dict.get("model_type")
189 if model_type in _CONFIG_REGISTRY:
190 config_class = _CONFIG_REGISTRY[model_type]
192 - config = config_class.from_pretrained(model,
193 - revision=revision,
194 - code_revision=code_revision)
191 + config = config_class.from_pretrained(
192 + model,
193 + revision=revision,
194 + code_revision=code_revision,
195 + token=token,
196 + **kwargs,
197 + )
198 else:
199 try:
200 config = AutoConfig.from_pretrained(
@@ -199,6 +202,7 @@ def get_config(
202 trust_remote_code=trust_remote_code,
203 revision=revision,
204 code_revision=code_revision,
205 + token=token,
206 **kwargs,
207 )
208 except ValueError as e:
@@ -216,7 +220,7 @@ def get_config(
220 raise e
221
222 elif config_format == ConfigFormat.MISTRAL:
219 - config = load_params_config(model, revision, token=kwargs.get("token"))
223 + config = load_params_config(model, revision, token=token, **kwargs)
224 else:
225 raise ValueError(f"Unsupported config format: {config_format}")
226
@@ -228,19 +232,6 @@ def get_config(
232 model_type = MODEL_FOR_CAUSAL_LM_MAPPING_NAMES[config.model_type]
233 config.update({"architectures": [model_type]})
234
231 - for key, value in [
232 - ("rope_scaling", rope_scaling),
233 - ("rope_theta", rope_theta),
234 - ]:
235 - if value is not None:
236 - logger.info(
237 - "Updating %s from %r to %r",
238 - key,
239 - getattr(config, key, None),
240 - value,
241 - )
242 - config.update({key: value})
243 -
235 patch_rope_scaling(config)
236
237 return config
@@ -462,13 +453,15 @@ def maybe_register_config_serialize_by_value(trust_remote_code: bool) -> None:
453
454 def load_params_config(model: Union[str, Path],
455 revision: Optional[str],
465 - token: Optional[str] = None) -> PretrainedConfig:
456 + token: Optional[str] = None,
457 + **kwargs) -> PretrainedConfig:
458 # This function loads a params.json config which
459 # should be used when loading models in mistral format
460
461 config_file_name = "params.json"
462
463 config_dict = get_hf_file_to_dict(config_file_name, model, revision, token)
464 + assert isinstance(config_dict, dict)
465
466 config_mapping = {
467 "dim": "hidden_size",
@@ -512,6 +505,8 @@ def load_params_config(model: Union[str, Path],
505 config_dict["architectures"] = ["PixtralForConditionalGeneration"]
506 config_dict["model_type"] = "pixtral"
507
508 + config_dict.update(kwargs)
509 +
510 config = recurse_elems(config_dict)
511 return config
512
vllm/v1/engine/llm_engine.py
+1 -4
@@ -74,8 +74,7 @@ class LLMEngine:
74 "Initializing an LLM engine (v%s) with config: "
75 "model=%r, speculative_config=%r, tokenizer=%r, "
76 "skip_tokenizer_init=%s, tokenizer_mode=%s, revision=%s, "
77 - "override_neuron_config=%s, "
78 - "rope_scaling=%r, rope_theta=%r, tokenizer_revision=%s, "
77 + "override_neuron_config=%s, tokenizer_revision=%s, "
78 "trust_remote_code=%s, dtype=%s, max_seq_len=%d, "
79 "download_dir=%r, load_format=%s, tensor_parallel_size=%d, "
80 "pipeline_parallel_size=%d, "
@@ -94,8 +93,6 @@ class LLMEngine:
93 model_config.tokenizer_mode,
94 model_config.revision,
95 model_config.override_neuron_config,
97 - model_config.rope_scaling,
98 - model_config.rope_theta,
96 model_config.tokenizer_revision,
97 model_config.trust_remote_code,
98 model_config.dtype,