A high-throughput and memory-efficient inference and serving engine for LLMs
Mirrored from https://github.com/tduncan-lum/Nvidia-Optimization-Stack---VLLM.git
Synced Nov 14, 05:30 UTC
[Core] Support configuration parsing plugin (#24277)
Signed-off-by: Xingyu Liu <charlotteliu12x@gmail.com> Signed-off-by: Xingyu Liu <38244988+charlotte12l@users.noreply.github.com> Co-authored-by: gemini-code-assist[bot] <176961590+gemini-code-assist[bot]@users.noreply.github.com>
Xingyu Liu committed
Sep 10, 2025 at 11:32 UTC
9fb74c27a78cdc8d8c6bd0a917d036b776784956
6 files changed
+237
-107
tests/transformers_utils/__init__.py
tests/transformers_utils/test_config_parser_registry.py
new
+37
@@ -0,0 +1,37 @@
1
+# SPDX-License-Identifier: Apache-2.0
2
+# SPDX-FileCopyrightText: Copyright contributors to the vLLM project
3
+
4
+from pathlib import Path
5
+from typing import Optional, Union
6
+
7
+import pytest
8
+from transformers import PretrainedConfig
9
+
10
+from vllm.transformers_utils.config import (get_config_parser,
11
+ register_config_parser)
12
+from vllm.transformers_utils.config_parser_base import ConfigParserBase
13
+
14
+
15
+@register_config_parser("custom_config_parser")
16
+class CustomConfigParser(ConfigParserBase):
17
+
18
+ def parse(self,
19
+ model: Union[str, Path],
20
+ trust_remote_code: bool,
21
+ revision: Optional[str] = None,
22
+ code_revision: Optional[str] = None,
23
+ **kwargs) -> tuple[dict, PretrainedConfig]:
24
+ raise NotImplementedError
25
+
26
+
27
+def test_register_config_parser():
28
+ assert isinstance(get_config_parser("custom_config_parser"),
29
+ CustomConfigParser)
30
+
31
+
32
+def test_invalid_config_parser():
33
+ with pytest.raises(ValueError):
34
+
35
+ @register_config_parser("invalid_config_parser")
36
+ class InvalidConfigParser:
37
+ pass
vllm/config/__init__.py
+1
-4
@@ -419,7 +419,7 @@ class ModelConfig:
419
`--media-io-kwargs '{"video": {"num_frames": 40} }'` """
420
use_async_output_proc: bool = True
421
"""Whether to use async output processor."""
422
- config_format: Union[str, ConfigFormat] = ConfigFormat.AUTO.value
422
+ config_format: Union[str, ConfigFormat] = "auto"
423
"""The format of the model config to load:\n
424
- "auto" will try to load the config in hf format if available else it
425
will try to load in mistral format.\n
@@ -624,9 +624,6 @@ class ModelConfig:
624
raise ValueError(
625
"Sleep mode is not supported on current platform.")
626
627
- if isinstance(self.config_format, str):
628
- self.config_format = ConfigFormat(self.config_format)
629
-
627
hf_config = get_config(self.hf_config_path or self.model,
628
self.trust_remote_code,
629
self.revision,
vllm/engine/arg_utils.py
+3
-4
@@ -22,9 +22,9 @@ from typing_extensions import TypeIs, deprecated
22
23
import vllm.envs as envs
24
from vllm.config import (BlockSize, CacheConfig, CacheDType, CompilationConfig,
25
- ConfigFormat, ConfigType, ConvertOption,
26
- DecodingConfig, DetailedTraceModules, Device,
27
- DeviceConfig, DistributedExecutorBackend, EPLBConfig,
25
+ ConfigType, ConvertOption, DecodingConfig,
26
+ DetailedTraceModules, Device, DeviceConfig,
27
+ DistributedExecutorBackend, EPLBConfig,
28
GuidedDecodingBackend, HfOverrides, KVEventsConfig,
29
KVTransferConfig, LoadConfig, LogprobsMode,
30
LoRAConfig, MambaDType, MMEncoderTPMode, ModelConfig,
@@ -547,7 +547,6 @@ class EngineArgs:
547
help="Disable async output processing. This may result in "
548
"lower performance.")
549
model_group.add_argument("--config-format",
550
- choices=[f.value for f in ConfigFormat],
550
**model_kwargs["config_format"])
551
# This one is a special case because it can bool
552
# or str. TODO: Handle this in get_kwargs
vllm/transformers_utils/config.py
+176
-99
@@ -1,13 +1,12 @@
1
# SPDX-License-Identifier: Apache-2.0
2
# SPDX-FileCopyrightText: Copyright contributors to the vLLM project
3
4
-import enum
4
import json
5
import os
6
import time
7
from functools import cache, partial
8
from pathlib import Path
10
-from typing import Any, Callable, Optional, TypeVar, Union
9
+from typing import Any, Callable, Literal, Optional, TypeVar, Union
10
11
import huggingface_hub
12
from huggingface_hub import get_safetensors_metadata, hf_hub_download
@@ -27,6 +26,7 @@ from transformers.utils import CONFIG_NAME as HF_CONFIG_NAME
26
27
from vllm import envs
28
from vllm.logger import init_logger
29
+from vllm.transformers_utils.config_parser_base import ConfigParserBase
30
from vllm.transformers_utils.utils import check_gguf_file
31
32
if envs.VLLM_USE_MODELSCOPE:
@@ -100,10 +100,163 @@ _AUTO_CONFIG_KWARGS_OVERRIDES: dict[str, dict[str, Any]] = {
100
}
101
102
103
-class ConfigFormat(str, enum.Enum):
104
- AUTO = "auto"
105
- HF = "hf"
106
- MISTRAL = "mistral"
103
+class HFConfigParser(ConfigParserBase):
104
+
105
+ def parse(self,
106
+ model: Union[str, Path],
107
+ trust_remote_code: bool,
108
+ revision: Optional[str] = None,
109
+ code_revision: Optional[str] = None,
110
+ **kwargs) -> tuple[dict, PretrainedConfig]:
111
+ kwargs["local_files_only"] = huggingface_hub.constants.HF_HUB_OFFLINE
112
+ config_dict, _ = PretrainedConfig.get_config_dict(
113
+ model,
114
+ revision=revision,
115
+ code_revision=code_revision,
116
+ token=_get_hf_token(),
117
+ **kwargs,
118
+ )
119
+ # Use custom model class if it's in our registry
120
+ model_type = config_dict.get("model_type")
121
+ if model_type is None:
122
+ model_type = "speculators" if config_dict.get(
123
+ "speculators_config") is not None else model_type
124
+
125
+ if model_type in _CONFIG_REGISTRY:
126
+ config_class = _CONFIG_REGISTRY[model_type]
127
+ config = config_class.from_pretrained(
128
+ model,
129
+ revision=revision,
130
+ code_revision=code_revision,
131
+ token=_get_hf_token(),
132
+ **kwargs,
133
+ )
134
+ else:
135
+ try:
136
+ kwargs = _maybe_update_auto_config_kwargs(
137
+ kwargs, model_type=model_type)
138
+ config = AutoConfig.from_pretrained(
139
+ model,
140
+ trust_remote_code=trust_remote_code,
141
+ revision=revision,
142
+ code_revision=code_revision,
143
+ token=_get_hf_token(),
144
+ **kwargs,
145
+ )
146
+ except ValueError as e:
147
+ if (not trust_remote_code
148
+ and "requires you to execute the configuration file"
149
+ in str(e)):
150
+ err_msg = (
151
+ "Failed to load the model config. If the model "
152
+ "is a custom model not yet available in the "
153
+ "HuggingFace transformers library, consider setting "
154
+ "`trust_remote_code=True` in LLM or using the "
155
+ "`--trust-remote-code` flag in the CLI.")
156
+ raise RuntimeError(err_msg) from e
157
+ else:
158
+ raise e
159
+ config = _maybe_remap_hf_config_attrs(config)
160
+ return config_dict, config
161
+
162
+
163
+class MistralConfigParser(ConfigParserBase):
164
+
165
+ def parse(self,
166
+ model: Union[str, Path],
167
+ trust_remote_code: bool,
168
+ revision: Optional[str] = None,
169
+ code_revision: Optional[str] = None,
170
+ **kwargs) -> tuple[dict, PretrainedConfig]:
171
+ # This function loads a params.json config which
172
+ # should be used when loading models in mistral format
173
+ config_dict = _download_mistral_config_file(model, revision)
174
+ if (max_position_embeddings :=
175
+ config_dict.get("max_position_embeddings")) is None:
176
+ max_position_embeddings = _maybe_retrieve_max_pos_from_hf(
177
+ model, revision, **kwargs)
178
+ config_dict["max_position_embeddings"] = max_position_embeddings
179
+
180
+ from vllm.transformers_utils.configs.mistral import adapt_config_dict
181
+
182
+ config = adapt_config_dict(config_dict)
183
+
184
+ # Mistral configs may define sliding_window as list[int]. Convert it
185
+ # to int and add the layer_types list[str] to make it HF compatible
186
+ if ((sliding_window := getattr(config, "sliding_window", None))
187
+ and isinstance(sliding_window, list)):
188
+ pattern_repeats = config.num_hidden_layers // len(sliding_window)
189
+ layer_types = sliding_window * pattern_repeats
190
+ config.layer_types = [
191
+ "full_attention" if layer_type is None else "sliding_attention"
192
+ for layer_type in layer_types
193
+ ]
194
+ config.sliding_window = next(filter(None, sliding_window), None)
195
+
196
+ return config_dict, config
197
+
198
+
199
+_CONFIG_FORMAT_TO_CONFIG_PARSER: dict[str, type[ConfigParserBase]] = {
200
+ "hf": HFConfigParser,
201
+ "mistral": MistralConfigParser,
202
+}
203
+
204
+ConfigFormat = Literal[
205
+ "auto",
206
+ "hf",
207
+ "mistral",
208
+]
209
+
210
+
211
+def get_config_parser(config_format: str) -> ConfigParserBase:
212
+ """Get the config parser for a given config format."""
213
+ if config_format not in _CONFIG_FORMAT_TO_CONFIG_PARSER:
214
+ raise ValueError(f"Unknown config format `{config_format}`.")
215
+ return _CONFIG_FORMAT_TO_CONFIG_PARSER[config_format]()
216
+
217
+
218
+def register_config_parser(config_format: str):
219
+
220
+ """Register a customized vllm config parser.
221
+ When a config format is not supported by vllm, you can register a customized
222
+ config parser to support it.
223
+ Args:
224
+ config_format (str): The config parser format name.
225
+ Examples:
226
+
227
+ >>> from vllm.transformers_utils.config import (get_config_parser,
228
+ register_config_parser)
229
+ >>> from vllm.transformers_utils.config_parser_base import ConfigParserBase
230
+ >>>
231
+ >>> @register_config_parser("custom_config_parser")
232
+ ... class CustomConfigParser(ConfigParserBase):
233
+ ... def parse(self,
234
+ ... model: Union[str, Path],
235
+ ... trust_remote_code: bool,
236
+ ... revision: Optional[str] = None,
237
+ ... code_revision: Optional[str] = None,
238
+ ... **kwargs) -> tuple[dict, PretrainedConfig]:
239
+ ... raise NotImplementedError
240
+ >>>
241
+ >>> type(get_config_parser("custom_config_parser"))
242
+ <class 'CustomConfigParser'>
243
+ """ # noqa: E501
244
+
245
+ def _wrapper(config_parser_cls):
246
+ if config_format in _CONFIG_FORMAT_TO_CONFIG_PARSER:
247
+ logger.warning(
248
+ "Config format `%s` is already registered, and will be "
249
+ "overwritten by the new parser class `%s`.", config_format,
250
+ config_parser_cls)
251
+ if not issubclass(config_parser_cls, ConfigParserBase):
252
+ raise ValueError("The config parser must be a subclass of "
253
+ "`ConfigParserBase`.")
254
+ _CONFIG_FORMAT_TO_CONFIG_PARSER[config_format] = config_parser_cls
255
+ logger.info("Registered config parser `%s` with config format `%s`",
256
+ config_parser_cls, config_format)
257
+ return config_parser_cls
258
+
259
+ return _wrapper
260
261
262
_R = TypeVar("_R")
@@ -350,7 +503,7 @@ def get_config(
503
trust_remote_code: bool,
504
revision: Optional[str] = None,
505
code_revision: Optional[str] = None,
353
- config_format: ConfigFormat = ConfigFormat.AUTO,
506
+ config_format: Union[str, ConfigFormat] = "auto",
507
hf_overrides_kw: Optional[dict[str, Any]] = None,
508
hf_overrides_fn: Optional[Callable[[PretrainedConfig],
509
PretrainedConfig]] = None,
@@ -363,20 +516,22 @@ def get_config(
516
kwargs["gguf_file"] = Path(model).name
517
model = Path(model).parent
518
366
- if config_format == ConfigFormat.AUTO:
519
+ if config_format == "auto":
520
try:
521
if is_gguf or file_or_path_exists(
522
model, HF_CONFIG_NAME, revision=revision):
370
- config_format = ConfigFormat.HF
523
+ config_format = "hf"
524
elif file_or_path_exists(model,
525
MISTRAL_CONFIG_NAME,
526
revision=revision):
374
- config_format = ConfigFormat.MISTRAL
527
+ config_format = "mistral"
528
else:
529
raise ValueError(
530
"Could not detect config format for no config file found. "
378
- "Ensure your model has either config.json (HF format) "
379
- "or params.json (Mistral format).")
531
+ "With config_format 'auto', ensure your model has either"
532
+ "config.json (HF format) or params.json (Mistral format)."
533
+ "Otherwise please specify your_custom_config_format"
534
+ "in engine args for customized config parser")
535
536
except Exception as e:
537
error_message = (
@@ -395,92 +550,14 @@ def get_config(
550
551
raise ValueError(error_message) from e
552
398
- if config_format == ConfigFormat.HF:
399
- kwargs["local_files_only"] = huggingface_hub.constants.HF_HUB_OFFLINE
400
- config_dict, _ = PretrainedConfig.get_config_dict(
401
- model,
402
- revision=revision,
403
- code_revision=code_revision,
404
- token=_get_hf_token(),
405
- **kwargs,
406
- )
407
- # Use custom model class if it's in our registry
408
- model_type = config_dict.get("model_type")
409
- if model_type is None:
410
- model_type = "speculators" if config_dict.get(
411
- "speculators_config") is not None else model_type
412
-
413
- if model_type in _CONFIG_REGISTRY:
414
- config_class = _CONFIG_REGISTRY[model_type]
415
- config = config_class.from_pretrained(
416
- model,
417
- revision=revision,
418
- code_revision=code_revision,
419
- token=_get_hf_token(),
420
- **kwargs,
421
- )
422
- else:
423
- try:
424
- kwargs = _maybe_update_auto_config_kwargs(
425
- kwargs, model_type=model_type)
426
- config = AutoConfig.from_pretrained(
427
- model,
428
- trust_remote_code=trust_remote_code,
429
- revision=revision,
430
- code_revision=code_revision,
431
- token=_get_hf_token(),
432
- **kwargs,
433
- )
434
- except ValueError as e:
435
- if (not trust_remote_code
436
- and "requires you to execute the configuration file"
437
- in str(e)):
438
- err_msg = (
439
- "Failed to load the model config. If the model "
440
- "is a custom model not yet available in the "
441
- "HuggingFace transformers library, consider setting "
442
- "`trust_remote_code=True` in LLM or using the "
443
- "`--trust-remote-code` flag in the CLI.")
444
- raise RuntimeError(err_msg) from e
445
- else:
446
- raise e
447
- config = _maybe_remap_hf_config_attrs(config)
448
-
449
- elif config_format == ConfigFormat.MISTRAL:
450
- # This function loads a params.json config which
451
- # should be used when loading models in mistral format
452
- config_dict = _download_mistral_config_file(model, revision)
453
- if (max_position_embeddings :=
454
- config_dict.get("max_position_embeddings")) is None:
455
- max_position_embeddings = _maybe_retrieve_max_pos_from_hf(
456
- model, revision, **kwargs)
457
- config_dict["max_position_embeddings"] = max_position_embeddings
458
-
459
- from vllm.transformers_utils.configs.mistral import adapt_config_dict
460
-
461
- config = adapt_config_dict(config_dict)
462
-
463
- # Mistral configs may define sliding_window as list[int]. Convert it
464
- # to int and add the layer_types list[str] to make it HF compatible
465
- if ((sliding_window := getattr(config, "sliding_window", None))
466
- and isinstance(sliding_window, list)):
467
- pattern_repeats = config.num_hidden_layers // len(sliding_window)
468
- layer_types = sliding_window * pattern_repeats
469
- config.layer_types = [
470
- "full_attention" if layer_type is None else "sliding_attention"
471
- for layer_type in layer_types
472
- ]
473
- config.sliding_window = next(filter(None, sliding_window), None)
474
- else:
475
- supported_formats = [
476
- fmt.value for fmt in ConfigFormat if fmt != ConfigFormat.AUTO
477
- ]
478
- raise ValueError(
479
- f"Unsupported config format: {config_format}. "
480
- f"Supported formats are: {', '.join(supported_formats)}. "
481
- f"Ensure your model uses one of these configuration formats "
482
- f"or specify the correct format explicitly.")
483
-
553
+ config_parser = get_config_parser(config_format)
554
+ config_dict, config = config_parser.parse(
555
+ model,
556
+ trust_remote_code=trust_remote_code,
557
+ revision=revision,
558
+ code_revision=code_revision,
559
+ **kwargs,
560
+ )
561
# Special architecture mapping check for GGUF models
562
if is_gguf:
563
if config.model_type not in MODEL_FOR_CAUSAL_LM_MAPPING_NAMES:
@@ -914,7 +991,7 @@ def _maybe_retrieve_max_pos_from_hf(model, revision, **kwargs) -> int:
991
hf_config = get_config(model=model,
992
trust_remote_code=trust_remote_code_val,
993
revision=revision,
917
- config_format=ConfigFormat.HF)
994
+ config_format="hf")
995
if hf_value := hf_config.get_text_config().max_position_embeddings:
996
max_position_embeddings = hf_value
997
except Exception as e:
vllm/transformers_utils/config_parser_base.py
new
+20
@@ -0,0 +1,20 @@
1
+# SPDX-License-Identifier: Apache-2.0
2
+# SPDX-FileCopyrightText: Copyright contributors to the vLLM project
3
+
4
+from abc import ABC, abstractmethod
5
+from pathlib import Path
6
+from typing import Optional, Union
7
+
8
+from transformers import PretrainedConfig
9
+
10
+
11
+class ConfigParserBase(ABC):
12
+
13
+ @abstractmethod
14
+ def parse(self,
15
+ model: Union[str, Path],
16
+ trust_remote_code: bool,
17
+ revision: Optional[str] = None,
18
+ code_revision: Optional[str] = None,
19
+ **kwargs) -> tuple[dict, PretrainedConfig]:
20
+ raise NotImplementedError