A high-throughput and memory-efficient inference and serving engine for LLMs
Mirrored from https://github.com/tduncan-lum/Nvidia-Optimization-Stack---VLLM.git
Synced Nov 15, 12:15 UTC
Automatically tell users that dict args must be valid JSON in CLI (#17577)
Signed-off-by: Harry Mellor <19981378+hmellor@users.noreply.github.com>
Harry Mellor committed
May 2, 2025 at 13:24 UTC
785d75a03b73a903ff86cd9aa23a3addcdbbd8ab
3 files changed
+15
-13
tests/engine/test_arg_utils.py
+5
@@ -106,6 +106,8 @@ class DummyConfigClass:
106
"""List with literal choices"""
107
literal_literal: Literal[Literal[1], Literal[2]] = 1
108
"""Literal of literals with default 1"""
109
+ json_tip: dict = field(default_factory=dict)
110
+ """Dict which will be JSON in CLI"""
111
112
113
@pytest.mark.parametrize(("type_hint", "expected"), [
@@ -137,6 +139,9 @@ def test_get_kwargs():
139
assert kwargs["list_literal"]["choices"] == [1, 2]
140
# literals of literals should have merged choices
141
assert kwargs["literal_literal"]["choices"] == [1, 2]
142
+ # dict should have json tip in help
143
+ json_tip = "\n\nShould be a valid JSON string."
144
+ assert kwargs["json_tip"]["help"].endswith(json_tip)
145
146
147
@pytest.mark.parametrize(("arg", "expected"), [
vllm/config.py
+7
-12
@@ -268,7 +268,7 @@ class ModelConfig:
268
It can be a branch name, a tag name, or a commit id. If unspecified, will
269
use the default version."""
270
rope_scaling: dict[str, Any] = field(default_factory=dict)
271
- """RoPE scaling configuration in JSON format. For example,
271
+ """RoPE scaling configuration. For example,
272
`{"rope_type":"dynamic","factor":2.0}`."""
273
rope_theta: Optional[float] = None
274
"""RoPE theta. Use with `rope_scaling`. In some cases, changing the RoPE
@@ -346,14 +346,13 @@ class ModelConfig:
346
(stored in `~/.huggingface`)."""
347
hf_overrides: HfOverrides = field(default_factory=dict)
348
"""If a dictionary, contains arguments to be forwarded to the Hugging Face
349
- config. If a callable, it is called to update the HuggingFace config. When
350
- specified via CLI, the argument must be a valid JSON string."""
349
+ config. If a callable, it is called to update the HuggingFace config."""
350
mm_processor_kwargs: Optional[dict[str, Any]] = None
351
"""Arguments to be forwarded to the model's processor for multi-modal data,
352
e.g., image processor. Overrides for the multi-modal processor obtained
353
from `AutoProcessor.from_pretrained`. The available overrides depend on the
354
model that is being run. For example, for Phi-3-Vision: `{"num_crops": 4}`.
356
- When specified via CLI, the argument must be a valid JSON string."""
355
+ """
356
disable_mm_preprocessor_cache: bool = False
357
"""If `True`, disable caching of the multi-modal preprocessor/mapper (not
358
recommended)."""
@@ -361,15 +360,14 @@ class ModelConfig:
360
"""Initialize non-default neuron config or override default neuron config
361
that are specific to Neuron devices, this argument will be used to
362
configure the neuron config that can not be gathered from the vllm
364
- arguments. e.g. `{"cast_logits_dtype": "bloat16"}`. When specified via CLI,
365
- the argument must be a valid JSON string."""
363
+ arguments. e.g. `{"cast_logits_dtype": "bloat16"}`."""
364
pooler_config: Optional["PoolerConfig"] = field(init=False)
365
"""Pooler config which controls the behaviour of output pooling in pooling
366
models."""
367
override_pooler_config: Optional[Union[dict, "PoolerConfig"]] = None
368
"""Initialize non-default pooling config or override default pooling config
369
for the pooling model. e.g. `{"pooling_type": "mean", "normalize": false}`.
372
- When specified via CLI, the argument must be a valid JSON string."""
370
+ """
371
logits_processor_pattern: Optional[str] = None
372
"""Optional regex pattern specifying valid logits processor qualified names
373
that can be passed with the `logits_processors` extra completion argument.
@@ -385,8 +383,7 @@ class ModelConfig:
383
"""Overrides or sets generation config. e.g. `{"temperature": 0.5}`. If
384
used with `--generation-config auto`, the override parameters will be
385
merged with the default config from the model. If used with
388
- `--generation-config vllm`, only the override parameters are used.
389
- When specified via CLI, the argument must be a valid JSON string."""
386
+ `--generation-config vllm`, only the override parameters are used."""
387
enable_sleep_mode: bool = False
388
"""Enable sleep mode for the engine (only cuda platform is supported)."""
389
model_impl: Union[str, ModelImpl] = ModelImpl.AUTO.value
@@ -1556,8 +1553,7 @@ class LoadConfig:
1553
cache directory of Hugging Face."""
1554
model_loader_extra_config: dict = field(default_factory=dict)
1555
"""Extra config for model loader. This will be passed to the model loader
1559
- corresponding to the chosen load_format. This should be a JSON string that
1560
- will be parsed into a dictionary."""
1556
+ corresponding to the chosen load_format."""
1557
ignore_patterns: Optional[Union[list[str], str]] = None
1558
"""The list of patterns to ignore when loading the model. Default to
1559
"original/**/*" to avoid repeated loading of llama's checkpoints."""
@@ -2826,7 +2822,6 @@ class MultiModalConfig:
2822
"limit_mm_per_prompt")
2823
"""
2824
The maximum number of input items allowed per prompt for each modality.
2829
- This should be a JSON string that will be parsed into a dictionary.
2825
Defaults to 1 (V0) or 999 (V1) for each modality.
2826
2827
For example, to allow up to 16 images and 2 videos per prompt:
vllm/engine/arg_utils.py
+3
-1
@@ -150,7 +150,7 @@ def get_kwargs(cls: ConfigType) -> dict[str, Any]:
150
151
# Get the help text for the field
152
name = field.name
153
- help = cls_docs[name]
153
+ help = cls_docs[name].strip()
154
# Escape % for argparse
155
help = help.replace("%", "%%")
156
@@ -165,6 +165,7 @@ def get_kwargs(cls: ConfigType) -> dict[str, Any]:
165
type_hints.add(field.type)
166
167
# Set other kwargs based on the type hints
168
+ json_tip = "\n\nShould be a valid JSON string."
169
if contains_type(type_hints, bool):
170
# Creates --no-<name> and --<name> flags
171
kwargs[name]["action"] = argparse.BooleanOptionalAction
@@ -201,6 +202,7 @@ def get_kwargs(cls: ConfigType) -> dict[str, Any]:
202
elif contains_type(type_hints, dict):
203
# Dict arguments will always be optional
204
kwargs[name]["type"] = optional_type(json.loads)
205
+ kwargs[name]["help"] += json_tip
206
elif (contains_type(type_hints, str)
207
or any(is_not_builtin(th) for th in type_hints)):
208
kwargs[name]["type"] = str