A high-throughput and memory-efficient inference and serving engine for LLMs
Mirrored from https://github.com/tduncan-lum/Nvidia-Optimization-Stack---VLLM.git
Synced Nov 15, 09:45 UTC
Allow users to pass arbitrary JSON keys from CLI (#18208)
Signed-off-by: Harry Mellor <19981378+hmellor@users.noreply.github.com>
Harry Mellor committed
May 16, 2025 at 05:05 UTC
b18201fe060a3ddcc088f8aea3cf1d7c4b461288
4 files changed
+78
-3
tests/engine/test_arg_utils.py
+2
-2
@@ -181,8 +181,8 @@ def test_get_kwargs():
181
# literals of literals should have merged choices
182
assert kwargs["literal_literal"]["choices"] == [1, 2]
183
# dict should have json tip in help
184
- json_tip = "\n\nShould be a valid JSON string."
185
- assert kwargs["json_tip"]["help"].endswith(json_tip)
184
+ json_tip = "Should either be a valid JSON string or JSON keys"
185
+ assert json_tip in kwargs["json_tip"]["help"]
186
# nested config should should construct the nested config
187
assert kwargs["nested_config"]["type"]('{"field": 2}') == NestedConfig(2)
188
# from_cli configs should be constructed with the correct method
tests/test_utils.py
+25
@@ -3,6 +3,7 @@
3
4
import asyncio
5
import hashlib
6
+import json
7
import pickle
8
import socket
9
from collections.abc import AsyncIterator
@@ -138,6 +139,7 @@ def parser():
139
parser.add_argument('--model-name')
140
parser.add_argument('--batch-size', type=int)
141
parser.add_argument('--enable-feature', action='store_true')
142
+ parser.add_argument('--hf-overrides', type=json.loads)
143
return parser
144
145
@@ -251,6 +253,29 @@ def test_no_model_tag(parser_with_config, cli_config_file):
253
parser_with_config.parse_args(['serve', '--config', cli_config_file])
254
255
256
+def test_dict_args(parser):
257
+ args = [
258
+ "--model-name=something.something",
259
+ "--hf-overrides.key1",
260
+ "val1",
261
+ "--hf-overrides.key2.key3",
262
+ "val2",
263
+ "--hf-overrides.key2.key4",
264
+ "val3",
265
+ "--hf-overrides.key5=val4",
266
+ ]
267
+ parsed_args = parser.parse_args(args)
268
+ assert parsed_args.model_name == "something.something"
269
+ assert parsed_args.hf_overrides == {
270
+ "key1": "val1",
271
+ "key2": {
272
+ "key3": "val2",
273
+ "key4": "val3",
274
+ },
275
+ "key5": "val4",
276
+ }
277
+
278
+
279
# yapf: enable
280
@pytest.mark.parametrize(
281
"callable,kw_name,requires_kw_only,allow_var_kwargs,is_supported",
vllm/engine/arg_utils.py
+5
-1
@@ -183,7 +183,11 @@ def get_kwargs(cls: ConfigType) -> dict[str, Any]:
183
kwargs[name] = {"default": default, "help": help}
184
185
# Set other kwargs based on the type hints
186
- json_tip = "\n\nShould be a valid JSON string."
186
+ json_tip = """\n\nShould either be a valid JSON string or JSON keys
187
+ passed individually. For example, the following sets of arguments are
188
+ equivalent:\n\n
189
+ - `--json-arg '{"key1": "value1", "key2": {"key3": "value2"}}'`\n
190
+ - `--json-arg.key1 value1 --json-arg.key2.key3 value2`\n\n"""
191
if dataclass_cls is not None:
192
dataclass_init = lambda x, f=dataclass_cls: f(**json.loads(x))
193
# Special case for configs with a from_cli method
vllm/utils.py
+46
@@ -15,6 +15,7 @@ import importlib.metadata
15
import importlib.util
16
import inspect
17
import ipaddress
18
+import json
19
import multiprocessing
20
import os
21
import pickle
@@ -1419,6 +1420,51 @@ class FlexibleArgumentParser(ArgumentParser):
1420
else:
1421
processed_args.append(arg)
1422
1423
+ def create_nested_dict(keys: list[str], value: str):
1424
+ """Creates a nested dictionary from a list of keys and a value.
1425
+
1426
+ For example, `keys = ["a", "b", "c"]` and `value = 1` will create:
1427
+ `{"a": {"b": {"c": 1}}}`
1428
+ """
1429
+ nested_dict: Any = value
1430
+ for key in reversed(keys):
1431
+ nested_dict = {key: nested_dict}
1432
+ return nested_dict
1433
+
1434
+ def recursive_dict_update(original: dict, update: dict):
1435
+ """Recursively updates a dictionary with another dictionary."""
1436
+ for k, v in update.items():
1437
+ if isinstance(v, dict) and isinstance(original.get(k), dict):
1438
+ recursive_dict_update(original[k], v)
1439
+ else:
1440
+ original[k] = v
1441
+
1442
+ delete = set()
1443
+ dict_args: dict[str, dict] = defaultdict(dict)
1444
+ for i, processed_arg in enumerate(processed_args):
1445
+ if processed_arg.startswith("--") and "." in processed_arg:
1446
+ if "=" in processed_arg:
1447
+ processed_arg, value = processed_arg.split("=", 1)
1448
+ if "." not in processed_arg:
1449
+ # False positive, . was only in the value
1450
+ continue
1451
+ else:
1452
+ value = processed_args[i + 1]
1453
+ delete.add(i + 1)
1454
+ key, *keys = processed_arg.split(".")
1455
+ # Merge all values with the same key into a single dict
1456
+ arg_dict = create_nested_dict(keys, value)
1457
+ recursive_dict_update(dict_args[key], arg_dict)
1458
+ delete.add(i)
1459
+ # Filter out the dict args we set to None
1460
+ processed_args = [
1461
+ a for i, a in enumerate(processed_args) if i not in delete
1462
+ ]
1463
+ # Add the dict args back as if they were originally passed as JSON
1464
+ for dict_arg, dict_value in dict_args.items():
1465
+ processed_args.append(dict_arg)
1466
+ processed_args.append(json.dumps(dict_value))
1467
+
1468
return super().parse_args(processed_args, namespace)
1469
1470
def check_port(self, value):