A high-throughput and memory-efficient inference and serving engine for LLMs
Mirrored from https://github.com/tduncan-lum/Nvidia-Optimization-Stack---VLLM.git
Synced Nov 4, 18:45 UTC
[Misc] Support Human-readable (k/K/m/M..) json cli arg (#40473)
Signed-off-by: NickLucche <nlucches@redhat.com>
Nicolò Lucchesi committed
Apr 23, 2026 at 09:42 UTC
9c2492e501d91d5c69a163084123a06fd9ce25d3
3 files changed
+130
-67
tests/engine/test_arg_utils.py
+30
@@ -12,6 +12,7 @@ from pydantic import Field
12
from vllm.config import AttentionConfig, CompilationConfig, config
13
from vllm.engine.arg_utils import (
14
EngineArgs,
15
+ _expand_json_human_readable_numbers,
16
contains_type,
17
get_kwargs,
18
get_type,
@@ -563,3 +564,32 @@ def test_ir_op_priority():
564
ir_op_priority=ir_op_priority,
565
kernel_config=KernelConfig(ir_op_priority=ir_op_priority),
566
).create_engine_config()
567
+
568
+
569
+@pytest.mark.parametrize(
570
+ ("input_json", "expected_json"),
571
+ [
572
+ # Decimal suffixes (lowercase)
573
+ ('{"x": 80g}', '{"x": 80000000000}'),
574
+ ('{"x": 1k}', '{"x": 1000}'),
575
+ ('{"x": 5m}', '{"x": 5000000}'),
576
+ ('{"x": 2t}', '{"x": 2000000000000}'),
577
+ # Binary suffixes (uppercase)
578
+ ('{"x": 1K}', f'{{"x": {2**10}}}'),
579
+ ('{"x": 1G}', f'{{"x": {2**30}}}'),
580
+ # Decimal values
581
+ ('{"x": 1.5g}', '{"x": 1500000000}'),
582
+ # Quoted strings must NOT be modified
583
+ ('{"my_key": 80g}', '{"my_key": 80000000000}'),
584
+ ('{"name": "80g"}', '{"name": "80g"}'),
585
+ ('{"model_name": "foo_bar"}', '{"model_name": "foo_bar"}'),
586
+ # Multiple values
587
+ ('{"a": 1k, "b": 2m}', '{"a": 1000, "b": 2000000}'),
588
+ # Plain numbers are untouched
589
+ ('{"x": 42}', '{"x": 42}'),
590
+ # Nested JSON
591
+ ('{"outer": {"inner": 10g}}', '{"outer": {"inner": 10000000000}}'),
592
+ ],
593
+)
594
+def test_expand_json_human_readable_numbers(input_json, expected_json):
595
+ assert _expand_json_human_readable_numbers(input_json) == expected_json
vllm/engine/arg_utils.py
+28
-66
@@ -105,7 +105,11 @@ from vllm.transformers_utils.config import (
105
from vllm.transformers_utils.gguf_utils import is_gguf
106
from vllm.transformers_utils.repo_utils import get_model_path
107
from vllm.transformers_utils.utils import is_cloud_storage
108
-from vllm.utils.argparse_utils import FlexibleArgumentParser
108
+from vllm.utils.argparse_utils import (
109
+ FlexibleArgumentParser,
110
+ human_readable_int,
111
+ human_readable_int_or_auto,
112
+)
113
from vllm.utils.mem_constants import GiB_bytes
114
from vllm.utils.network_utils import get_ip
115
from vllm.utils.torch_utils import resolve_kv_cache_dtype_string
@@ -256,6 +260,28 @@ def _maybe_add_docs_url(cls: Any) -> str:
260
return f"\n\nAPI docs: https://docs.vllm.ai/en/{version}/api/vllm/config/#vllm.config.{cls.__name__}"
261
262
263
+def _expand_json_human_readable_numbers(val: str) -> str:
264
+ """Expand human-readable number suffixes in a JSON string.
265
+
266
+ Based on :func:`human_readable_int` so that the ``k/m/g/t`` (decimal) and
267
+ ``K/M/G/T`` (binary) conventions work out the box.
268
+ Also works inside JSON config arguments such
269
+ as ``--kv-transfer-config '{"cpu_bytes_to_use": 80m}'``.
270
+
271
+ Only bare (unquoted) tokens are replaced so that JSON string values
272
+ like ``"model_name"`` are never modified.
273
+ """
274
+ # Split on quoted strings so we only touch non-string regions.
275
+ parts = re.split(r'("(?:[^"\\]|\\.)*")', val)
276
+ for i in range(0, len(parts), 2): # even indices = outside strings
277
+ parts[i] = re.sub(
278
+ r"\b\d+(?:\.\d+)?[kKmMgGtT]\b",
279
+ lambda m: str(human_readable_int(m.group())),
280
+ parts[i],
281
+ )
282
+ return "".join(parts)
283
+
284
+
285
@functools.lru_cache(maxsize=30)
286
def _compute_kwargs(cls: ConfigType) -> dict[str, dict[str, Any]]:
287
# Save time only getting attr docs if we're generating help text
@@ -301,6 +327,7 @@ def _compute_kwargs(cls: ConfigType) -> dict[str, dict[str, Any]]:
327
328
def parse_dataclass(val: str, cls=dataclass_cls) -> Any:
329
try:
330
+ val = _expand_json_human_readable_numbers(val)
331
return TypeAdapter(cls).validate_json(val)
332
except ValidationError as e:
333
raise argparse.ArgumentTypeError(repr(e)) from e
@@ -2419,68 +2446,3 @@ def _raise_unsupported_error(feature_name: str):
2446
f"remove {feature_name} from your config."
2447
)
2448
raise NotImplementedError(msg)
2422
-
2423
-
2424
-def human_readable_int(value: str) -> int:
2425
- """Parse human-readable integers like '1k', '2M', etc.
2426
- Including decimal values with decimal multipliers.
2427
-
2428
- Examples:
2429
- - '1k' -> 1,000
2430
- - '1K' -> 1,024
2431
- - '25.6k' -> 25,600
2432
- """
2433
- value = value.strip()
2434
-
2435
- match = re.fullmatch(r"(\d+(?:\.\d+)?)([kKmMgGtT])", value)
2436
- if match:
2437
- decimal_multiplier = {
2438
- "k": 10**3,
2439
- "m": 10**6,
2440
- "g": 10**9,
2441
- "t": 10**12,
2442
- }
2443
- binary_multiplier = {
2444
- "K": 2**10,
2445
- "M": 2**20,
2446
- "G": 2**30,
2447
- "T": 2**40,
2448
- }
2449
-
2450
- number, suffix = match.groups()
2451
- if suffix in decimal_multiplier:
2452
- mult = decimal_multiplier[suffix]
2453
- return int(float(number) * mult)
2454
- elif suffix in binary_multiplier:
2455
- mult = binary_multiplier[suffix]
2456
- # Do not allow decimals with binary multipliers
2457
- try:
2458
- return int(number) * mult
2459
- except ValueError as e:
2460
- raise argparse.ArgumentTypeError(
2461
- "Decimals are not allowed "
2462
- f"with binary suffixes like {suffix}. Did you mean to use "
2463
- f"{number}{suffix.lower()} instead?"
2464
- ) from e
2465
-
2466
- # Regular plain number.
2467
- return int(value)
2468
-
2469
-
2470
-def human_readable_int_or_auto(value: str) -> int:
2471
- """Parse human-readable integers like '1k', '2M', etc.
2472
- Including decimal values with decimal multipliers.
2473
- Also accepts -1 or 'auto' as a special value for auto-detection.
2474
-
2475
- Examples:
2476
- - '1k' -> 1,000
2477
- - '1K' -> 1,024
2478
- - '25.6k' -> 25,600
2479
- - '-1' or 'auto' -> -1 (special value for auto-detection)
2480
- """
2481
- value = value.strip()
2482
-
2483
- if value == "-1" or value.lower() == "auto":
2484
- return -1
2485
-
2486
- return human_readable_int(value)
vllm/utils/argparse_utils.py
+72
-1
@@ -2,6 +2,7 @@
2
# SPDX-FileCopyrightText: Copyright contributors to the vLLM project
3
"""Argument parsing utilities for vLLM."""
4
5
+import argparse
6
import json
7
import sys
8
import textwrap
@@ -25,6 +26,71 @@ from vllm.logger import init_logger
26
logger = init_logger(__name__)
27
28
29
+def human_readable_int(value: str) -> int:
30
+ """Parse human-readable integers like '1k', '2M', etc.
31
+ Including decimal values with decimal multipliers.
32
+
33
+ Examples:
34
+ - '1k' -> 1,000
35
+ - '1K' -> 1,024
36
+ - '25.6k' -> 25,600
37
+ """
38
+ value = value.strip()
39
+
40
+ match = re.fullmatch(r"(\d+(?:\.\d+)?)([kKmMgGtT])", value)
41
+ if match:
42
+ decimal_multiplier = {
43
+ "k": 10**3,
44
+ "m": 10**6,
45
+ "g": 10**9,
46
+ "t": 10**12,
47
+ }
48
+ binary_multiplier = {
49
+ "K": 2**10,
50
+ "M": 2**20,
51
+ "G": 2**30,
52
+ "T": 2**40,
53
+ }
54
+
55
+ number, suffix = match.groups()
56
+ if suffix in decimal_multiplier:
57
+ mult = decimal_multiplier[suffix]
58
+ return int(float(number) * mult)
59
+ elif suffix in binary_multiplier:
60
+ mult = binary_multiplier[suffix]
61
+ # Do not allow decimals with binary multipliers
62
+ try:
63
+ return int(number) * mult
64
+ except ValueError as e:
65
+ raise argparse.ArgumentTypeError(
66
+ "Decimals are not allowed "
67
+ f"with binary suffixes like {suffix}. Did you mean to use "
68
+ f"{number}{suffix.lower()} instead?"
69
+ ) from e
70
+
71
+ # Regular plain number.
72
+ return int(value)
73
+
74
+
75
+def human_readable_int_or_auto(value: str) -> int:
76
+ """Parse human-readable integers like '1k', '2M', etc.
77
+ Including decimal values with decimal multipliers.
78
+ Also accepts -1 or 'auto' as a special value for auto-detection.
79
+
80
+ Examples:
81
+ - '1k' -> 1,000
82
+ - '1K' -> 1,024
83
+ - '25.6k' -> 25,600
84
+ - '-1' or 'auto' -> -1 (special value for auto-detection)
85
+ """
86
+ value = value.strip()
87
+
88
+ if value == "-1" or value.lower() == "auto":
89
+ return -1
90
+
91
+ return human_readable_int(value)
92
+
93
+
94
class SortedHelpFormatter(ArgumentDefaultsHelpFormatter, RawDescriptionHelpFormatter):
95
"""SortedHelpFormatter that sorts arguments by their option strings."""
96
@@ -338,7 +404,12 @@ class FlexibleArgumentParser(ArgumentParser):
404
try:
405
value = json.loads(value_str)
406
except json.decoder.JSONDecodeError:
341
- value = value_str
407
+ # Support human-readable suffixes (e.g. 1k, 80g) for
408
+ # dotted config args like --config.field 80g
409
+ try:
410
+ value = human_readable_int(value_str) # type: ignore[assignment]
411
+ except (ValueError, ArgumentTypeError):
412
+ value = value_str
413
414
# Merge all values with the same key into a single dict
415
arg_dict = create_nested_dict(keys, value)