[Misc][Docs] Add human-readable integer support for more cli-args (#47608)
Signed-off-by: NickLucche <nicolo.lucchesi@mistral.ai>
Nicolò Lucchesi committed
Jul 8, 2026 at 11:43 UTC
bd3bb4eb2635dc93749ef22c2558b98fa0519823
3 files changed
+66
-5
docs/cli/README.md
+15
@@ -50,6 +50,21 @@ vllm serve --help=max-num-seqs
50
vllm serve --help=max
51
```
52
53
+!!! tip "Human-readable integer arguments"
54
+ Many integer arguments accept human-readable suffixes for convenience. For example:
55
+
56
+ - `1k` = 1,000 (decimal kilo)
57
+ - `1K` = 1,024 (binary kibibyte)
58
+ - `1m` = 1,000,000 (decimal mega)
59
+ - `1M` = 1,048,576 (binary mebibyte)
60
+ - `1g` / `1G` = 1 billion / 1 gibibyte
61
+ - `1t` / `1T` = 1 trillion / 1 tebibyte
62
+
63
+ Decimal suffixes (`k`, `m`, `g`, `t`) also accept floating point: `25.6k` = 25,600.
64
+ Binary suffixes (`K`, `M`, `G`, `T`) require integers: `32K` = 32,768.
65
+
66
+ Supported arguments include: `--max-model-len`, `--max-num-batched-tokens`, `--max-num-scheduled-tokens`, `--kv-cache-memory-bytes`, `--safetensors-prefetch-block-size`.
67
+
68
See [vllm serve](./serve.md) for the full reference of all available arguments.
69
70
## launch
tests/engine/test_arg_utils.py
+34
@@ -565,6 +565,40 @@ def test_human_readable_model_len():
565
parser.parse_args(["--max-model-len", invalid])
566
567
568
+def test_human_readable_other_args():
569
+ # Test human-readable parsing for other integer args
570
+ # that were added to use human_readable_int parser
571
+ parser = EngineArgs.add_cli_args(FlexibleArgumentParser(exit_on_error=False))
572
+
573
+ # Test max_num_scheduled_tokens
574
+ args = parser.parse_args(["--max-num-scheduled-tokens", "1024"])
575
+ assert args.max_num_scheduled_tokens == 1024
576
+ args = parser.parse_args(["--max-num-scheduled-tokens", "2k"])
577
+ assert args.max_num_scheduled_tokens == 2_000
578
+ args = parser.parse_args(["--max-num-scheduled-tokens", "4K"])
579
+ assert args.max_num_scheduled_tokens == 2**10 * 4
580
+ args = parser.parse_args(["--max-num-scheduled-tokens", "10.5k"])
581
+ assert args.max_num_scheduled_tokens == 10500
582
+
583
+ # Test kv_cache_memory_bytes (existing human-readable arg)
584
+ args = parser.parse_args(["--kv-cache-memory-bytes", "100000"])
585
+ assert args.kv_cache_memory_bytes == 100000
586
+ args = parser.parse_args(["--kv-cache-memory-bytes", "100k"])
587
+ assert args.kv_cache_memory_bytes == 100_000
588
+ args = parser.parse_args(["--kv-cache-memory-bytes", "1M"])
589
+ assert args.kv_cache_memory_bytes == 2**20
590
+ args = parser.parse_args(["--kv-cache-memory-bytes", "1m"])
591
+ assert args.kv_cache_memory_bytes == 1_000_000
592
+
593
+ # Test max_num_batched_tokens (existing human-readable arg)
594
+ args = parser.parse_args(["--max-num-batched-tokens", "1024"])
595
+ assert args.max_num_batched_tokens == 1024
596
+ args = parser.parse_args(["--max-num-batched-tokens", "2k"])
597
+ assert args.max_num_batched_tokens == 2_000
598
+ args = parser.parse_args(["--max-num-batched-tokens", "4K"])
599
+ assert args.max_num_batched_tokens == 2**10 * 4
600
+
601
+
602
def test_numa_bind_args():
603
parser = EngineArgs.add_cli_args(FlexibleArgumentParser())
604
args = parser.parse_args(
vllm/engine/arg_utils.py
+17
-5
@@ -354,14 +354,17 @@ def _compute_kwargs(cls: ConfigType) -> dict[str, dict[str, Any]]:
354
elif contains_type(type_hints, set):
355
kwargs[name].update(collection_to_kwargs(type_hints, set))
356
elif contains_type(type_hints, int):
357
- if name == "max_model_len":
358
- kwargs[name]["type"] = human_readable_int_or_auto
359
- kwargs[name]["help"] += f"\n\n{human_readable_int_or_auto.__doc__}"
360
- elif name in (
357
+ # Arguments that accept human-readable integer strings (e.g., 1K, 2M, 1G)
358
+ human_readable_int_args = {
359
"max_num_batched_tokens",
360
+ "max_num_scheduled_tokens",
361
"kv_cache_memory_bytes",
362
"safetensors_prefetch_block_size",
364
- ):
363
+ }
364
+ if name == "max_model_len":
365
+ kwargs[name]["type"] = human_readable_int_or_auto
366
+ kwargs[name]["help"] += f"\n\n{human_readable_int_or_auto.__doc__}"
367
+ elif name in human_readable_int_args:
368
kwargs[name]["type"] = human_readable_int
369
kwargs[name]["help"] += f"\n\n{human_readable_int.__doc__}"
370
else:
@@ -523,6 +526,7 @@ class EngineArgs:
526
gpu_memory_utilization: float = CacheConfig.gpu_memory_utilization
527
kv_cache_memory_bytes: int | None = CacheConfig.kv_cache_memory_bytes
528
max_num_batched_tokens: int | None = None
529
+ max_num_scheduled_tokens: int | None = None
530
max_num_partial_prefills: int = SchedulerConfig.max_num_partial_prefills
531
max_long_partial_prefills: int = SchedulerConfig.max_long_partial_prefills
532
long_prefill_token_threshold: int = SchedulerConfig.long_prefill_token_threshold
@@ -1412,6 +1416,13 @@ class EngineArgs:
1416
"default": None,
1417
},
1418
)
1419
+ scheduler_group.add_argument(
1420
+ "--max-num-scheduled-tokens",
1421
+ **{
1422
+ **scheduler_kwargs["max_num_scheduled_tokens"],
1423
+ "default": None,
1424
+ },
1425
+ )
1426
scheduler_group.add_argument(
1427
"--max-num-seqs",
1428
**{
@@ -2151,6 +2162,7 @@ class EngineArgs:
2162
scheduler_config = SchedulerConfig(
2163
runner_type=model_config.runner_type,
2164
max_num_batched_tokens=self.max_num_batched_tokens,
2165
+ max_num_scheduled_tokens=self.max_num_scheduled_tokens,
2166
max_num_seqs=self.max_num_seqs,
2167
max_model_len=model_config.max_model_len,
2168
enable_chunked_prefill=self.enable_chunked_prefill,