A high-throughput and memory-efficient inference and serving engine for LLMs
Mirrored from https://github.com/tduncan-lum/Nvidia-Optimization-Stack---VLLM.git
Synced Nov 11, 16:15 UTC
fix: enhance human_readable_int function (#30337)
Signed-off-by: Andy Xie <andy.xning@gmail.com>
Ning Xie committed
Dec 11, 2025 at 15:30 UTC
d02d1043dea56e4d2b1149a311079d82ff251d9d
2 files changed
+21
-4
tests/engine/test_arg_utils.py
+18
-4
@@ -350,21 +350,35 @@ def test_human_readable_model_len():
350
assert args.max_model_len == 1_000_000
351
args = parser.parse_args(["--max-model-len", "10k"])
352
assert args.max_model_len == 10_000
353
+ args = parser.parse_args(["--max-model-len", "2g"])
354
+ assert args.max_model_len == 2_000_000_000
355
+ args = parser.parse_args(["--max-model-len", "2t"])
356
+ assert args.max_model_len == 2_000_000_000_000
357
358
# Capital
359
args = parser.parse_args(["--max-model-len", "3K"])
356
- assert args.max_model_len == 1024 * 3
360
+ assert args.max_model_len == 2**10 * 3
361
args = parser.parse_args(["--max-model-len", "10M"])
362
assert args.max_model_len == 2**20 * 10
363
+ args = parser.parse_args(["--max-model-len", "4G"])
364
+ assert args.max_model_len == 2**30 * 4
365
+ args = parser.parse_args(["--max-model-len", "4T"])
366
+ assert args.max_model_len == 2**40 * 4
367
368
# Decimal values
369
args = parser.parse_args(["--max-model-len", "10.2k"])
370
assert args.max_model_len == 10200
371
# ..truncated to the nearest int
364
- args = parser.parse_args(["--max-model-len", "10.212345k"])
372
+ args = parser.parse_args(["--max-model-len", "10.2123451234567k"])
373
assert args.max_model_len == 10212
374
+ args = parser.parse_args(["--max-model-len", "10.2123451234567m"])
375
+ assert args.max_model_len == 10212345
376
+ args = parser.parse_args(["--max-model-len", "10.2123451234567g"])
377
+ assert args.max_model_len == 10212345123
378
+ args = parser.parse_args(["--max-model-len", "10.2123451234567t"])
379
+ assert args.max_model_len == 10212345123456
380
381
# Invalid (do not allow decimals with binary multipliers)
368
- for invalid in ["1a", "pwd", "10.24", "1.23M"]:
382
+ for invalid in ["1a", "pwd", "10.24", "1.23M", "1.22T"]:
383
with pytest.raises(ArgumentError):
370
- args = parser.parse_args(["--max-model-len", invalid])
384
+ parser.parse_args(["--max-model-len", invalid])
vllm/engine/arg_utils.py
+3
@@ -1783,6 +1783,7 @@ class EngineArgs:
1783
except Exception:
1784
# This is only used to set default_max_num_batched_tokens
1785
device_memory = 0
1786
+ device_name = ""
1787
1788
# NOTE(Kuntai): Setting large `max_num_batched_tokens` for A100 reduces
1789
# throughput, see PR #17885 for more details.
@@ -2042,11 +2043,13 @@ def human_readable_int(value):
2043
"k": 10**3,
2044
"m": 10**6,
2045
"g": 10**9,
2046
+ "t": 10**12,
2047
}
2048
binary_multiplier = {
2049
"K": 2**10,
2050
"M": 2**20,
2051
"G": 2**30,
2052
+ "T": 2**40,
2053
}
2054
2055
number, suffix = match.groups()