A high-throughput and memory-efficient inference and serving engine for LLMs
Mirrored from https://github.com/tduncan-lum/Nvidia-Optimization-Stack---VLLM.git
Synced Nov 10, 22:45 UTC
[Bugfix] Fix `max_model_len="auto"` handling (#31260)
Signed-off-by: DarkLight1337 <tlleungac@connect.ust.hk>
Cyrus Leung committed
Dec 24, 2025 at 19:15 UTC
7adeb4bfa8630773c84f9ad9e97830becd540896
3 files changed
+79
-82
vllm/config/model.py
+1
-1
@@ -164,7 +164,7 @@ class ModelConfig:
164
"""The specific revision to use for the tokenizer on the Hugging Face Hub.
165
It can be a branch name, a tag name, or a commit id. If unspecified, will
166
use the default version."""
167
- max_model_len: int = Field(default=None, gt=0)
167
+ max_model_len: int = Field(default=None, ge=-1)
168
"""Model context length (prompt and output). If unspecified, will be
169
automatically derived from the model config.
170
vllm/engine/arg_utils.py
+26
-15
@@ -297,16 +297,14 @@ def _compute_kwargs(cls: ConfigType) -> dict[str, dict[str, Any]]:
297
elif contains_type(type_hints, set):
298
kwargs[name].update(collection_to_kwargs(type_hints, set))
299
elif contains_type(type_hints, int):
300
- kwargs[name]["type"] = int
301
- # Special case for large integers
302
- human_readable_ints = {
303
- "max_model_len",
304
- "max_num_batched_tokens",
305
- "kv_cache_memory_bytes",
306
- }
307
- if name in human_readable_ints:
300
+ if name == "max_model_len":
301
+ kwargs[name]["type"] = human_readable_int_or_auto
302
+ kwargs[name]["help"] += f"\n\n{human_readable_int_or_auto.__doc__}"
303
+ elif name in ("max_num_batched_tokens", "kv_cache_memory_bytes"):
304
kwargs[name]["type"] = human_readable_int
305
kwargs[name]["help"] += f"\n\n{human_readable_int.__doc__}"
306
+ else:
307
+ kwargs[name]["type"] = int
308
elif contains_type(type_hints, float):
309
kwargs[name]["type"] = float
310
elif contains_type(type_hints, dict) and (
@@ -2042,23 +2040,17 @@ def _raise_unsupported_error(feature_name: str):
2040
raise NotImplementedError(msg)
2041
2042
2045
-def human_readable_int(value):
2043
+def human_readable_int(value: str) -> int:
2044
"""Parse human-readable integers like '1k', '2M', etc.
2045
Including decimal values with decimal multipliers.
2048
- Also accepts -1 or 'auto' as a special value for auto-detection.
2046
2047
Examples:
2048
- '1k' -> 1,000
2049
- '1K' -> 1,024
2050
- '25.6k' -> 25,600
2054
- - '-1' or 'auto' -> -1 (special value for auto-detection)
2051
"""
2052
value = value.strip()
2053
2058
- # Handle -1 or 'auto' as a special value for auto-detection
2059
- if value == "-1" or value.lower() == "auto":
2060
- return -1
2061
-
2054
match = re.fullmatch(r"(\d+(?:\.\d+)?)([kKmMgGtT])", value)
2055
if match:
2056
decimal_multiplier = {
@@ -2092,3 +2084,22 @@ def human_readable_int(value):
2084
2085
# Regular plain number.
2086
return int(value)
2087
+
2088
+
2089
+def human_readable_int_or_auto(value: str) -> int:
2090
+ """Parse human-readable integers like '1k', '2M', etc.
2091
+ Including decimal values with decimal multipliers.
2092
+ Also accepts -1 or 'auto' as a special value for auto-detection.
2093
+
2094
+ Examples:
2095
+ - '1k' -> 1,000
2096
+ - '1K' -> 1,024
2097
+ - '25.6k' -> 25,600
2098
+ - '-1' or 'auto' -> -1 (special value for auto-detection)
2099
+ """
2100
+ value = value.strip()
2101
+
2102
+ if value == "-1" or value.lower() == "auto":
2103
+ return -1
2104
+
2105
+ return human_readable_int(value)
vllm/v1/core/kv_cache_utils.py
+52
-66
@@ -606,6 +606,43 @@ def get_request_block_hasher(
606
return request_block_hasher
607
608
609
+def _check_enough_kv_cache_memory(
610
+ available_memory: int,
611
+ get_needed_memory: Callable[[], int],
612
+ max_model_len: int,
613
+ estimate_max_model_len: Callable[[int], int],
614
+):
615
+ if available_memory <= 0:
616
+ raise ValueError(
617
+ "No available memory for the cache blocks. "
618
+ "Try increasing `gpu_memory_utilization` when initializing the engine. "
619
+ "See https://docs.vllm.ai/en/latest/configuration/conserving_memory/ "
620
+ "for more details."
621
+ )
622
+
623
+ needed_memory = get_needed_memory()
624
+
625
+ if needed_memory > available_memory:
626
+ estimated_max_len = estimate_max_model_len(available_memory)
627
+ estimated_msg = ""
628
+ if estimated_max_len > 0:
629
+ estimated_msg = (
630
+ "Based on the available memory, "
631
+ f"the estimated maximum model length is {estimated_max_len}. "
632
+ )
633
+
634
+ raise ValueError(
635
+ f"To serve at least one request with the models's max seq len "
636
+ f"({max_model_len}), ({needed_memory / GiB_bytes:.2f} GiB KV "
637
+ f"cache is needed, which is larger than the available KV cache "
638
+ f"memory ({available_memory / GiB_bytes:.2f} GiB). {estimated_msg}"
639
+ f"Try increasing `gpu_memory_utilization` or decreasing `max_model_len` "
640
+ f"when initializing the engine. "
641
+ f"See https://docs.vllm.ai/en/latest/configuration/conserving_memory/ "
642
+ f"for more details."
643
+ )
644
+
645
+
646
def max_memory_usage_bytes(
647
vllm_config: VllmConfig, kv_cache_specs: Iterable[KVCacheSpec]
648
) -> int:
@@ -688,43 +725,12 @@ def check_enough_kv_cache_memory(
725
"""
726
727
# No need to check for available memory if the kv_cache_spec is empty
691
- if not kv_cache_spec:
692
- return
693
-
694
- if available_memory <= 0:
695
- raise ValueError(
696
- "No available memory for the cache blocks. "
697
- "Try increasing `gpu_memory_utilization` when "
698
- "initializing the engine. "
699
- "See https://docs.vllm.ai/en/latest/configuration/conserving_memory/ "
700
- "for more details."
701
- )
702
-
703
- max_model_len = vllm_config.model_config.max_model_len
704
- needed_memory = max_memory_usage_bytes(vllm_config, kv_cache_spec.values())
705
-
706
- if needed_memory > available_memory:
707
- # Estimate the maximum model length that can fit in the available memory
708
- estimated_max_len = estimate_max_model_len(
709
- vllm_config, kv_cache_spec, available_memory
710
- )
711
- estimated_msg = ""
712
- if estimated_max_len > 0:
713
- estimated_msg = (
714
- "Based on the available memory, "
715
- f"the estimated maximum model length is {estimated_max_len}."
716
- )
717
-
718
- raise ValueError(
719
- f"To serve at least one request with the models's max seq len "
720
- f"({max_model_len}), ({needed_memory / GiB_bytes:.2f} GiB KV "
721
- f"cache is needed, which is larger than the available KV cache "
722
- f"memory ({available_memory / GiB_bytes:.2f} GiB). "
723
- f"{estimated_msg} "
724
- f"Try increasing `gpu_memory_utilization` or decreasing `max_model_len` "
725
- f"when initializing the engine. "
726
- f"See https://docs.vllm.ai/en/latest/configuration/conserving_memory/ "
727
- f"for more details."
728
+ if kv_cache_spec:
729
+ _check_enough_kv_cache_memory(
730
+ available_memory,
731
+ lambda: max_memory_usage_bytes(vllm_config, kv_cache_spec.values()),
732
+ vllm_config.model_config.max_model_len,
733
+ lambda am: estimate_max_model_len(vllm_config, kv_cache_spec, am),
734
)
735
736
@@ -1505,36 +1511,16 @@ def get_kv_cache_configs(
1511
# Check if the available memory is enough (using min across all workers).
1512
# We use the global groups to correctly account for padding.
1513
if global_kv_cache_groups:
1508
- min_available_memory = min(available_memory)
1509
- if min_available_memory <= 0:
1510
- raise ValueError(
1511
- "No available memory for the cache blocks. "
1512
- "Try increasing `gpu_memory_utilization` when "
1513
- "initializing the engine."
1514
- )
1515
- max_model_len = vllm_config.model_config.max_model_len
1516
- needed_memory = _max_memory_usage_bytes_from_groups(
1517
- vllm_config, global_kv_cache_groups
1514
+ _check_enough_kv_cache_memory(
1515
+ min(available_memory),
1516
+ lambda: _max_memory_usage_bytes_from_groups(
1517
+ vllm_config, global_kv_cache_groups
1518
+ ),
1519
+ vllm_config.model_config.max_model_len,
1520
+ lambda am: _estimate_max_model_len_from_groups(
1521
+ vllm_config, global_kv_cache_groups, am
1522
+ ),
1523
)
1519
- if needed_memory > min_available_memory:
1520
- estimated_max_len = _estimate_max_model_len_from_groups(
1521
- vllm_config, global_kv_cache_groups, min_available_memory
1522
- )
1523
- estimated_msg = ""
1524
- if estimated_max_len > 0:
1525
- estimated_msg = (
1526
- f"Based on the available memory, the estimated maximum "
1527
- f"model length is {estimated_max_len}. "
1528
- )
1529
- raise ValueError(
1530
- f"To serve at least one request with the models's max seq len "
1531
- f"({max_model_len}), ({needed_memory / GiB_bytes:.2f} GiB KV "
1532
- f"cache is needed, which is larger than the available KV cache "
1533
- f"memory ({min_available_memory / GiB_bytes:.2f} GiB). "
1534
- f"{estimated_msg}"
1535
- f"Try increasing `gpu_memory_utilization` or decreasing "
1536
- f"`max_model_len` when initializing the engine."
1537
- )
1524
1525
kv_cache_configs: list[KVCacheConfig] = []
1526
for kv_cache_spec_one_worker, available_memory_one_worker in zip(