A high-throughput and memory-efficient inference and serving engine for LLMs
Mirrored from https://github.com/tduncan-lum/Nvidia-Optimization-Stack---VLLM.git
Synced Nov 15, 07:15 UTC
Let max_num_batched_tokens use human_readable_int for large numbers (#18968)
Signed-off-by: mgoin <mgoin64@gmail.com>
Michael Goin committed
May 31, 2025 at 23:41 UTC
2ad6194a0250932da7354e4d9dbebaf610f90202
1 file changed
+1
-1
vllm/engine/arg_utils.py
+1
-1
@@ -224,7 +224,7 @@ def get_kwargs(cls: ConfigType) -> dict[str, Any]:
224
elif contains_type(type_hints, int):
225
kwargs[name]["type"] = int
226
# Special case for large integers
227
- if name in {"max_model_len"}:
227
+ if name in {"max_model_len", "max_num_batched_tokens"}:
228
kwargs[name]["type"] = human_readable_int
229
elif contains_type(type_hints, float):
230
kwargs[name]["type"] = float