A high-throughput and memory-efficient inference and serving engine for LLMs
Mirrored from https://github.com/tduncan-lum/Nvidia-Optimization-Stack---VLLM.git
Synced Nov 14, 05:30 UTC
[Bug]: Limit num_reqs in dummy_run when max_num_seqs is small (#26144)
Signed-off-by: Benjamin Chislett <bchislett@nvidia.com>
Benjamin Chislett committed
Oct 3, 2025 at 00:00 UTC
6a7796e87139e9848359444d10b6489d9e15fee7
1 file changed
+2
-2
vllm/v1/worker/gpu_model_runner.py
+2
-2
@@ -3060,7 +3060,7 @@ class GPUModelRunner(LoRAModelRunnerMixin, KVConnectorModelRunnerMixin):
3060
assert not uniform_decode
3061
# Create mixed batch:
3062
# first half decode tokens, second half one prefill
3063
- num_decode_tokens = num_tokens // 2
3063
+ num_decode_tokens = min(max_num_reqs - 1, num_tokens // 2)
3064
num_prefill_tokens = num_tokens - num_decode_tokens
3065
num_reqs = num_decode_tokens + 1
3066
@@ -3072,7 +3072,7 @@ class GPUModelRunner(LoRAModelRunnerMixin, KVConnectorModelRunnerMixin):
3072
max_query_len = num_prefill_tokens
3073
elif uniform_decode:
3074
assert not create_mixed_batch
3075
- num_reqs = cdiv(num_tokens, max_query_len)
3075
+ num_reqs = min(max_num_reqs, cdiv(num_tokens, max_query_len))
3076
num_scheduled_tokens_list = [max_query_len] * num_reqs
3077
if num_tokens % max_query_len != 0:
3078
num_scheduled_tokens_list[-1] = num_tokens % max_query_len