A high-throughput and memory-efficient inference and serving engine for LLMs
Mirrored from https://github.com/tduncan-lum/Nvidia-Optimization-Stack---VLLM.git
Synced Nov 15, 07:15 UTC
[BugFix][Minor] Fix full cuda graph bug when max_num_seqs < 512 (#19171)
Signed-off-by: Woosuk Kwon <woosuk.kwon@berkeley.edu>
Woosuk Kwon committed
Jun 4, 2025 at 22:41 UTC
af7fc84fd2e9b0fb70bc6349730b3ee73f0e1f8d
1 file changed
+2
-2
vllm/v1/worker/gpu_model_runner.py
+2
-2
@@ -1737,7 +1737,7 @@ class GPUModelRunner(LoRAModelRunnerMixin):
1737
# has num_tokens in total.
1738
assert num_tokens <= self.scheduler_config.max_num_batched_tokens
1739
max_num_reqs = self.scheduler_config.max_num_seqs
1740
- num_reqs = max_num_reqs if num_tokens >= max_num_reqs else num_tokens
1740
+ num_reqs = min(num_tokens, max_num_reqs)
1741
min_tokens_per_req = num_tokens // num_reqs
1742
num_scheduled_tokens_list = [min_tokens_per_req] * num_reqs
1743
num_scheduled_tokens_list[-1] += num_tokens % num_reqs
@@ -1765,7 +1765,7 @@ class GPUModelRunner(LoRAModelRunnerMixin):
1765
self.kv_cache_config.kv_cache_groups):
1766
attn_metadata_i = (
1767
self.attn_metadata_builders[kv_cache_group_id].build(
1768
- num_reqs=num_tokens,
1768
+ num_reqs=num_reqs,
1769
num_actual_tokens=num_tokens,
1770
max_query_len=num_tokens,
1771
common_prefix_len=0,