A high-throughput and memory-efficient inference and serving engine for LLMs
Mirrored from https://github.com/tduncan-lum/Nvidia-Optimization-Stack---VLLM.git
Synced Nov 14, 05:30 UTC
[MISC] Fix misleading batch_size_capture_list when cuda_graph_sizes < 4 (#25829)
Signed-off-by: billishyahao <bill.he@amd.com> Co-authored-by: Luka Govedic <ProExpertProg@users.noreply.github.com>
billishyahao committed
Oct 1, 2025 at 20:39 UTC
2518230d3edaf7b4b8d9f5ed6e78015969c7fc23
1 file changed
+6
-3
vllm/config/vllm.py
+6
-3
@@ -580,9 +580,12 @@ class VllmConfig:
580
not self.model_config.enforce_eager:
581
cuda_graph_sizes = self.scheduler_config.cuda_graph_sizes
582
if len(cuda_graph_sizes) == 1:
583
- batch_size_capture_list = [1, 2, 4] + [
584
- i for i in range(8, cuda_graph_sizes[0] + 1, 8)
585
- ]
583
+ max_graph_size = cuda_graph_sizes[0]
584
+ assert max_graph_size >= 1, "Maximum cudagraph size should be" \
585
+ " greater than or equal to 1."
586
+ batch_size_capture_list = [
587
+ i for i in [1, 2, 4] if i <= max_graph_size
588
+ ] + list(range(8, max_graph_size + 1, 8))
589
elif len(cuda_graph_sizes) > 1:
590
batch_size_capture_list = sorted(cuda_graph_sizes)
591
else: