A high-throughput and memory-efficient inference and serving engine for LLMs
Mirrored from https://github.com/tduncan-lum/Nvidia-Optimization-Stack---VLLM.git
Synced Nov 10, 21:30 UTC
[Core] Enable async scheduling by default (#27614)
Signed-off-by: Nick Hill <nickhill123@gmail.com>
Nick Hill committed
Dec 29, 2025 at 12:20 UTC
c2ff33cc8c461e3aca045f6125662ca4ded888a8
2 files changed
+31
-14
vllm/config/scheduler.py
+6
-5
@@ -130,11 +130,12 @@ class SchedulerConfig:
130
and starting configuration.
131
"""
132
133
- async_scheduling: bool = False
134
- """If set to True, perform async scheduling. This helps to avoid gaps in
135
- GPU utilization, leading to better latency and throughput.
136
- Async scheduling is currently not supported with some features such as
137
- speculative decoding and pipeline parallelism.
133
+ async_scheduling: bool = Field(default=None)
134
+ """If set to False, disable async scheduling. Async scheduling helps to
135
+ avoid gaps in GPU utilization, leading to better latency and throughput.
136
+ It is currently not supported with some features such as
137
+ speculative decoding and pipeline parallelism, and will be automatically
138
+ disabled in those cases.
139
"""
140
141
stream_interval: int = Field(default=1, ge=1)
vllm/config/vllm.py
+25
-9
@@ -552,7 +552,7 @@ class VllmConfig:
552
if self.speculative_config.method not in get_args(EagleModelTypes):
553
raise ValueError(
554
"Currently, async scheduling is only supported "
555
- "with EAGLE/MTP kind of speculative decoding"
555
+ "with EAGLE/MTP kind of speculative decoding."
556
)
557
if self.speculative_config.disable_padded_drafter_batch:
558
raise ValueError(
@@ -570,16 +570,27 @@ class VllmConfig:
570
)
571
elif self.scheduler_config.async_scheduling is None:
572
# Enable async scheduling unless there is an incompatible option.
573
- # NOTE: we won't reach here until async scheduling is enabled by default.
574
- if (
575
- self.parallel_config.pipeline_parallel_size > 1
576
- or self.speculative_config is not None
577
- ):
573
+ if self.parallel_config.pipeline_parallel_size > 1:
574
logger.warning(
579
- "Async scheduling is not yet supported with speculative decoding "
580
- " or pipeline_parallel_size > 1 and will be disabled."
575
+ "Async scheduling is not yet supported with "
576
+ "pipeline_parallel_size > 1 and will be disabled."
577
)
578
self.scheduler_config.async_scheduling = False
579
+ elif self.speculative_config is not None:
580
+ if self.speculative_config.method not in get_args(EagleModelTypes):
581
+ logger.warning(
582
+ "Async scheduling not supported with %s-based "
583
+ "speculative decoding and will be disabled.",
584
+ self.speculative_config.method,
585
+ )
586
+ else:
587
+ logger.warning(
588
+ "Async scheduling will be disabled because some features do "
589
+ "not currently work in conjunction with speculative decoding. "
590
+ "To use async scheduling with spec decoding anyway, "
591
+ "enable it explicitly via async_scheduling=True."
592
+ )
593
+ self.scheduler_config.async_scheduling = False
594
elif not executor_supports_async_sched:
595
logger.warning(
596
"Async scheduling will be disabled because it is not supported "
@@ -595,11 +606,16 @@ class VllmConfig:
606
self.scheduler_config.async_scheduling
607
and not self.parallel_config.disable_nccl_for_dp_synchronization
608
):
598
- logger.info(
609
+ logger.info_once(
610
"Disabling NCCL for DP synchronization when using async scheduling."
611
)
612
self.parallel_config.disable_nccl_for_dp_synchronization = True
613
614
+ logger.info_once(
615
+ "Asynchronous scheduling is %s.",
616
+ "enabled" if self.scheduler_config.async_scheduling else "disabled",
617
+ )
618
+
619
from vllm.platforms import current_platform
620
621
if (