A high-throughput and memory-efficient inference and serving engine for LLMs
Mirrored from https://github.com/tduncan-lum/Nvidia-Optimization-Stack---VLLM.git
Synced Nov 10, 02:45 UTC
[Feature] Support async scheduling + PP (#32359)
Signed-off-by: yewentao256 <zhyanwentao@126.com>
Wentao Ye committed
Jan 15, 2026 at 12:06 UTC
b34474bf2c207f2a7f08f6c830c71c3f85c82630
5 files changed
+16
-19
tests/v1/core/utils.py
+3
@@ -9,6 +9,7 @@ from vllm.config import (
9
ECTransferConfig,
10
KVTransferConfig,
11
ModelConfig,
12
+ ParallelConfig,
13
SchedulerConfig,
14
SpeculativeConfig,
15
VllmConfig,
@@ -53,6 +54,7 @@ def create_scheduler(
54
num_speculative_tokens: int | None = None,
55
skip_tokenizer_init: bool = False,
56
async_scheduling: bool = False,
57
+ pipeline_parallel_size: int = 1,
58
use_ec_connector: bool = False,
59
ec_role: str | None = None,
60
) -> Scheduler | AsyncScheduler:
@@ -133,6 +135,7 @@ def create_scheduler(
135
scheduler_config=scheduler_config,
136
model_config=model_config,
137
cache_config=cache_config,
138
+ parallel_config=ParallelConfig(pipeline_parallel_size=pipeline_parallel_size),
139
kv_transfer_config=kv_transfer_config,
140
speculative_config=speculative_config,
141
ec_transfer_config=ec_transfer_config,
vllm/config/vllm.py
+1
-13
@@ -563,11 +563,6 @@ class VllmConfig:
563
564
if self.scheduler_config.async_scheduling:
565
# Async scheduling explicitly enabled, hard fail any incompatibilities.
566
- if self.parallel_config.pipeline_parallel_size > 1:
567
- raise ValueError(
568
- "Async scheduling is not yet compatible with "
569
- "pipeline_parallel_size > 1."
570
- )
566
# Currently, async scheduling only support eagle speculative
567
# decoding.
568
if self.speculative_config is not None:
@@ -589,14 +584,7 @@ class VllmConfig:
584
)
585
elif self.scheduler_config.async_scheduling is None:
586
# Enable async scheduling unless there is an incompatible option.
592
- if self.parallel_config.pipeline_parallel_size > 1:
593
- logger.warning_once(
594
- "Async scheduling is not yet supported with "
595
- "pipeline_parallel_size > 1 and will be disabled.",
596
- scope="local",
597
- )
598
- self.scheduler_config.async_scheduling = False
599
- elif (
587
+ if (
588
self.speculative_config is not None
589
and self.speculative_config.method not in get_args(EagleModelTypes)
590
):
vllm/v1/core/sched/scheduler.py
+7
@@ -283,6 +283,13 @@ class Scheduler(SchedulerInterface):
283
while req_index < len(self.running) and token_budget > 0:
284
request = self.running[req_index]
285
286
+ # do not schedule another step for the same request while it still has
287
+ # output placeholders for PP.
288
+ # TODO: support PP + async scheduling without this limit
289
+ if self.use_pp and request.num_output_placeholders > 0:
290
+ req_index += 1
291
+ continue
292
+
293
if (
294
request.num_output_placeholders > 0
295
# This is (num_computed_tokens + 1) - (num_output_placeholders - 1).
vllm/v1/executor/multiproc_executor.py
+3
-3
@@ -411,9 +411,9 @@ class MultiprocExecutor(Executor):
411
412
@cached_property
413
def max_concurrent_batches(self) -> int:
414
- if self.scheduler_config.async_scheduling:
415
- return 2
416
- return self.parallel_config.pipeline_parallel_size
414
+ # PP requires PP-size concurrent batches to fill the pipeline.
415
+ pp_size = self.parallel_config.pipeline_parallel_size
416
+ return 2 if pp_size <= 1 and self.scheduler_config.async_scheduling else pp_size
417
418
def _get_output_rank(self) -> int:
419
# Only returns ModelRunnerOutput from TP rank=0 and PP rank=-1
vllm/v1/executor/ray_executor.py
+2
-3
@@ -111,9 +111,8 @@ class RayDistributedExecutor(Executor):
111
"""Ray distributed executor supports pipeline parallelism,
112
meaning that it allows PP size batches to be executed concurrently.
113
"""
114
- if self.scheduler_config.async_scheduling:
115
- return 2
116
- return self.parallel_config.pipeline_parallel_size
114
+ pp_size = self.parallel_config.pipeline_parallel_size
115
+ return 2 if pp_size <= 1 and self.scheduler_config.async_scheduling else pp_size
116
117
def shutdown(self) -> None:
118
if logger: