[Scheduler] Add --max-num-active-seqs to cap RUNNING admission (#56758)
Signed-off-by: Li <chuali@amd.com> Co-authored-by: Cursor <cursoragent@cursor.com>
Chuan (Richard) Li committed
Sep 16, 2026 at 19:22 UTC
f730a93d2bf2cad3002578238d5a1035ee1f4177
5 files changed
+51
-1
tests/v1/core/test_scheduler.py
+16
@@ -193,6 +193,22 @@ def test_get_num_unfinished_requests():
193
assert scheduler.get_num_unfinished_requests() == len(requests) - i - 1
194
195
196
+def test_max_num_active_seqs_caps_admission():
197
+ """Admission is capped by max_num_active_seqs, not max_num_seqs."""
198
+ scheduler = create_scheduler(max_num_seqs=16, max_num_active_seqs=3)
199
+ requests = create_requests(num_requests=10)
200
+ for request in requests:
201
+ scheduler.add_request(request)
202
+
203
+ output = scheduler.schedule()
204
+
205
+ assert len(output.scheduled_new_reqs) == 3
206
+ assert len(scheduler.running) == 3
207
+ assert len(scheduler.waiting) == 7
208
+ # Execution capacity is untouched.
209
+ assert scheduler.max_num_running_reqs == 16
210
+
211
+
212
def _bind_hisparse_connector(scheduler):
213
connector = object.__new__(HiSparseConnector)
214
connector.connector_scheduler = HiSparseConnectorScheduler(async_speculative=False)
tests/v1/core/utils.py
+2
@@ -52,6 +52,7 @@ def mock_kv(matched_tokens: int, is_async: bool, num_defers_before_matching: int
52
def create_scheduler(
53
model: str = "facebook/opt-125m",
54
max_num_seqs: int = 16,
55
+ max_num_active_seqs: int | None = None,
56
max_num_batched_tokens: int = 8192,
57
enable_chunked_prefill: bool = True,
58
enable_prefix_caching: bool = False,
@@ -107,6 +108,7 @@ def create_scheduler(
108
max_model_len = max_num_batched_tokens
109
scheduler_config = SchedulerConfig(
110
max_num_seqs=max_num_seqs,
111
+ max_num_active_seqs=max_num_active_seqs,
112
max_num_batched_tokens=max_num_batched_tokens,
113
max_model_len=max_model_len,
114
long_prefill_token_threshold=long_prefill_token_threshold,
vllm/config/scheduler.py
+19
@@ -67,6 +67,16 @@ class SchedulerConfig:
67
In real usage, this should be set in `EngineArgs.create_engine_config`.
68
"""
69
70
+ max_num_active_seqs: int | None = Field(default=None, ge=1)
71
+ """Maximum number of requests the scheduler admits into RUNNING.
72
+
73
+ ``max_num_seqs`` sizes the model runner (per-request buffers and CUDA
74
+ graph capture) and is also the default admission limit. Setting this
75
+ lowers only the number of requests that may occupy RUNNING, so decode
76
+ batches stay smaller without shrinking runner or graph capacity. Must
77
+ be ``<= max_num_seqs``. ``None`` (default) keeps current behavior.
78
+ """
79
+
80
long_prefill_token_threshold: int = Field(default=0, ge=0)
81
"""For chunked prefill, a request is considered long if the prompt is
82
longer than this number of tokens. 0 disables the cap (default)."""
@@ -310,6 +320,15 @@ class SchedulerConfig:
320
f"({self.max_num_seqs})."
321
)
322
323
+ if (
324
+ self.max_num_active_seqs is not None
325
+ and self.max_num_active_seqs > self.max_num_seqs
326
+ ):
327
+ raise ValueError(
328
+ f"max_num_active_seqs ({self.max_num_active_seqs}) cannot be "
329
+ f"greater than max_num_seqs ({self.max_num_seqs})."
330
+ )
331
+
332
if self.max_num_batched_tokens > self.max_num_seqs * max_model_len:
333
logger.warning(
334
"max_num_batched_tokens (%d) exceeds max_num_seqs "
vllm/engine/arg_utils.py
+6
@@ -556,6 +556,7 @@ class EngineArgs:
556
max_num_scheduled_tokens: int | None = None
557
long_prefill_token_threshold: int = SchedulerConfig.long_prefill_token_threshold
558
max_num_seqs: int | None = None
559
+ max_num_active_seqs: int | None = SchedulerConfig.max_num_active_seqs
560
max_num_queued_reqs: int | None = None
561
max_num_queued_tokens: int | None = None
562
max_logprobs: int = ModelConfig.max_logprobs
@@ -1589,6 +1590,10 @@ class EngineArgs:
1590
"default": None,
1591
},
1592
)
1593
+ scheduler_group.add_argument(
1594
+ "--max-num-active-seqs",
1595
+ **scheduler_kwargs["max_num_active_seqs"],
1596
+ )
1597
scheduler_group.add_argument(
1598
"--max-num-queued-reqs", **scheduler_kwargs["max_num_queued_reqs"]
1599
)
@@ -2421,6 +2426,7 @@ class EngineArgs:
2426
max_num_batched_tokens=self.max_num_batched_tokens,
2427
max_num_scheduled_tokens=self.max_num_scheduled_tokens,
2428
max_num_seqs=self.max_num_seqs,
2429
+ max_num_active_seqs=self.max_num_active_seqs,
2430
max_num_queued_reqs=self.max_num_queued_reqs,
2431
max_num_queued_tokens=self.max_num_queued_tokens,
2432
max_model_len=model_config.max_model_len,
vllm/v1/core/sched/scheduler.py
+8
-1
@@ -123,6 +123,13 @@ class Scheduler(SchedulerInterface):
123
124
# Scheduling constraints.
125
self.max_num_running_reqs = self.scheduler_config.max_num_seqs
126
+ # Admission into RUNNING. Separate from max_num_running_reqs, which
127
+ # is the model-runner slot count.
128
+ self.max_num_active_reqs = (
129
+ self.scheduler_config.max_num_active_seqs
130
+ if self.scheduler_config.max_num_active_seqs is not None
131
+ else self.max_num_running_reqs
132
+ )
133
self.max_num_scheduled_tokens = (
134
self.scheduler_config.max_num_scheduled_tokens
135
if self.scheduler_config.max_num_scheduled_tokens is not None
@@ -850,7 +857,7 @@ class Scheduler(SchedulerInterface):
857
# Paused streaming sessions (WAITING_FOR_STREAMING_REQ) are not
858
# in `running` but still hold a model-runner request slot.
859
num_running = len(self.running) + self.num_waiting_for_streaming_input
853
- if num_running >= self.max_num_running_reqs:
860
+ if num_running >= self.max_num_active_reqs:
861
break
862
863
request_queue = self._select_waiting_queue_for_scheduling()