A high-throughput and memory-efficient inference and serving engine for LLMs
Mirrored from https://github.com/tduncan-lum/Nvidia-Optimization-Stack---VLLM.git
Synced Nov 2, 04:45 UTC
[Core] Move `max_concurrent_batches` to `VllmConfig` (#44274)
Signed-off-by: Nick Hill <nickhill123@gmail.com>
Nick Hill committed
Jun 2, 2026 at 08:57 UTC
cab5c9a2a9601ce27bd765db7158f6fce6c73fdf
11 files changed
+25
-37
tests/distributed/test_multiproc_executor.py
+1
-1
@@ -284,7 +284,7 @@ def test_multiproc_executor_pipeline_parallel():
284
assert output_rank == 2, "Output rank should be 2 (first rank of last PP stage)"
285
286
# Verify max_concurrent_batches for pipeline parallel
287
- assert executor.max_concurrent_batches == 2, (
287
+ assert vllm_config.max_concurrent_batches == 2, (
288
"Max concurrent batches should equal PP size"
289
)
290
tests/distributed/test_ray_v2_executor.py
+1
-1
@@ -83,7 +83,7 @@ def assert_executor(executor, tp_size, pp_size):
83
assert executor._get_output_rank() == expected_output_rank
84
85
if pp_size > 1:
86
- assert executor.max_concurrent_batches == pp_size
86
+ assert executor.vllm_config.max_concurrent_batches == pp_size
87
88
executor.check_health()
89
assert not executor.is_failed
tests/model_executor/model_loader/tensorizer_loader/conftest.py
-4
@@ -87,10 +87,6 @@ class DummyExecutor(UniProcExecutor):
87
self.collective_rpc("init_worker", args=([kwargs],))
88
self.collective_rpc("init_device")
89
90
- @property
91
- def max_concurrent_batches(self) -> int:
92
- return 2
93
-
90
def shutdown(self):
91
if hasattr(self, "thread_pool"):
92
self.thread_pool.shutdown(wait=False)
tests/v1/engine/test_engine_core.py
+12
-5
@@ -5,6 +5,7 @@ import copy
5
import time
6
import uuid
7
from concurrent.futures import Future, ThreadPoolExecutor
8
+from unittest.mock import PropertyMock, patch
9
10
import pytest
11
from transformers import AutoTokenizer
@@ -293,10 +294,6 @@ def test_engine_core_concurrent_batches():
294
# Use the thread pool instead of creating a new thread
295
return self.thread_pool.submit(_execute)
296
296
- @property
297
- def max_concurrent_batches(self) -> int:
298
- return 2
299
-
297
def shutdown(self):
298
if hasattr(self, "thread_pool"):
299
self.thread_pool.shutdown(wait=False)
@@ -314,7 +311,17 @@ def test_engine_core_concurrent_batches():
311
async_scheduling=False,
312
)
313
vllm_config = engine_args.create_engine_config()
317
- with set_default_torch_num_threads(1):
314
+ # Force two concurrent batches to exercise the batch queue independently
315
+ # of async scheduling (which is disabled above).
316
+ with (
317
+ set_default_torch_num_threads(1),
318
+ patch.object(
319
+ VllmConfig,
320
+ "max_concurrent_batches",
321
+ new_callable=PropertyMock,
322
+ return_value=2,
323
+ ),
324
+ ):
325
engine_core = EngineCore(
326
vllm_config=vllm_config, log_stats=False, executor_class=DummyExecutor
327
)
tests/v1/engine/test_engine_core_client.py
-1
@@ -1212,7 +1212,6 @@ def test_engine_core_proc_instantiation_cuda_empty(monkeypatch: pytest.MonkeyPat
1212
mock_executor.get_kv_cache_specs.return_value = [{"default": mock_spec}]
1213
mock_executor.determine_available_memory.return_value = [1024 * 1024 * 1024]
1214
mock_executor.initialize_from_config.return_value = None
1215
- mock_executor.max_concurrent_batches = 1
1215
1216
return mock_executor
1217
vllm/config/vllm.py
+9
@@ -487,6 +487,15 @@ class VllmConfig:
487
]
488
return hash_str
489
490
+ @property
491
+ def max_concurrent_batches(self) -> int:
492
+ # PP requires PP-size concurrent batches to fill the pipeline.
493
+ # Async scheduling requires 2 concurrent batches to overlap.
494
+ pp_size = self.parallel_config.pipeline_parallel_size
495
+ if pp_size > 1:
496
+ return pp_size
497
+ return 2 if self.scheduler_config.async_scheduling else 1
498
+
499
@property
500
def num_speculative_tokens(self) -> int:
501
if (
vllm/v1/engine/core.py
+1
-1
@@ -188,7 +188,7 @@ class EngineCore:
188
# Batch queue for scheduled batches. This enables us to asynchronously
189
# schedule and execute batches, and is required by pipeline parallelism
190
# to eliminate pipeline bubbles.
191
- self.batch_queue_size = self.model_executor.max_concurrent_batches
191
+ self.batch_queue_size = vllm_config.max_concurrent_batches
192
self.batch_queue: (
193
deque[tuple[Future[ModelRunnerOutput], SchedulerOutput, Future[Any]]] | None
194
) = None
vllm/v1/executor/abstract.py
-4
@@ -253,10 +253,6 @@ class Executor(ABC):
253
output: list[DraftTokenIds] = self.collective_rpc("take_draft_token_ids")
254
return output[0]
255
256
- @property
257
- def max_concurrent_batches(self) -> int:
258
- return 1
259
-
256
def profile(self, is_start: bool = True, profile_prefix: str | None = None):
257
self.collective_rpc("profile", args=(is_start, profile_prefix))
258
vllm/v1/executor/multiproc_executor.py
+1
-7
@@ -15,7 +15,7 @@ from concurrent.futures import Future, InvalidStateError
15
from contextlib import suppress
16
from dataclasses import dataclass
17
from enum import Enum, auto
18
-from functools import cached_property, partial
18
+from functools import partial
19
from multiprocessing.connection import Connection
20
from multiprocessing.process import BaseProcess
21
from multiprocessing.synchronize import Lock as LockType
@@ -472,12 +472,6 @@ class MultiprocExecutor(Executor):
472
self.collective_rpc("check_health", timeout=10)
473
return
474
475
- @cached_property
476
- def max_concurrent_batches(self) -> int:
477
- # PP requires PP-size concurrent batches to fill the pipeline.
478
- pp_size = self.parallel_config.pipeline_parallel_size
479
- return 2 if pp_size <= 1 and self.scheduler_config.async_scheduling else pp_size
480
-
475
def _get_output_rank(self) -> int:
476
# Only returns ModelRunnerOutput from TP rank=0 and PP rank=-1
477
# (the first TP worker of the last PP stage).
vllm/v1/executor/ray_executor.py
-8
@@ -96,14 +96,6 @@ class RayDistributedExecutor(Executor):
96
97
self.scheduler_output: SchedulerOutput | None = None
98
99
- @property
100
- def max_concurrent_batches(self) -> int:
101
- """Ray distributed executor supports pipeline parallelism,
102
- meaning that it allows PP size batches to be executed concurrently.
103
- """
104
- pp_size = self.parallel_config.pipeline_parallel_size
105
- return 2 if pp_size <= 1 and self.scheduler_config.async_scheduling else pp_size
106
-
99
def shutdown(self) -> None:
100
if logger:
101
# Somehow logger can be None here.
vllm/v1/executor/uniproc_executor.py
-5
@@ -3,7 +3,6 @@
3
import os
4
from collections.abc import Callable
5
from concurrent.futures import Future
6
-from functools import cached_property
6
from multiprocessing import Lock
7
from typing import Any
8
@@ -77,10 +76,6 @@ class UniProcExecutor(Executor):
76
local_rank = int(device_info[1]) if len(device_info) > 1 else 0
77
return distributed_init_method, 0, local_rank
78
80
- @cached_property
81
- def max_concurrent_batches(self) -> int:
82
- return 2 if self.scheduler_config.async_scheduling else 1
83
-
79
def collective_rpc( # type: ignore[override]
80
self,
81
method: str | Callable,