A high-throughput and memory-efficient inference and serving engine for LLMs
Mirrored from https://github.com/tduncan-lum/Nvidia-Optimization-Stack---VLLM.git
Synced Nov 15, 23:30 UTC
[bugfix] respect distributed_executor_backend in world_size=1 (#12934)
Signed-off-by: youkaichao <youkaichao@gmail.com>
youkaichao committed
Feb 8, 2025 at 16:17 UTC
91dd8f7aa63a1923cc17868c7646d1277d64ed53
4 files changed
+53
-32
tests/engine/test_executor.py
renamed
+20
-1
@@ -55,6 +55,7 @@ def test_custom_executor(model, tmp_path):
55
engine_args = EngineArgs(
56
model=model,
57
distributed_executor_backend=CustomUniExecutor,
58
+ enforce_eager=True, # reduce test time
59
)
60
engine = LLMEngine.from_engine_args(engine_args)
61
sampling_params = SamplingParams(max_tokens=1)
@@ -75,7 +76,10 @@ def test_custom_executor_async(model, tmp_path):
76
assert not os.path.exists(".marker")
77
78
engine_args = AsyncEngineArgs(
78
- model=model, distributed_executor_backend=CustomUniExecutorAsync)
79
+ model=model,
80
+ distributed_executor_backend=CustomUniExecutorAsync,
81
+ enforce_eager=True, # reduce test time
82
+ )
83
engine = AsyncLLMEngine.from_engine_args(engine_args)
84
sampling_params = SamplingParams(max_tokens=1)
85
@@ -89,3 +93,18 @@ def test_custom_executor_async(model, tmp_path):
93
assert os.path.exists(".marker")
94
finally:
95
os.chdir(cwd)
96
+
97
+
98
+@pytest.mark.parametrize("model", ["facebook/opt-125m"])
99
+def test_respect_ray(model):
100
+ # even for TP=1 and PP=1,
101
+ # if users specify ray, we should use ray.
102
+ # users might do this if they want to manage the
103
+ # resources using ray.
104
+ engine_args = EngineArgs(
105
+ model=model,
106
+ distributed_executor_backend="ray",
107
+ enforce_eager=True, # reduce test time
108
+ )
109
+ engine = LLMEngine.from_engine_args(engine_args)
110
+ assert engine.model_executor.uses_ray
vllm/config.py
+3
@@ -1401,6 +1401,9 @@ class ParallelConfig:
1401
logger.info("Defaulting to use %s for distributed inference",
1402
backend)
1403
1404
+ if self.distributed_executor_backend is None and self.world_size == 1:
1405
+ self.distributed_executor_backend = "uni"
1406
+
1407
self._verify_args()
1408
1409
@property
vllm/engine/llm_engine.py
+22
-22
@@ -434,6 +434,7 @@ class LLMEngine:
434
@classmethod
435
def _get_executor_cls(cls,
436
engine_config: VllmConfig) -> Type[ExecutorBase]:
437
+ # distributed_executor_backend must be set in VllmConfig.__post_init__
438
distributed_executor_backend = (
439
engine_config.parallel_config.distributed_executor_backend)
440
# Initialize the cluster and specify the executor class.
@@ -443,30 +444,29 @@ class LLMEngine:
444
"distributed_executor_backend must be a subclass of "
445
f"ExecutorBase. Got {distributed_executor_backend}.")
446
executor_class = distributed_executor_backend
446
- elif engine_config.parallel_config.world_size > 1:
447
- if distributed_executor_backend == "ray":
448
- from vllm.executor.ray_distributed_executor import (
449
- RayDistributedExecutor)
450
- executor_class = RayDistributedExecutor
451
- elif distributed_executor_backend == "mp":
452
- from vllm.executor.mp_distributed_executor import (
453
- MultiprocessingDistributedExecutor)
454
- assert not envs.VLLM_USE_RAY_SPMD_WORKER, (
455
- "multiprocessing distributed executor backend does not "
456
- "support VLLM_USE_RAY_SPMD_WORKER=1")
457
- executor_class = MultiprocessingDistributedExecutor
458
- elif distributed_executor_backend == "uni":
459
- # JAX-style, single-process, multi-device executor.
460
- from vllm.executor.uniproc_executor import UniProcExecutor
461
- executor_class = UniProcExecutor
462
- elif distributed_executor_backend == "external_launcher":
463
- # executor with external launcher
464
- from vllm.executor.uniproc_executor import ( # noqa
465
- ExecutorWithExternalLauncher)
466
- executor_class = ExecutorWithExternalLauncher
467
- else:
447
+ elif distributed_executor_backend == "ray":
448
+ from vllm.executor.ray_distributed_executor import (
449
+ RayDistributedExecutor)
450
+ executor_class = RayDistributedExecutor
451
+ elif distributed_executor_backend == "mp":
452
+ from vllm.executor.mp_distributed_executor import (
453
+ MultiprocessingDistributedExecutor)
454
+ assert not envs.VLLM_USE_RAY_SPMD_WORKER, (
455
+ "multiprocessing distributed executor backend does not "
456
+ "support VLLM_USE_RAY_SPMD_WORKER=1")
457
+ executor_class = MultiprocessingDistributedExecutor
458
+ elif distributed_executor_backend == "uni":
459
+ # JAX-style, single-process, multi-device executor.
460
from vllm.executor.uniproc_executor import UniProcExecutor
461
executor_class = UniProcExecutor
462
+ elif distributed_executor_backend == "external_launcher":
463
+ # executor with external launcher
464
+ from vllm.executor.uniproc_executor import ( # noqa
465
+ ExecutorWithExternalLauncher)
466
+ executor_class = ExecutorWithExternalLauncher
467
+ else:
468
+ raise ValueError("unrecognized distributed_executor_backend: "
469
+ f"{distributed_executor_backend}")
470
return executor_class
471
472
@classmethod
vllm/v1/executor/abstract.py
+8
-9
@@ -25,15 +25,14 @@ class Executor(ExecutorBase):
25
parallel_config = vllm_config.parallel_config
26
distributed_executor_backend = (
27
parallel_config.distributed_executor_backend)
28
- if distributed_executor_backend is None:
29
- # If the user does not specify the distributed executor backend,
30
- # we will choose the backend based on the world size.
31
- if parallel_config.world_size > 1:
32
- distributed_executor_backend = "mp"
33
- else:
34
- distributed_executor_backend = "uni"
35
-
36
- if distributed_executor_backend == "ray":
28
+ # distributed_executor_backend must be set in VllmConfig.__post_init__
29
+ if isinstance(distributed_executor_backend, type):
30
+ if not issubclass(distributed_executor_backend, ExecutorBase):
31
+ raise TypeError(
32
+ "distributed_executor_backend must be a subclass of "
33
+ f"ExecutorBase. Got {distributed_executor_backend}.")
34
+ executor_class = distributed_executor_backend
35
+ elif distributed_executor_backend == "ray":
36
executor_class = RayDistributedExecutor
37
elif distributed_executor_backend == "mp":
38
from vllm.v1.executor.multiproc_executor import MultiprocExecutor