@tduncan / Nvidia-Optimization-Stack-VLLM Mirror · read-only / commits / 91dd8f7aa6
Fork 0

A high-throughput and memory-efficient inference and serving engine for LLMs

Mirrored from https://github.com/tduncan-lum/Nvidia-Optimization-Stack---VLLM.git

Synced Nov 15, 23:30 UTC

[bugfix] respect distributed_executor_backend in world_size=1 (#12934)

Signed-off-by: youkaichao <youkaichao@gmail.com>

youkaichao committed Feb 8, 2025 at 16:17 UTC 91dd8f7aa63a1923cc17868c7646d1277d64ed53
4 files changed +53 -32
tests/engine/test_executor.py renamed
+20 -1
@@ -55,6 +55,7 @@ def test_custom_executor(model, tmp_path):
55 engine_args = EngineArgs(
56 model=model,
57 distributed_executor_backend=CustomUniExecutor,
58 + enforce_eager=True, # reduce test time
59 )
60 engine = LLMEngine.from_engine_args(engine_args)
61 sampling_params = SamplingParams(max_tokens=1)
@@ -75,7 +76,10 @@ def test_custom_executor_async(model, tmp_path):
76 assert not os.path.exists(".marker")
77
78 engine_args = AsyncEngineArgs(
78 - model=model, distributed_executor_backend=CustomUniExecutorAsync)
79 + model=model,
80 + distributed_executor_backend=CustomUniExecutorAsync,
81 + enforce_eager=True, # reduce test time
82 + )
83 engine = AsyncLLMEngine.from_engine_args(engine_args)
84 sampling_params = SamplingParams(max_tokens=1)
85
@@ -89,3 +93,18 @@ def test_custom_executor_async(model, tmp_path):
93 assert os.path.exists(".marker")
94 finally:
95 os.chdir(cwd)
96 +
97 +
98 +@pytest.mark.parametrize("model", ["facebook/opt-125m"])
99 +def test_respect_ray(model):
100 + # even for TP=1 and PP=1,
101 + # if users specify ray, we should use ray.
102 + # users might do this if they want to manage the
103 + # resources using ray.
104 + engine_args = EngineArgs(
105 + model=model,
106 + distributed_executor_backend="ray",
107 + enforce_eager=True, # reduce test time
108 + )
109 + engine = LLMEngine.from_engine_args(engine_args)
110 + assert engine.model_executor.uses_ray
vllm/config.py
+3
@@ -1401,6 +1401,9 @@ class ParallelConfig:
1401 logger.info("Defaulting to use %s for distributed inference",
1402 backend)
1403
1404 + if self.distributed_executor_backend is None and self.world_size == 1:
1405 + self.distributed_executor_backend = "uni"
1406 +
1407 self._verify_args()
1408
1409 @property
vllm/engine/llm_engine.py
+22 -22
@@ -434,6 +434,7 @@ class LLMEngine:
434 @classmethod
435 def _get_executor_cls(cls,
436 engine_config: VllmConfig) -> Type[ExecutorBase]:
437 + # distributed_executor_backend must be set in VllmConfig.__post_init__
438 distributed_executor_backend = (
439 engine_config.parallel_config.distributed_executor_backend)
440 # Initialize the cluster and specify the executor class.
@@ -443,30 +444,29 @@ class LLMEngine:
444 "distributed_executor_backend must be a subclass of "
445 f"ExecutorBase. Got {distributed_executor_backend}.")
446 executor_class = distributed_executor_backend
446 - elif engine_config.parallel_config.world_size > 1:
447 - if distributed_executor_backend == "ray":
448 - from vllm.executor.ray_distributed_executor import (
449 - RayDistributedExecutor)
450 - executor_class = RayDistributedExecutor
451 - elif distributed_executor_backend == "mp":
452 - from vllm.executor.mp_distributed_executor import (
453 - MultiprocessingDistributedExecutor)
454 - assert not envs.VLLM_USE_RAY_SPMD_WORKER, (
455 - "multiprocessing distributed executor backend does not "
456 - "support VLLM_USE_RAY_SPMD_WORKER=1")
457 - executor_class = MultiprocessingDistributedExecutor
458 - elif distributed_executor_backend == "uni":
459 - # JAX-style, single-process, multi-device executor.
460 - from vllm.executor.uniproc_executor import UniProcExecutor
461 - executor_class = UniProcExecutor
462 - elif distributed_executor_backend == "external_launcher":
463 - # executor with external launcher
464 - from vllm.executor.uniproc_executor import ( # noqa
465 - ExecutorWithExternalLauncher)
466 - executor_class = ExecutorWithExternalLauncher
467 - else:
447 + elif distributed_executor_backend == "ray":
448 + from vllm.executor.ray_distributed_executor import (
449 + RayDistributedExecutor)
450 + executor_class = RayDistributedExecutor
451 + elif distributed_executor_backend == "mp":
452 + from vllm.executor.mp_distributed_executor import (
453 + MultiprocessingDistributedExecutor)
454 + assert not envs.VLLM_USE_RAY_SPMD_WORKER, (
455 + "multiprocessing distributed executor backend does not "
456 + "support VLLM_USE_RAY_SPMD_WORKER=1")
457 + executor_class = MultiprocessingDistributedExecutor
458 + elif distributed_executor_backend == "uni":
459 + # JAX-style, single-process, multi-device executor.
460 from vllm.executor.uniproc_executor import UniProcExecutor
461 executor_class = UniProcExecutor
462 + elif distributed_executor_backend == "external_launcher":
463 + # executor with external launcher
464 + from vllm.executor.uniproc_executor import ( # noqa
465 + ExecutorWithExternalLauncher)
466 + executor_class = ExecutorWithExternalLauncher
467 + else:
468 + raise ValueError("unrecognized distributed_executor_backend: "
469 + f"{distributed_executor_backend}")
470 return executor_class
471
472 @classmethod
vllm/v1/executor/abstract.py
+8 -9
@@ -25,15 +25,14 @@ class Executor(ExecutorBase):
25 parallel_config = vllm_config.parallel_config
26 distributed_executor_backend = (
27 parallel_config.distributed_executor_backend)
28 - if distributed_executor_backend is None:
29 - # If the user does not specify the distributed executor backend,
30 - # we will choose the backend based on the world size.
31 - if parallel_config.world_size > 1:
32 - distributed_executor_backend = "mp"
33 - else:
34 - distributed_executor_backend = "uni"
35 -
36 - if distributed_executor_backend == "ray":
28 + # distributed_executor_backend must be set in VllmConfig.__post_init__
29 + if isinstance(distributed_executor_backend, type):
30 + if not issubclass(distributed_executor_backend, ExecutorBase):
31 + raise TypeError(
32 + "distributed_executor_backend must be a subclass of "
33 + f"ExecutorBase. Got {distributed_executor_backend}.")
34 + executor_class = distributed_executor_backend
35 + elif distributed_executor_backend == "ray":
36 executor_class = RayDistributedExecutor
37 elif distributed_executor_backend == "mp":
38 from vllm.v1.executor.multiproc_executor import MultiprocExecutor