A high-throughput and memory-efficient inference and serving engine for LLMs
Mirrored from https://github.com/tduncan-lum/Nvidia-Optimization-Stack---VLLM.git
Synced Nov 11, 22:30 UTC
Better error when world size is larger than node and `distributed_executor_backend` is not set (#30140)
Signed-off-by: Harry Mellor <19981378+hmellor@users.noreply.github.com>
Harry Mellor committed
Dec 6, 2025 at 04:53 UTC
bf4a901af91e431a4cdb51ed4557b31bf89c0e5d
1 file changed
+13
-9
vllm/config/parallel.py
+13
-9
@@ -184,13 +184,14 @@ class ParallelConfig:
184
distributed_executor_backend: (
185
str | DistributedExecutorBackend | type[Executor] | None
186
) = None
187
- """Backend to use for distributed model
188
- workers, either "ray" or "mp" (multiprocessing). If the product
189
- of pipeline_parallel_size and tensor_parallel_size is less than
190
- or equal to the number of GPUs available, "mp" will be used to
191
- keep processing on a single host. Otherwise, this will default
192
- to "ray" if Ray is installed and fail otherwise. Note that tpu
193
- only support Ray for distributed inference."""
187
+ """Backend to use for distributed model workers, either "ray" or "mp"
188
+ (multiprocessing). If the product of pipeline_parallel_size and tensor_parallel_size
189
+ is less than or equal to the number of GPUs available, "mp" will be used to
190
+ keep processing on a single host. Otherwise, an error will be raised. To use "mp"
191
+ you must also set nnodes, and to use "ray" you must manually set
192
+ distributed_executor_backend to "ray".
193
+
194
+ Note that tpu only support Ray for distributed inference."""
195
196
worker_cls: str = "auto"
197
"""The full name of the worker class to use. If "auto", the worker class
@@ -566,8 +567,11 @@ class ParallelConfig:
567
):
568
gpu_count = cuda_device_count_stateless()
569
raise ValueError(
569
- f"Tensor parallel size ({self.world_size}) cannot be "
570
- f"larger than the number of available GPUs ({gpu_count})."
570
+ f"World size ({self.world_size}) is larger than the number of "
571
+ f"available GPUs ({gpu_count}) in this node. If this is "
572
+ "intentional and you are using:\n"
573
+ "- ray, set '--distributed-executor-backend ray'.\n"
574
+ "- multiprocessing, set '--nnodes' appropriately."
575
)
576
elif self.data_parallel_backend == "ray":
577
logger.info(