@tduncan / Nvidia-Optimization-Stack-VLLM Mirror · read-only / commits / 99dac099ab
Fork 0

A high-throughput and memory-efficient inference and serving engine for LLMs

Mirrored from https://github.com/tduncan-lum/Nvidia-Optimization-Stack---VLLM.git

Synced Nov 16, 23:15 UTC

[Core][Doc] Default to multiprocessing for single-node distributed case (#5230)

Co-authored-by: Antoni Baum <antoni.baum@protonmail.com>

Nick Hill committed Jun 11, 2024 at 11:10 UTC 99dac099ab5205d40bfaf5cf5652884b8764a400
5 files changed +31 -14
docs/source/serving/distributed_serving.rst
+5 -5
@@ -3,11 +3,9 @@
3 Distributed Inference and Serving
4 =================================
5
6 -vLLM supports distributed tensor-parallel inference and serving. Currently, we support `Megatron-LM's tensor parallel algorithm <https://arxiv.org/pdf/1909.08053.pdf>`_. We manage the distributed runtime with `Ray <https://github.com/ray-project/ray>`_. To run distributed inference, install Ray with:
6 +vLLM supports distributed tensor-parallel inference and serving. Currently, we support `Megatron-LM's tensor parallel algorithm <https://arxiv.org/pdf/1909.08053.pdf>`_. We manage the distributed runtime with either `Ray <https://github.com/ray-project/ray>`_ or python native multiprocessing. Multiprocessing can be used when deploying on a single node, multi-node inferencing currently requires Ray.
7
8 -.. code-block:: console
9 -
10 - $ pip install ray
8 +Multiprocessing will be used by default when not running in a Ray placement group and if there are sufficient GPUs available on the same node for the configured :code:`tensor_parallel_size`, otherwise Ray will be used. This default can be overridden via the :code:`LLM` class :code:`distributed-executor-backend` argument or :code:`--distributed-executor-backend` API server argument. Set it to :code:`mp` for multiprocessing or :code:`ray` for Ray. It's not required for Ray to be installed for the multiprocessing case.
9
10 To run multi-GPU inference with the :code:`LLM` class, set the :code:`tensor_parallel_size` argument to the number of GPUs you want to use. For example, to run inference on 4 GPUs:
11
@@ -25,10 +23,12 @@ To run multi-GPU serving, pass in the :code:`--tensor-parallel-size` argument wh
23 $ --model facebook/opt-13b \
24 $ --tensor-parallel-size 4
25
28 -To scale vLLM beyond a single machine, start a `Ray runtime <https://docs.ray.io/en/latest/ray-core/starting-ray.html>`_ via CLI before running vLLM:
26 +To scale vLLM beyond a single machine, install and start a `Ray runtime <https://docs.ray.io/en/latest/ray-core/starting-ray.html>`_ via CLI before running vLLM:
27
28 .. code-block:: console
29
30 + $ pip install ray
31 +
32 $ # On head node
33 $ ray start --head
34
tests/spec_decode/e2e/conftest.py
+4
@@ -77,7 +77,11 @@ class AsyncLLM:
77 swap_space=swap_space,
78 enforce_eager=enforce_eager,
79 max_seq_len_to_capture=max_seq_len_to_capture,
80 + # For now use ray for the distributed back-end, since
81 + # we rely on the use of engine_use_ray=True to avoid
82 + # reinitializing CUDA in the same process (driver worker)
83 engine_use_ray=True,
84 + distributed_executor_backend="ray",
85 disable_custom_all_reduce=disable_custom_all_reduce,
86 **kwargs,
87 )
vllm/config.py
+17 -1
@@ -603,9 +603,25 @@ class ParallelConfig:
603 f"'{self.distributed_executor_backend}'.")
604
605 if self.distributed_executor_backend is None and self.world_size > 1:
606 + # We use multiprocessing by default if world_size fits on the
607 + # current node and we aren't in a ray placement group.
608 + from torch.cuda import device_count
609 +
610 from vllm.executor import ray_utils
611 + backend = "mp"
612 ray_found = ray_utils.ray is not None
608 - self.distributed_executor_backend = "ray" if ray_found else "mp"
613 + if device_count() < self.world_size:
614 + if not ray_found:
615 + raise ValueError("Unable to load Ray which is "
616 + "required for multi-node inference")
617 + backend = "ray"
618 + elif ray_found:
619 + from ray.util import get_current_placement_group
620 + if self.placement_group or get_current_placement_group():
621 + backend = "ray"
622 + self.distributed_executor_backend = backend
623 + logger.info("Defaulting to use %s for distributed inference",
624 + backend)
625
626 self._verify_args()
627
vllm/executor/multiproc_gpu_executor.py
-4
@@ -19,10 +19,6 @@ class MultiprocessingGPUExecutor(DistributedGPUExecutor):
19 """Python multiprocessing-based multi-GPU executor"""
20
21 def _init_executor(self) -> None:
22 - assert (
23 - not self.speculative_config
24 - ), "Speculative decoding not yet supported for MultiProcGPU backend."
25 -
22 # Create the parallel GPU workers.
23 world_size = self.parallel_config.tensor_parallel_size
24
vllm/executor/multiproc_worker_utils.py
+5 -4
@@ -65,10 +65,11 @@ def _set_future_result(future: Union[ResultFuture, asyncio.Future],
65 future.set_result(result)
66 return
67 loop = future.get_loop()
68 - if result.exception is not None:
69 - loop.call_soon_threadsafe(future.set_exception, result.exception)
70 - else:
71 - loop.call_soon_threadsafe(future.set_result, result.value)
68 + if not loop.is_closed():
69 + if result.exception is not None:
70 + loop.call_soon_threadsafe(future.set_exception, result.exception)
71 + else:
72 + loop.call_soon_threadsafe(future.set_result, result.value)
73
74
75 class ResultHandler(threading.Thread):