@tduncan / Nvidia-Optimization-Stack-VLLM Mirror · read-only / commits / 2b83503227
Fork 0

A high-throughput and memory-efficient inference and serving engine for LLMs

Mirrored from https://github.com/tduncan-lum/Nvidia-Optimization-Stack---VLLM.git

Synced Nov 16, 02:00 UTC

[misc] fix cross-node TP (#12166)

Signed-off-by: youkaichao <youkaichao@gmail.com>

youkaichao committed Jan 18, 2025 at 10:53 UTC 2b835032275622d70b19b8cd834740336dc26138
2 files changed +36 -24
vllm/executor/mp_distributed_executor.py
+36 -2
@@ -1,4 +1,5 @@
1 import asyncio
2 +import os
3 from typing import Any, Callable, List, Optional, Union
4
5 import cloudpickle
@@ -10,8 +11,9 @@ from vllm.executor.multiproc_worker_utils import (
11 from vllm.logger import init_logger
12 from vllm.model_executor.layers.sampler import SamplerOutput
13 from vllm.sequence import ExecuteModelRequest
13 -from vllm.utils import (_run_task_with_lock, get_distributed_init_method,
14 - get_ip, get_open_port, make_async, run_method)
14 +from vllm.utils import (_run_task_with_lock, cuda_device_count_stateless,
15 + get_distributed_init_method, get_ip, get_open_port,
16 + make_async, run_method, update_environment_variables)
17 from vllm.worker.worker_base import WorkerWrapperBase
18
19 logger = init_logger(__name__)
@@ -22,7 +24,39 @@ class MultiprocessingDistributedExecutor(DistributedExecutorBase):
24
25 uses_ray: bool = False
26
27 + def _check_cuda(self) -> None:
28 + """Check that the number of GPUs is sufficient for the parallel
29 + configuration. Separate from _init_executor to reduce the number of
30 + indented blocks.
31 + """
32 + parallel_config = self.parallel_config
33 + world_size = parallel_config.world_size
34 + tensor_parallel_size = parallel_config.tensor_parallel_size
35 +
36 + cuda_device_count = cuda_device_count_stateless()
37 + # Use confusing message for more common TP-only case.
38 + if tensor_parallel_size > cuda_device_count:
39 + raise RuntimeError(
40 + f"please set tensor_parallel_size ({tensor_parallel_size}) "
41 + f"to less than max local gpu count ({cuda_device_count})")
42 +
43 + if world_size > cuda_device_count:
44 + raise RuntimeError(
45 + f"please ensure that world_size ({world_size}) "
46 + f"is less than than max local gpu count ({cuda_device_count})")
47 +
48 + # Set CUDA_VISIBLE_DEVICES for the driver, inherited by workers
49 + if "CUDA_VISIBLE_DEVICES" not in os.environ:
50 + update_environment_variables({
51 + "CUDA_VISIBLE_DEVICES": (",".join(map(str, range(world_size))))
52 + })
53 +
54 def _init_executor(self) -> None:
55 +
56 + from vllm.platforms import current_platform
57 + if current_platform.is_cuda_alike():
58 + self._check_cuda()
59 +
60 # Create the parallel GPU workers.
61 world_size = self.parallel_config.world_size
62 tensor_parallel_size = self.parallel_config.tensor_parallel_size
vllm/platforms/cuda.py
-22
@@ -139,28 +139,6 @@ class CudaPlatformBase(Platform):
139 else:
140 parallel_config.worker_cls = "vllm.worker.worker.Worker"
141
142 - world_size = parallel_config.world_size
143 - tensor_parallel_size = parallel_config.tensor_parallel_size
144 -
145 - from vllm.utils import (cuda_device_count_stateless,
146 - update_environment_variables)
147 -
148 - # Set CUDA_VISIBLE_DEVICES for the driver, inherited by workers
149 - if "CUDA_VISIBLE_DEVICES" not in os.environ:
150 - update_environment_variables({
151 - "CUDA_VISIBLE_DEVICES": (",".join(map(str, range(world_size))))
152 - })
153 -
154 - cuda_device_count = cuda_device_count_stateless()
155 - # Use confusing message for more common TP-only case.
156 - assert tensor_parallel_size <= cuda_device_count, (
157 - f"please set tensor_parallel_size ({tensor_parallel_size}) "
158 - f"to less than max local gpu count ({cuda_device_count})")
159 -
160 - assert world_size <= cuda_device_count, (
161 - f"please ensure that world_size ({world_size}) "
162 - f"is less than than max local gpu count ({cuda_device_count})")
163 -
142 cache_config = vllm_config.cache_config
143 if cache_config and cache_config.block_size is None:
144 cache_config.block_size = 16