@tduncan / Nvidia-Optimization-Stack-VLLM Mirror · read-only / commits / f666207161
Fork 0

A high-throughput and memory-efficient inference and serving engine for LLMs

Mirrored from https://github.com/tduncan-lum/Nvidia-Optimization-Stack---VLLM.git

Synced Nov 16, 22:00 UTC

[misc][distributed] error on invalid state (#6092)

youkaichao committed Jul 2, 2024 at 23:37 UTC f666207161e5ecc2cd6ebab93c5a62cf44f30641
3 files changed +29 -1
vllm/executor/multiproc_gpu_executor.py
+3
@@ -10,6 +10,7 @@ from vllm.executor.multiproc_worker_utils import (ProcessWorkerWrapper,
10 from vllm.logger import init_logger
11 from vllm.sequence import ExecuteModelRequest, SamplerOutput
12 from vllm.utils import (cuda_device_count_stateless,
13 + error_on_invalid_device_count_status,
14 get_distributed_init_method, get_open_port,
15 get_vllm_instance_id, make_async,
16 update_environment_variables)
@@ -39,6 +40,8 @@ class MultiprocessingGPUExecutor(DistributedGPUExecutor):
40 assert world_size <= cuda_device_count_stateless(), (
41 "please set tensor_parallel_size to less than max local gpu count")
42
43 + error_on_invalid_device_count_status()
44 +
45 # Multiprocessing-based executor does not support multi-node setting.
46 # Since it only works for single node, we can use the loopback address
47 # 127.0.0.1 for communication.
vllm/executor/ray_gpu_executor.py
+4 -1
@@ -11,7 +11,8 @@ from vllm.executor.distributed_gpu_executor import ( # yapf: disable
11 from vllm.executor.ray_utils import RayWorkerWrapper, ray
12 from vllm.logger import init_logger
13 from vllm.sequence import ExecuteModelRequest, SamplerOutput
14 -from vllm.utils import (get_distributed_init_method, get_ip, get_open_port,
14 +from vllm.utils import (error_on_invalid_device_count_status,
15 + get_distributed_init_method, get_ip, get_open_port,
16 get_vllm_instance_id, make_async)
17
18 if ray is not None:
@@ -175,6 +176,8 @@ class RayGPUExecutor(DistributedGPUExecutor):
176 distributed_init_method = get_distributed_init_method(
177 driver_ip, get_open_port())
178
179 + error_on_invalid_device_count_status()
180 +
181 # Initialize the actual workers inside worker wrapper.
182 init_worker_all_kwargs = [
183 self._get_worker_kwargs(
vllm/utils.py
+22
@@ -1,5 +1,6 @@
1 import argparse
2 import asyncio
3 +import contextlib
4 import datetime
5 import enum
6 import gc
@@ -816,6 +817,27 @@ def cuda_device_count_stateless() -> int:
817 return _cuda_device_count_stateless(envs.CUDA_VISIBLE_DEVICES)
818
819
820 +def error_on_invalid_device_count_status():
821 + cache_entries = 0
822 + with contextlib.suppress(Exception):
823 + # future pytorch will fix the issue, device_count will not be cached
824 + # at that time, `.cache_info().currsize` will error out
825 + cache_entries = torch.cuda.device_count.cache_info().currsize
826 + if cache_entries != 0:
827 + # the function is already called, and the result is cached
828 + remembered = torch.cuda.device_count()
829 + current = cuda_device_count_stateless()
830 + if remembered > current:
831 + raise RuntimeError(
832 + "The number of CUDA devices has changed since the first "
833 + "call to torch.cuda.device_count(). This is not allowed "
834 + "and may result in undefined behavior. Please check out "
835 + "https://github.com/vllm-project/vllm/issues/6056 to "
836 + "find the first call to torch.cuda.device_count() "
837 + "and defer it until the engine is up. Or you can set "
838 + "CUDA_VISIBLE_DEVICES to the GPUs you want to use.")
839 +
840 +
841 # NVML utils
842 # Note that NVML is not affected by `CUDA_VISIBLE_DEVICES`,
843 # all the related functions work on real physical device ids.