@tduncan / Nvidia-Optimization-Stack-VLLM Mirror · read-only / commits / 660dea1235
Fork 0

A high-throughput and memory-efficient inference and serving engine for LLMs

Mirrored from https://github.com/tduncan-lum/Nvidia-Optimization-Stack---VLLM.git

Synced Nov 16, 19:30 UTC

[cuda][misc] remove error_on_invalid_device_count_status (#7069)

youkaichao committed Aug 2, 2024 at 00:14 UTC 660dea1235bfe8987e4e9136ce70269084384b2f
3 files changed +3 -32
vllm/executor/multiproc_gpu_executor.py
-3
@@ -17,7 +17,6 @@ from vllm.logger import init_logger
17 from vllm.sequence import ExecuteModelRequest, SamplerOutput
18 from vllm.triton_utils import maybe_set_triton_cache_manager
19 from vllm.utils import (_run_task_with_lock, cuda_device_count_stateless,
20 - error_on_invalid_device_count_status,
20 get_distributed_init_method, get_open_port,
21 get_vllm_instance_id, make_async,
22 update_environment_variables)
@@ -79,8 +78,6 @@ class MultiprocessingGPUExecutor(DistributedGPUExecutor):
78 f"please ensure that world_size ({world_size}) "
79 f"is less than than max local gpu count ({cuda_device_count})")
80
82 - error_on_invalid_device_count_status()
83 -
81 # Multiprocessing-based executor does not support multi-node setting.
82 # Since it only works for single node, we can use the loopback address
83 # 127.0.0.1 for communication.
vllm/executor/ray_gpu_executor.py
+3 -6
@@ -10,10 +10,9 @@ from vllm.executor.distributed_gpu_executor import ( # yapf: disable
10 from vllm.executor.ray_utils import RayWorkerWrapper, ray
11 from vllm.logger import init_logger
12 from vllm.sequence import ExecuteModelRequest, SamplerOutput
13 -from vllm.utils import (_run_task_with_lock,
14 - error_on_invalid_device_count_status,
15 - get_distributed_init_method, get_ip, get_open_port,
16 - get_vllm_instance_id, make_async)
13 +from vllm.utils import (_run_task_with_lock, get_distributed_init_method,
14 + get_ip, get_open_port, get_vllm_instance_id,
15 + make_async)
16
17 if ray is not None:
18 from ray.util.scheduling_strategies import PlacementGroupSchedulingStrategy
@@ -216,8 +215,6 @@ class RayGPUExecutor(DistributedGPUExecutor):
215 distributed_init_method = get_distributed_init_method(
216 driver_ip, get_open_port())
217
219 - error_on_invalid_device_count_status()
220 -
218 # Initialize the actual workers inside worker wrapper.
219 init_worker_all_kwargs = [
220 self._get_worker_kwargs(
vllm/utils.py
-23
@@ -1,6 +1,5 @@
1 import argparse
2 import asyncio
3 -import contextlib
3 import datetime
4 import enum
5 import gc
@@ -923,28 +922,6 @@ def cuda_device_count_stateless() -> int:
922 return _cuda_device_count_stateless(envs.CUDA_VISIBLE_DEVICES)
923
924
926 -def error_on_invalid_device_count_status():
927 - cache_entries = 0
928 - with contextlib.suppress(Exception):
929 - # future pytorch will fix the issue, device_count will not be cached
930 - # at that time, `.cache_info().currsize` will error out
931 - cache_entries = torch.cuda.device_count.cache_info( # type: ignore
932 - ).currsize
933 - if cache_entries != 0:
934 - # the function is already called, and the result is cached
935 - remembered = torch.cuda.device_count()
936 - current = cuda_device_count_stateless()
937 - if remembered > current:
938 - raise RuntimeError(
939 - "The number of CUDA devices has changed since the first "
940 - "call to torch.cuda.device_count(). This is not allowed "
941 - "and may result in undefined behavior. Please check out "
942 - "https://github.com/vllm-project/vllm/issues/6056 to "
943 - "find the first call to torch.cuda.device_count() "
944 - "and defer it until the engine is up. Or you can set "
945 - "CUDA_VISIBLE_DEVICES to the GPUs you want to use.")
946 -
947 -
925 # NVML utils
926 # Note that NVML is not affected by `CUDA_VISIBLE_DEVICES`,
927 # all the related functions work on real physical device ids.