A high-throughput and memory-efficient inference and serving engine for LLMs
Mirrored from https://github.com/tduncan-lum/Nvidia-Optimization-Stack---VLLM.git
Synced Nov 16, 19:30 UTC
[cuda][misc] remove error_on_invalid_device_count_status (#7069)
youkaichao committed
Aug 2, 2024 at 00:14 UTC
660dea1235bfe8987e4e9136ce70269084384b2f
3 files changed
+3
-32
vllm/executor/multiproc_gpu_executor.py
-3
@@ -17,7 +17,6 @@ from vllm.logger import init_logger
17
from vllm.sequence import ExecuteModelRequest, SamplerOutput
18
from vllm.triton_utils import maybe_set_triton_cache_manager
19
from vllm.utils import (_run_task_with_lock, cuda_device_count_stateless,
20
- error_on_invalid_device_count_status,
20
get_distributed_init_method, get_open_port,
21
get_vllm_instance_id, make_async,
22
update_environment_variables)
@@ -79,8 +78,6 @@ class MultiprocessingGPUExecutor(DistributedGPUExecutor):
78
f"please ensure that world_size ({world_size}) "
79
f"is less than than max local gpu count ({cuda_device_count})")
80
82
- error_on_invalid_device_count_status()
83
-
81
# Multiprocessing-based executor does not support multi-node setting.
82
# Since it only works for single node, we can use the loopback address
83
# 127.0.0.1 for communication.
vllm/executor/ray_gpu_executor.py
+3
-6
@@ -10,10 +10,9 @@ from vllm.executor.distributed_gpu_executor import ( # yapf: disable
10
from vllm.executor.ray_utils import RayWorkerWrapper, ray
11
from vllm.logger import init_logger
12
from vllm.sequence import ExecuteModelRequest, SamplerOutput
13
-from vllm.utils import (_run_task_with_lock,
14
- error_on_invalid_device_count_status,
15
- get_distributed_init_method, get_ip, get_open_port,
16
- get_vllm_instance_id, make_async)
13
+from vllm.utils import (_run_task_with_lock, get_distributed_init_method,
14
+ get_ip, get_open_port, get_vllm_instance_id,
15
+ make_async)
16
17
if ray is not None:
18
from ray.util.scheduling_strategies import PlacementGroupSchedulingStrategy
@@ -216,8 +215,6 @@ class RayGPUExecutor(DistributedGPUExecutor):
215
distributed_init_method = get_distributed_init_method(
216
driver_ip, get_open_port())
217
219
- error_on_invalid_device_count_status()
220
-
218
# Initialize the actual workers inside worker wrapper.
219
init_worker_all_kwargs = [
220
self._get_worker_kwargs(
vllm/utils.py
-23
@@ -1,6 +1,5 @@
1
import argparse
2
import asyncio
3
-import contextlib
3
import datetime
4
import enum
5
import gc
@@ -923,28 +922,6 @@ def cuda_device_count_stateless() -> int:
922
return _cuda_device_count_stateless(envs.CUDA_VISIBLE_DEVICES)
923
924
926
-def error_on_invalid_device_count_status():
927
- cache_entries = 0
928
- with contextlib.suppress(Exception):
929
- # future pytorch will fix the issue, device_count will not be cached
930
- # at that time, `.cache_info().currsize` will error out
931
- cache_entries = torch.cuda.device_count.cache_info( # type: ignore
932
- ).currsize
933
- if cache_entries != 0:
934
- # the function is already called, and the result is cached
935
- remembered = torch.cuda.device_count()
936
- current = cuda_device_count_stateless()
937
- if remembered > current:
938
- raise RuntimeError(
939
- "The number of CUDA devices has changed since the first "
940
- "call to torch.cuda.device_count(). This is not allowed "
941
- "and may result in undefined behavior. Please check out "
942
- "https://github.com/vllm-project/vllm/issues/6056 to "
943
- "find the first call to torch.cuda.device_count() "
944
- "and defer it until the engine is up. Or you can set "
945
- "CUDA_VISIBLE_DEVICES to the GPUs you want to use.")
946
-
947
-
925
# NVML utils
926
# Note that NVML is not affected by `CUDA_VISIBLE_DEVICES`,
927
# all the related functions work on real physical device ids.