A high-throughput and memory-efficient inference and serving engine for LLMs
Mirrored from https://github.com/tduncan-lum/Nvidia-Optimization-Stack---VLLM.git
Synced Nov 16, 23:15 UTC
Add `cuda_device_count_stateless` (#5473)
Antoni Baum committed
Jun 13, 2024 at 16:06 UTC
50eed24d252965a81ce50b64fd387d60fb1f4f6e
8 files changed
+79
-23
.buildkite/test-pipeline.yaml
+1
@@ -48,6 +48,7 @@ steps:
48
- TEST_DIST_MODEL=meta-llama/Llama-2-7b-hf DISTRIBUTED_EXECUTOR_BACKEND=mp pytest -v -s distributed/test_chunked_prefill_distributed.py
49
- pytest -v -s spec_decode/e2e/test_integration_dist.py
50
- CUDA_VISIBLE_DEVICES=0,1 pytest -v -s test_sharded_state_loader.py
51
+ - CUDA_VISIBLE_DEVICES=0,1 pytest -v -s distributed/test_utils.py
52
53
- label: Distributed Tests (Multiple Groups)
54
#mirror_hardwares: [amd]
tests/conftest.py
+2
-15
@@ -1,8 +1,6 @@
1
import contextlib
2
import gc
3
import os
4
-import subprocess
5
-import sys
4
from typing import Any, Dict, List, Optional, Tuple, TypeVar
5
6
import pytest
@@ -22,7 +20,7 @@ from vllm.logger import init_logger
20
from vllm.multimodal import MultiModalData
21
from vllm.multimodal.image import ImageFeatureData, ImagePixelData
22
from vllm.sequence import SampleLogprobs
25
-from vllm.utils import is_cpu
23
+from vllm.utils import cuda_device_count_stateless, is_cpu
24
25
logger = init_logger(__name__)
26
@@ -539,15 +537,4 @@ def num_gpus_available():
537
"""Get number of GPUs without initializing the CUDA context
538
in current process."""
539
542
- try:
543
- out = subprocess.run([
544
- sys.executable, "-c",
545
- "import torch; print(torch.cuda.device_count())"
546
- ],
547
- capture_output=True,
548
- check=True,
549
- text=True)
550
- except subprocess.CalledProcessError as e:
551
- logger.warning("Failed to get number of GPUs.", exc_info=e)
552
- return 0
553
- return int(out.stdout.strip())
540
+ return cuda_device_count_stateless()
tests/distributed/test_utils.py
new
+31
@@ -0,0 +1,31 @@
1
+import os
2
+
3
+import ray
4
+
5
+from vllm.utils import cuda_device_count_stateless
6
+
7
+
8
+@ray.remote
9
+class _CUDADeviceCountStatelessTestActor():
10
+
11
+ def get_count(self):
12
+ return cuda_device_count_stateless()
13
+
14
+ def set_cuda_visible_devices(self, cuda_visible_devices: str):
15
+ os.environ["CUDA_VISIBLE_DEVICES"] = cuda_visible_devices
16
+
17
+ def get_cuda_visible_devices(self):
18
+ return os.environ["CUDA_VISIBLE_DEVICES"]
19
+
20
+
21
+def test_cuda_device_count_stateless():
22
+ """Test that cuda_device_count_stateless changes return value if
23
+ CUDA_VISIBLE_DEVICES is changed."""
24
+
25
+ actor = _CUDADeviceCountStatelessTestActor.options(num_gpus=2).remote()
26
+ assert ray.get(actor.get_cuda_visible_devices.remote()) == "0,1"
27
+ assert ray.get(actor.get_count.remote()) == 2
28
+ ray.get(actor.set_cuda_visible_devices.remote("0"))
29
+ assert ray.get(actor.get_count.remote()) == 1
30
+ ray.get(actor.set_cuda_visible_devices.remote(""))
31
+ assert ray.get(actor.get_count.remote()) == 0
vllm/config.py
+3
-3
@@ -11,7 +11,8 @@ from vllm.logger import init_logger
11
from vllm.model_executor.layers.quantization import QUANTIZATION_METHODS
12
from vllm.model_executor.models import ModelRegistry
13
from vllm.transformers_utils.config import get_config, get_hf_text_config
14
-from vllm.utils import get_cpu_memory, is_cpu, is_hip, is_neuron, is_tpu
14
+from vllm.utils import (cuda_device_count_stateless, get_cpu_memory, is_cpu,
15
+ is_hip, is_neuron, is_tpu)
16
17
if TYPE_CHECKING:
18
from ray.util.placement_group import PlacementGroup
@@ -605,12 +606,11 @@ class ParallelConfig:
606
if self.distributed_executor_backend is None and self.world_size > 1:
607
# We use multiprocessing by default if world_size fits on the
608
# current node and we aren't in a ray placement group.
608
- from torch.cuda import device_count
609
610
from vllm.executor import ray_utils
611
backend = "mp"
612
ray_found = ray_utils.ray is not None
613
- if device_count() < self.world_size:
613
+ if cuda_device_count_stateless() < self.world_size:
614
if not ray_found:
615
raise ValueError("Unable to load Ray which is "
616
"required for multi-node inference")
vllm/distributed/device_communicators/custom_all_reduce.py
+2
-1
@@ -11,6 +11,7 @@ from vllm.distributed.device_communicators.custom_all_reduce_utils import (
11
gpu_p2p_access_check)
12
from vllm.distributed.parallel_state import is_in_the_same_node
13
from vllm.logger import init_logger
14
+from vllm.utils import cuda_device_count_stateless
15
16
try:
17
import pynvml
@@ -144,7 +145,7 @@ class CustomAllreduce:
145
if cuda_visible_devices:
146
device_ids = list(map(int, cuda_visible_devices.split(",")))
147
else:
147
- device_ids = list(range(torch.cuda.device_count()))
148
+ device_ids = list(range(cuda_device_count_stateless()))
149
150
physical_device_id = device_ids[device.index]
151
tensor = torch.tensor([physical_device_id],
vllm/distributed/device_communicators/custom_all_reduce_utils.py
+2
-1
@@ -12,6 +12,7 @@ import torch.multiprocessing as mp
12
13
import vllm.envs as envs
14
from vllm.logger import init_logger
15
+from vllm.utils import cuda_device_count_stateless
16
17
logger = init_logger(__name__)
18
@@ -152,7 +153,7 @@ def gpu_p2p_access_check(i: int, j: int) -> bool:
153
154
is_distributed = dist.is_initialized()
155
155
- num_dev = torch.cuda.device_count()
156
+ num_dev = cuda_device_count_stateless()
157
cuda_visible_devices = envs.CUDA_VISIBLE_DEVICES
158
if cuda_visible_devices is None:
159
cuda_visible_devices = ",".join(str(i) for i in range(num_dev))
vllm/executor/multiproc_gpu_executor.py
+3
-3
@@ -9,7 +9,8 @@ from vllm.executor.multiproc_worker_utils import (ProcessWorkerWrapper,
9
ResultHandler, WorkerMonitor)
10
from vllm.logger import init_logger
11
from vllm.sequence import ExecuteModelRequest, SamplerOutput
12
-from vllm.utils import (get_distributed_init_method, get_ip, get_open_port,
12
+from vllm.utils import (cuda_device_count_stateless,
13
+ get_distributed_init_method, get_ip, get_open_port,
14
get_vllm_instance_id, make_async)
15
16
logger = init_logger(__name__)
@@ -33,8 +34,7 @@ class MultiprocessingGPUExecutor(DistributedGPUExecutor):
34
# Disable torch async compiling which won't work with daemonic processes
35
os.environ["TORCHINDUCTOR_COMPILE_THREADS"] = "1"
36
36
- from torch.cuda import device_count
37
- assert world_size <= device_count(), (
37
+ assert world_size <= cuda_device_count_stateless(), (
38
"please set tensor_parallel_size to less than max local gpu count")
39
40
distributed_init_method = get_distributed_init_method(
vllm/utils.py
+35
@@ -693,3 +693,38 @@ def deprecate_kwargs(
693
return inner # type: ignore
694
695
return wrapper
696
+
697
+
698
+@lru_cache(maxsize=8)
699
+def _cuda_device_count_stateless(
700
+ cuda_visible_devices: Optional[str] = None) -> int:
701
+ # Note: cuda_visible_devices is not used, but we keep it as an argument for
702
+ # LRU Cache purposes.
703
+
704
+ # Code below is based on
705
+ # https://github.com/pytorch/pytorch/blob/
706
+ # c1cd946818442aca8c7f812b16d187ce1586c3bc/
707
+ # torch/cuda/__init__.py#L831C1-L831C17
708
+ import torch.cuda
709
+ import torch.version
710
+
711
+ if not torch.cuda._is_compiled():
712
+ return 0
713
+ # bypass _device_count_nvml() if rocm (not supported)
714
+ nvml_count = -1 if torch.version.hip else torch.cuda._device_count_nvml()
715
+ r = torch._C._cuda_getDeviceCount() if nvml_count < 0 else nvml_count
716
+ return r
717
+
718
+
719
+def cuda_device_count_stateless() -> int:
720
+ """Get number of CUDA devices, caching based on the value of
721
+ CUDA_VISIBLE_DEVICES at the time of call.
722
+
723
+ This should be used instead of torch.cuda.device_count()
724
+ unless CUDA_VISIBLE_DEVICES has already been set to the desired
725
+ value."""
726
+
727
+ # This can be removed and simply replaced with torch.cuda.get_device_count
728
+ # after https://github.com/pytorch/pytorch/pull/122815 is released.
729
+
730
+ return _cuda_device_count_stateless(envs.CUDA_VISIBLE_DEVICES)