@tduncan / Nvidia-Optimization-Stack-VLLM Mirror · read-only / commits / 50eed24d25
Fork 0

A high-throughput and memory-efficient inference and serving engine for LLMs

Mirrored from https://github.com/tduncan-lum/Nvidia-Optimization-Stack---VLLM.git

Synced Nov 16, 23:15 UTC

Add `cuda_device_count_stateless` (#5473)

Antoni Baum committed Jun 13, 2024 at 16:06 UTC 50eed24d252965a81ce50b64fd387d60fb1f4f6e
8 files changed +79 -23
.buildkite/test-pipeline.yaml
+1
@@ -48,6 +48,7 @@ steps:
48 - TEST_DIST_MODEL=meta-llama/Llama-2-7b-hf DISTRIBUTED_EXECUTOR_BACKEND=mp pytest -v -s distributed/test_chunked_prefill_distributed.py
49 - pytest -v -s spec_decode/e2e/test_integration_dist.py
50 - CUDA_VISIBLE_DEVICES=0,1 pytest -v -s test_sharded_state_loader.py
51 + - CUDA_VISIBLE_DEVICES=0,1 pytest -v -s distributed/test_utils.py
52
53 - label: Distributed Tests (Multiple Groups)
54 #mirror_hardwares: [amd]
tests/conftest.py
+2 -15
@@ -1,8 +1,6 @@
1 import contextlib
2 import gc
3 import os
4 -import subprocess
5 -import sys
4 from typing import Any, Dict, List, Optional, Tuple, TypeVar
5
6 import pytest
@@ -22,7 +20,7 @@ from vllm.logger import init_logger
20 from vllm.multimodal import MultiModalData
21 from vllm.multimodal.image import ImageFeatureData, ImagePixelData
22 from vllm.sequence import SampleLogprobs
25 -from vllm.utils import is_cpu
23 +from vllm.utils import cuda_device_count_stateless, is_cpu
24
25 logger = init_logger(__name__)
26
@@ -539,15 +537,4 @@ def num_gpus_available():
537 """Get number of GPUs without initializing the CUDA context
538 in current process."""
539
542 - try:
543 - out = subprocess.run([
544 - sys.executable, "-c",
545 - "import torch; print(torch.cuda.device_count())"
546 - ],
547 - capture_output=True,
548 - check=True,
549 - text=True)
550 - except subprocess.CalledProcessError as e:
551 - logger.warning("Failed to get number of GPUs.", exc_info=e)
552 - return 0
553 - return int(out.stdout.strip())
540 + return cuda_device_count_stateless()
tests/distributed/test_utils.py new
+31
@@ -0,0 +1,31 @@
1 +import os
2 +
3 +import ray
4 +
5 +from vllm.utils import cuda_device_count_stateless
6 +
7 +
8 +@ray.remote
9 +class _CUDADeviceCountStatelessTestActor():
10 +
11 + def get_count(self):
12 + return cuda_device_count_stateless()
13 +
14 + def set_cuda_visible_devices(self, cuda_visible_devices: str):
15 + os.environ["CUDA_VISIBLE_DEVICES"] = cuda_visible_devices
16 +
17 + def get_cuda_visible_devices(self):
18 + return os.environ["CUDA_VISIBLE_DEVICES"]
19 +
20 +
21 +def test_cuda_device_count_stateless():
22 + """Test that cuda_device_count_stateless changes return value if
23 + CUDA_VISIBLE_DEVICES is changed."""
24 +
25 + actor = _CUDADeviceCountStatelessTestActor.options(num_gpus=2).remote()
26 + assert ray.get(actor.get_cuda_visible_devices.remote()) == "0,1"
27 + assert ray.get(actor.get_count.remote()) == 2
28 + ray.get(actor.set_cuda_visible_devices.remote("0"))
29 + assert ray.get(actor.get_count.remote()) == 1
30 + ray.get(actor.set_cuda_visible_devices.remote(""))
31 + assert ray.get(actor.get_count.remote()) == 0
vllm/config.py
+3 -3
@@ -11,7 +11,8 @@ from vllm.logger import init_logger
11 from vllm.model_executor.layers.quantization import QUANTIZATION_METHODS
12 from vllm.model_executor.models import ModelRegistry
13 from vllm.transformers_utils.config import get_config, get_hf_text_config
14 -from vllm.utils import get_cpu_memory, is_cpu, is_hip, is_neuron, is_tpu
14 +from vllm.utils import (cuda_device_count_stateless, get_cpu_memory, is_cpu,
15 + is_hip, is_neuron, is_tpu)
16
17 if TYPE_CHECKING:
18 from ray.util.placement_group import PlacementGroup
@@ -605,12 +606,11 @@ class ParallelConfig:
606 if self.distributed_executor_backend is None and self.world_size > 1:
607 # We use multiprocessing by default if world_size fits on the
608 # current node and we aren't in a ray placement group.
608 - from torch.cuda import device_count
609
610 from vllm.executor import ray_utils
611 backend = "mp"
612 ray_found = ray_utils.ray is not None
613 - if device_count() < self.world_size:
613 + if cuda_device_count_stateless() < self.world_size:
614 if not ray_found:
615 raise ValueError("Unable to load Ray which is "
616 "required for multi-node inference")
vllm/distributed/device_communicators/custom_all_reduce.py
+2 -1
@@ -11,6 +11,7 @@ from vllm.distributed.device_communicators.custom_all_reduce_utils import (
11 gpu_p2p_access_check)
12 from vllm.distributed.parallel_state import is_in_the_same_node
13 from vllm.logger import init_logger
14 +from vllm.utils import cuda_device_count_stateless
15
16 try:
17 import pynvml
@@ -144,7 +145,7 @@ class CustomAllreduce:
145 if cuda_visible_devices:
146 device_ids = list(map(int, cuda_visible_devices.split(",")))
147 else:
147 - device_ids = list(range(torch.cuda.device_count()))
148 + device_ids = list(range(cuda_device_count_stateless()))
149
150 physical_device_id = device_ids[device.index]
151 tensor = torch.tensor([physical_device_id],
vllm/distributed/device_communicators/custom_all_reduce_utils.py
+2 -1
@@ -12,6 +12,7 @@ import torch.multiprocessing as mp
12
13 import vllm.envs as envs
14 from vllm.logger import init_logger
15 +from vllm.utils import cuda_device_count_stateless
16
17 logger = init_logger(__name__)
18
@@ -152,7 +153,7 @@ def gpu_p2p_access_check(i: int, j: int) -> bool:
153
154 is_distributed = dist.is_initialized()
155
155 - num_dev = torch.cuda.device_count()
156 + num_dev = cuda_device_count_stateless()
157 cuda_visible_devices = envs.CUDA_VISIBLE_DEVICES
158 if cuda_visible_devices is None:
159 cuda_visible_devices = ",".join(str(i) for i in range(num_dev))
vllm/executor/multiproc_gpu_executor.py
+3 -3
@@ -9,7 +9,8 @@ from vllm.executor.multiproc_worker_utils import (ProcessWorkerWrapper,
9 ResultHandler, WorkerMonitor)
10 from vllm.logger import init_logger
11 from vllm.sequence import ExecuteModelRequest, SamplerOutput
12 -from vllm.utils import (get_distributed_init_method, get_ip, get_open_port,
12 +from vllm.utils import (cuda_device_count_stateless,
13 + get_distributed_init_method, get_ip, get_open_port,
14 get_vllm_instance_id, make_async)
15
16 logger = init_logger(__name__)
@@ -33,8 +34,7 @@ class MultiprocessingGPUExecutor(DistributedGPUExecutor):
34 # Disable torch async compiling which won't work with daemonic processes
35 os.environ["TORCHINDUCTOR_COMPILE_THREADS"] = "1"
36
36 - from torch.cuda import device_count
37 - assert world_size <= device_count(), (
37 + assert world_size <= cuda_device_count_stateless(), (
38 "please set tensor_parallel_size to less than max local gpu count")
39
40 distributed_init_method = get_distributed_init_method(
vllm/utils.py
+35
@@ -693,3 +693,38 @@ def deprecate_kwargs(
693 return inner # type: ignore
694
695 return wrapper
696 +
697 +
698 +@lru_cache(maxsize=8)
699 +def _cuda_device_count_stateless(
700 + cuda_visible_devices: Optional[str] = None) -> int:
701 + # Note: cuda_visible_devices is not used, but we keep it as an argument for
702 + # LRU Cache purposes.
703 +
704 + # Code below is based on
705 + # https://github.com/pytorch/pytorch/blob/
706 + # c1cd946818442aca8c7f812b16d187ce1586c3bc/
707 + # torch/cuda/__init__.py#L831C1-L831C17
708 + import torch.cuda
709 + import torch.version
710 +
711 + if not torch.cuda._is_compiled():
712 + return 0
713 + # bypass _device_count_nvml() if rocm (not supported)
714 + nvml_count = -1 if torch.version.hip else torch.cuda._device_count_nvml()
715 + r = torch._C._cuda_getDeviceCount() if nvml_count < 0 else nvml_count
716 + return r
717 +
718 +
719 +def cuda_device_count_stateless() -> int:
720 + """Get number of CUDA devices, caching based on the value of
721 + CUDA_VISIBLE_DEVICES at the time of call.
722 +
723 + This should be used instead of torch.cuda.device_count()
724 + unless CUDA_VISIBLE_DEVICES has already been set to the desired
725 + value."""
726 +
727 + # This can be removed and simply replaced with torch.cuda.get_device_count
728 + # after https://github.com/pytorch/pytorch/pull/122815 is released.
729 +
730 + return _cuda_device_count_stateless(envs.CUDA_VISIBLE_DEVICES)