A high-throughput and memory-efficient inference and serving engine for LLMs
Mirrored from https://github.com/tduncan-lum/Nvidia-Optimization-Stack---VLLM.git
Synced Nov 16, 18:15 UTC
[misc] use nvml to get consistent device name (#7582)
youkaichao committed
Aug 16, 2024 at 21:15 UTC
eed020f6730813b7d1d97e8d81333a85b70b2af0
5 files changed
+46
-2
vllm/model_executor/layers/fused_moe/fused_moe.py
+2
-1
@@ -11,6 +11,7 @@ import triton.language as tl
11
import vllm.envs as envs
12
from vllm import _custom_ops as ops
13
from vllm.logger import init_logger
14
+from vllm.platforms import current_platform
15
16
logger = init_logger(__name__)
17
@@ -287,7 +288,7 @@ def invoke_fused_moe_kernel(A: torch.Tensor, B: torch.Tensor, C: torch.Tensor,
288
289
290
def get_config_file_name(E: int, N: int, dtype: Optional[str]) -> str:
290
- device_name = torch.cuda.get_device_name().replace(" ", "_")
291
+ device_name = current_platform.get_device_name().replace(" ", "_")
292
dtype_selector = "" if not dtype else f",dtype={dtype}"
293
return f"E={E},N={N},device_name={device_name}{dtype_selector}.json"
294
vllm/platforms/cuda.py
+34
@@ -44,6 +44,35 @@ def get_physical_device_capability(device_id: int = 0) -> Tuple[int, int]:
44
return pynvml.nvmlDeviceGetCudaComputeCapability(handle)
45
46
47
+@lru_cache(maxsize=8)
48
+@with_nvml_context
49
+def get_physical_device_name(device_id: int = 0) -> str:
50
+ handle = pynvml.nvmlDeviceGetHandleByIndex(device_id)
51
+ return pynvml.nvmlDeviceGetName(handle)
52
+
53
+
54
+@with_nvml_context
55
+def warn_if_different_devices():
56
+ device_ids: int = pynvml.nvmlDeviceGetCount()
57
+ if device_ids > 1:
58
+ device_names = [get_physical_device_name(i) for i in range(device_ids)]
59
+ if len(set(device_names)) > 1 and os.environ.get(
60
+ "CUDA_DEVICE_ORDER") != "PCI_BUS_ID":
61
+ logger.warning(
62
+ "Detected different devices in the system: \n%s\nPlease"
63
+ " make sure to set `CUDA_DEVICE_ORDER=PCI_BUS_ID` to "
64
+ "avoid unexpected behavior.", "\n".join(device_names))
65
+
66
+
67
+try:
68
+ from sphinx.ext.autodoc.mock import _MockModule
69
+
70
+ if not isinstance(pynvml, _MockModule):
71
+ warn_if_different_devices()
72
+except ModuleNotFoundError:
73
+ warn_if_different_devices()
74
+
75
+
76
def device_id_to_physical_device_id(device_id: int) -> int:
77
if "CUDA_VISIBLE_DEVICES" in os.environ:
78
device_ids = os.environ["CUDA_VISIBLE_DEVICES"].split(",")
@@ -61,6 +90,11 @@ class CudaPlatform(Platform):
90
physical_device_id = device_id_to_physical_device_id(device_id)
91
return get_physical_device_capability(physical_device_id)
92
93
+ @staticmethod
94
+ def get_device_name(device_id: int = 0) -> str:
95
+ physical_device_id = device_id_to_physical_device_id(device_id)
96
+ return get_physical_device_name(physical_device_id)
97
+
98
@staticmethod
99
@with_nvml_context
100
def is_full_nvlink(physical_device_ids: List[int]) -> bool:
vllm/platforms/interface.py
+4
@@ -27,6 +27,10 @@ class Platform:
27
def get_device_capability(device_id: int = 0) -> Tuple[int, int]:
28
raise NotImplementedError
29
30
+ @staticmethod
31
+ def get_device_name(device_id: int = 0) -> str:
32
+ raise NotImplementedError
33
+
34
@staticmethod
35
def inference_mode():
36
"""A device-specific wrapper of `torch.inference_mode`.
vllm/platforms/rocm.py
+5
@@ -13,3 +13,8 @@ class RocmPlatform(Platform):
13
@lru_cache(maxsize=8)
14
def get_device_capability(device_id: int = 0) -> Tuple[int, int]:
15
return torch.cuda.get_device_capability(device_id)
16
+
17
+ @staticmethod
18
+ @lru_cache(maxsize=8)
19
+ def get_device_name(device_id: int = 0) -> str:
20
+ return torch.cuda.get_device_name(device_id)
vllm/worker/worker.py
+1
-1
@@ -368,7 +368,7 @@ def _check_if_gpu_supports_dtype(torch_dtype: torch.dtype):
368
if torch_dtype == torch.bfloat16:
369
compute_capability = current_platform.get_device_capability()
370
if compute_capability[0] < 8:
371
- gpu_name = torch.cuda.get_device_name()
371
+ gpu_name = current_platform.get_device_name()
372
raise ValueError(
373
"Bfloat16 is only supported on GPUs with compute capability "
374
f"of at least 8.0. Your {gpu_name} GPU has compute capability "