@tduncan / Nvidia-Optimization-Stack-VLLM Mirror · read-only / commits / eed020f673
Fork 0

A high-throughput and memory-efficient inference and serving engine for LLMs

Mirrored from https://github.com/tduncan-lum/Nvidia-Optimization-Stack---VLLM.git

Synced Nov 16, 18:15 UTC

[misc] use nvml to get consistent device name (#7582)

youkaichao committed Aug 16, 2024 at 21:15 UTC eed020f6730813b7d1d97e8d81333a85b70b2af0
5 files changed +46 -2
vllm/model_executor/layers/fused_moe/fused_moe.py
+2 -1
@@ -11,6 +11,7 @@ import triton.language as tl
11 import vllm.envs as envs
12 from vllm import _custom_ops as ops
13 from vllm.logger import init_logger
14 +from vllm.platforms import current_platform
15
16 logger = init_logger(__name__)
17
@@ -287,7 +288,7 @@ def invoke_fused_moe_kernel(A: torch.Tensor, B: torch.Tensor, C: torch.Tensor,
288
289
290 def get_config_file_name(E: int, N: int, dtype: Optional[str]) -> str:
290 - device_name = torch.cuda.get_device_name().replace(" ", "_")
291 + device_name = current_platform.get_device_name().replace(" ", "_")
292 dtype_selector = "" if not dtype else f",dtype={dtype}"
293 return f"E={E},N={N},device_name={device_name}{dtype_selector}.json"
294
vllm/platforms/cuda.py
+34
@@ -44,6 +44,35 @@ def get_physical_device_capability(device_id: int = 0) -> Tuple[int, int]:
44 return pynvml.nvmlDeviceGetCudaComputeCapability(handle)
45
46
47 +@lru_cache(maxsize=8)
48 +@with_nvml_context
49 +def get_physical_device_name(device_id: int = 0) -> str:
50 + handle = pynvml.nvmlDeviceGetHandleByIndex(device_id)
51 + return pynvml.nvmlDeviceGetName(handle)
52 +
53 +
54 +@with_nvml_context
55 +def warn_if_different_devices():
56 + device_ids: int = pynvml.nvmlDeviceGetCount()
57 + if device_ids > 1:
58 + device_names = [get_physical_device_name(i) for i in range(device_ids)]
59 + if len(set(device_names)) > 1 and os.environ.get(
60 + "CUDA_DEVICE_ORDER") != "PCI_BUS_ID":
61 + logger.warning(
62 + "Detected different devices in the system: \n%s\nPlease"
63 + " make sure to set `CUDA_DEVICE_ORDER=PCI_BUS_ID` to "
64 + "avoid unexpected behavior.", "\n".join(device_names))
65 +
66 +
67 +try:
68 + from sphinx.ext.autodoc.mock import _MockModule
69 +
70 + if not isinstance(pynvml, _MockModule):
71 + warn_if_different_devices()
72 +except ModuleNotFoundError:
73 + warn_if_different_devices()
74 +
75 +
76 def device_id_to_physical_device_id(device_id: int) -> int:
77 if "CUDA_VISIBLE_DEVICES" in os.environ:
78 device_ids = os.environ["CUDA_VISIBLE_DEVICES"].split(",")
@@ -61,6 +90,11 @@ class CudaPlatform(Platform):
90 physical_device_id = device_id_to_physical_device_id(device_id)
91 return get_physical_device_capability(physical_device_id)
92
93 + @staticmethod
94 + def get_device_name(device_id: int = 0) -> str:
95 + physical_device_id = device_id_to_physical_device_id(device_id)
96 + return get_physical_device_name(physical_device_id)
97 +
98 @staticmethod
99 @with_nvml_context
100 def is_full_nvlink(physical_device_ids: List[int]) -> bool:
vllm/platforms/interface.py
+4
@@ -27,6 +27,10 @@ class Platform:
27 def get_device_capability(device_id: int = 0) -> Tuple[int, int]:
28 raise NotImplementedError
29
30 + @staticmethod
31 + def get_device_name(device_id: int = 0) -> str:
32 + raise NotImplementedError
33 +
34 @staticmethod
35 def inference_mode():
36 """A device-specific wrapper of `torch.inference_mode`.
vllm/platforms/rocm.py
+5
@@ -13,3 +13,8 @@ class RocmPlatform(Platform):
13 @lru_cache(maxsize=8)
14 def get_device_capability(device_id: int = 0) -> Tuple[int, int]:
15 return torch.cuda.get_device_capability(device_id)
16 +
17 + @staticmethod
18 + @lru_cache(maxsize=8)
19 + def get_device_name(device_id: int = 0) -> str:
20 + return torch.cuda.get_device_name(device_id)
vllm/worker/worker.py
+1 -1
@@ -368,7 +368,7 @@ def _check_if_gpu_supports_dtype(torch_dtype: torch.dtype):
368 if torch_dtype == torch.bfloat16:
369 compute_capability = current_platform.get_device_capability()
370 if compute_capability[0] < 8:
371 - gpu_name = torch.cuda.get_device_name()
371 + gpu_name = current_platform.get_device_name()
372 raise ValueError(
373 "Bfloat16 is only supported on GPUs with compute capability "
374 f"of at least 8.0. Your {gpu_name} GPU has compute capability "