A high-throughput and memory-efficient inference and serving engine for LLMs
Mirrored from https://github.com/tduncan-lum/Nvidia-Optimization-Stack---VLLM.git
Synced Nov 16, 22:00 UTC
[hardware][cuda] use device id under CUDA_VISIBLE_DEVICES for get_device_capability (#6216)
youkaichao committed
Jul 8, 2024 at 20:02 UTC
a3c9435d93fb7609977da5d90f839b9987c8b264
1 file changed
+20
-4
vllm/platforms/cuda.py
+20
-4
@@ -2,6 +2,7 @@
2
pynvml. However, it should not initialize cuda context.
3
"""
4
5
+import os
6
from functools import lru_cache, wraps
7
from typing import Tuple
8
@@ -23,12 +24,27 @@ def with_nvml_context(fn):
24
return wrapper
25
26
27
+@lru_cache(maxsize=8)
28
+@with_nvml_context
29
+def get_physical_device_capability(device_id: int = 0) -> Tuple[int, int]:
30
+ handle = pynvml.nvmlDeviceGetHandleByIndex(device_id)
31
+ return pynvml.nvmlDeviceGetCudaComputeCapability(handle)
32
+
33
+
34
+def device_id_to_physical_device_id(device_id: int) -> int:
35
+ if "CUDA_VISIBLE_DEVICES" in os.environ:
36
+ device_ids = os.environ["CUDA_VISIBLE_DEVICES"].split(",")
37
+ device_ids = [int(device_id) for device_id in device_ids]
38
+ physical_device_id = device_ids[device_id]
39
+ else:
40
+ physical_device_id = device_id
41
+ return physical_device_id
42
+
43
+
44
class CudaPlatform(Platform):
45
_enum = PlatformEnum.CUDA
46
47
@staticmethod
30
- @lru_cache(maxsize=8)
31
- @with_nvml_context
48
def get_device_capability(device_id: int = 0) -> Tuple[int, int]:
33
- handle = pynvml.nvmlDeviceGetHandleByIndex(device_id)
34
- return pynvml.nvmlDeviceGetCudaComputeCapability(handle)
49
+ physical_device_id = device_id_to_physical_device_id(device_id)
50
+ return get_physical_device_capability(physical_device_id)