@tduncan / Nvidia-Optimization-Stack-VLLM Mirror · read-only / commits / a3c9435d93
Fork 0

A high-throughput and memory-efficient inference and serving engine for LLMs

Mirrored from https://github.com/tduncan-lum/Nvidia-Optimization-Stack---VLLM.git

Synced Nov 16, 22:00 UTC

[hardware][cuda] use device id under CUDA_VISIBLE_DEVICES for get_device_capability (#6216)

youkaichao committed Jul 8, 2024 at 20:02 UTC a3c9435d93fb7609977da5d90f839b9987c8b264
1 file changed +20 -4
vllm/platforms/cuda.py
+20 -4
@@ -2,6 +2,7 @@
2 pynvml. However, it should not initialize cuda context.
3 """
4
5 +import os
6 from functools import lru_cache, wraps
7 from typing import Tuple
8
@@ -23,12 +24,27 @@ def with_nvml_context(fn):
24 return wrapper
25
26
27 +@lru_cache(maxsize=8)
28 +@with_nvml_context
29 +def get_physical_device_capability(device_id: int = 0) -> Tuple[int, int]:
30 + handle = pynvml.nvmlDeviceGetHandleByIndex(device_id)
31 + return pynvml.nvmlDeviceGetCudaComputeCapability(handle)
32 +
33 +
34 +def device_id_to_physical_device_id(device_id: int) -> int:
35 + if "CUDA_VISIBLE_DEVICES" in os.environ:
36 + device_ids = os.environ["CUDA_VISIBLE_DEVICES"].split(",")
37 + device_ids = [int(device_id) for device_id in device_ids]
38 + physical_device_id = device_ids[device_id]
39 + else:
40 + physical_device_id = device_id
41 + return physical_device_id
42 +
43 +
44 class CudaPlatform(Platform):
45 _enum = PlatformEnum.CUDA
46
47 @staticmethod
30 - @lru_cache(maxsize=8)
31 - @with_nvml_context
48 def get_device_capability(device_id: int = 0) -> Tuple[int, int]:
33 - handle = pynvml.nvmlDeviceGetHandleByIndex(device_id)
34 - return pynvml.nvmlDeviceGetCudaComputeCapability(handle)
49 + physical_device_id = device_id_to_physical_device_id(device_id)
50 + return get_physical_device_capability(physical_device_id)