A high-throughput and memory-efficient inference and serving engine for LLMs
Mirrored from https://github.com/tduncan-lum/Nvidia-Optimization-Stack---VLLM.git
Synced Nov 16, 04:30 UTC
[Bugfix] cuda error running llama 3.2 (#11047)
Gene Der Su committed
Dec 9, 2024 at 23:41 UTC
82c73fd5104e010c2c98820f3e761e1e4f36c135
1 file changed
+28
-7
vllm/platforms/cuda.py
+28
-7
@@ -4,7 +4,8 @@ pynvml. However, it should not initialize cuda context.
4
5
import os
6
from functools import lru_cache, wraps
7
-from typing import TYPE_CHECKING, Callable, List, Optional, TypeVar
7
+from typing import (TYPE_CHECKING, Callable, List, Optional, Tuple, TypeVar,
8
+ Union)
9
10
import pynvml
11
import torch
@@ -78,7 +79,9 @@ class CudaPlatformBase(Platform):
79
dispatch_key: str = "CUDA"
80
81
@classmethod
81
- def get_device_capability(cls, device_id: int = 0) -> DeviceCapability:
82
+ def get_device_capability(cls,
83
+ device_id: int = 0
84
+ ) -> Optional[DeviceCapability]:
85
raise NotImplementedError
86
87
@classmethod
@@ -144,11 +147,29 @@ class NvmlCudaPlatform(CudaPlatformBase):
147
@classmethod
148
@lru_cache(maxsize=8)
149
@with_nvml_context
147
- def get_device_capability(cls, device_id: int = 0) -> DeviceCapability:
148
- physical_device_id = device_id_to_physical_device_id(device_id)
149
- handle = pynvml.nvmlDeviceGetHandleByIndex(physical_device_id)
150
- major, minor = pynvml.nvmlDeviceGetCudaComputeCapability(handle)
151
- return DeviceCapability(major=major, minor=minor)
150
+ def get_device_capability(cls,
151
+ device_id: int = 0
152
+ ) -> Optional[DeviceCapability]:
153
+ try:
154
+ physical_device_id = device_id_to_physical_device_id(device_id)
155
+ handle = pynvml.nvmlDeviceGetHandleByIndex(physical_device_id)
156
+ major, minor = pynvml.nvmlDeviceGetCudaComputeCapability(handle)
157
+ return DeviceCapability(major=major, minor=minor)
158
+ except RuntimeError:
159
+ return None
160
+
161
+ @classmethod
162
+ @lru_cache(maxsize=8)
163
+ @with_nvml_context
164
+ def has_device_capability(
165
+ cls,
166
+ capability: Union[Tuple[int, int], int],
167
+ device_id: int = 0,
168
+ ) -> bool:
169
+ try:
170
+ return super().has_device_capability(capability, device_id)
171
+ except RuntimeError:
172
+ return False
173
174
@classmethod
175
@lru_cache(maxsize=8)