@tduncan / Nvidia-Optimization-Stack-VLLM Mirror · read-only / commits / 82c73fd510
Fork 0

A high-throughput and memory-efficient inference and serving engine for LLMs

Mirrored from https://github.com/tduncan-lum/Nvidia-Optimization-Stack---VLLM.git

Synced Nov 16, 04:30 UTC

[Bugfix] cuda error running llama 3.2 (#11047)

Gene Der Su committed Dec 9, 2024 at 23:41 UTC 82c73fd5104e010c2c98820f3e761e1e4f36c135
1 file changed +28 -7
vllm/platforms/cuda.py
+28 -7
@@ -4,7 +4,8 @@ pynvml. However, it should not initialize cuda context.
4
5 import os
6 from functools import lru_cache, wraps
7 -from typing import TYPE_CHECKING, Callable, List, Optional, TypeVar
7 +from typing import (TYPE_CHECKING, Callable, List, Optional, Tuple, TypeVar,
8 + Union)
9
10 import pynvml
11 import torch
@@ -78,7 +79,9 @@ class CudaPlatformBase(Platform):
79 dispatch_key: str = "CUDA"
80
81 @classmethod
81 - def get_device_capability(cls, device_id: int = 0) -> DeviceCapability:
82 + def get_device_capability(cls,
83 + device_id: int = 0
84 + ) -> Optional[DeviceCapability]:
85 raise NotImplementedError
86
87 @classmethod
@@ -144,11 +147,29 @@ class NvmlCudaPlatform(CudaPlatformBase):
147 @classmethod
148 @lru_cache(maxsize=8)
149 @with_nvml_context
147 - def get_device_capability(cls, device_id: int = 0) -> DeviceCapability:
148 - physical_device_id = device_id_to_physical_device_id(device_id)
149 - handle = pynvml.nvmlDeviceGetHandleByIndex(physical_device_id)
150 - major, minor = pynvml.nvmlDeviceGetCudaComputeCapability(handle)
151 - return DeviceCapability(major=major, minor=minor)
150 + def get_device_capability(cls,
151 + device_id: int = 0
152 + ) -> Optional[DeviceCapability]:
153 + try:
154 + physical_device_id = device_id_to_physical_device_id(device_id)
155 + handle = pynvml.nvmlDeviceGetHandleByIndex(physical_device_id)
156 + major, minor = pynvml.nvmlDeviceGetCudaComputeCapability(handle)
157 + return DeviceCapability(major=major, minor=minor)
158 + except RuntimeError:
159 + return None
160 +
161 + @classmethod
162 + @lru_cache(maxsize=8)
163 + @with_nvml_context
164 + def has_device_capability(
165 + cls,
166 + capability: Union[Tuple[int, int], int],
167 + device_id: int = 0,
168 + ) -> bool:
169 + try:
170 + return super().has_device_capability(capability, device_id)
171 + except RuntimeError:
172 + return False
173
174 @classmethod
175 @lru_cache(maxsize=8)