A high-throughput and memory-efficient inference and serving engine for LLMs
Mirrored from https://github.com/tduncan-lum/Nvidia-Optimization-Stack---VLLM.git
Synced Nov 16, 00:45 UTC
[cuda] manually import the correct pynvml module (#12679)
fixes problems like https://github.com/vllm-project/vllm/pull/12635 and https://github.com/vllm-project/vllm/pull/12636 and https://github.com/vllm-project/vllm/pull/12565 --------- Signed-off-by: youkaichao <youkaichao@gmail.com>
youkaichao committed
Feb 3, 2025 at 15:58 UTC
ad4a9dc817f00c266d1ca210342d1865aa69db27
3 files changed
+56
-9
vllm/platforms/__init__.py
+2
-1
@@ -33,7 +33,8 @@ def cuda_platform_plugin() -> Optional[str]:
33
is_cuda = False
34
35
try:
36
- import pynvml
36
+ from vllm.utils import import_pynvml
37
+ pynvml = import_pynvml()
38
pynvml.nvmlInit()
39
try:
40
if pynvml.nvmlDeviceGetCount() > 0:
vllm/platforms/cuda.py
+2
-8
@@ -8,7 +8,6 @@ from functools import lru_cache, wraps
8
from typing import (TYPE_CHECKING, Callable, List, Optional, Tuple, TypeVar,
9
Union)
10
11
-import pynvml
11
import torch
12
from typing_extensions import ParamSpec
13
@@ -16,6 +15,7 @@ from typing_extensions import ParamSpec
15
import vllm._C # noqa
16
import vllm.envs as envs
17
from vllm.logger import init_logger
18
+from vllm.utils import import_pynvml
19
20
from .interface import DeviceCapability, Platform, PlatformEnum, _Backend
21
@@ -29,13 +29,7 @@ logger = init_logger(__name__)
29
_P = ParamSpec("_P")
30
_R = TypeVar("_R")
31
32
-if pynvml.__file__.endswith("__init__.py"):
33
- logger.warning(
34
- "You are using a deprecated `pynvml` package. Please install"
35
- " `nvidia-ml-py` instead, and make sure to uninstall `pynvml`."
36
- " When both of them are installed, `pynvml` will take precedence"
37
- " and cause errors. See https://pypi.org/project/pynvml "
38
- "for more information.")
32
+pynvml = import_pynvml()
33
34
# pytorch 2.5 uses cudnn sdpa by default, which will cause crash on some models
35
# see https://github.com/huggingface/diffusers/issues/9704 for details
vllm/utils.py
+52
@@ -2208,3 +2208,55 @@ def run_method(obj: Any, method: Union[str, bytes, Callable], args: Tuple[Any],
2208
else:
2209
func = partial(method, obj) # type: ignore
2210
return func(*args, **kwargs)
2211
+
2212
+
2213
+def import_pynvml():
2214
+ """
2215
+ Historical comments:
2216
+
2217
+ libnvml.so is the library behind nvidia-smi, and
2218
+ pynvml is a Python wrapper around it. We use it to get GPU
2219
+ status without initializing CUDA context in the current process.
2220
+ Historically, there are two packages that provide pynvml:
2221
+ - `nvidia-ml-py` (https://pypi.org/project/nvidia-ml-py/): The official
2222
+ wrapper. It is a dependency of vLLM, and is installed when users
2223
+ install vLLM. It provides a Python module named `pynvml`.
2224
+ - `pynvml` (https://pypi.org/project/pynvml/): An unofficial wrapper.
2225
+ Prior to version 12.0, it also provides a Python module `pynvml`,
2226
+ and therefore conflicts with the official one. What's worse,
2227
+ the module is a Python package, and has higher priority than
2228
+ the official one which is a standalone Python file.
2229
+ This causes errors when both of them are installed.
2230
+ Starting from version 12.0, it migrates to a new module
2231
+ named `pynvml_utils` to avoid the conflict.
2232
+
2233
+ TL;DR: if users have pynvml<12.0 installed, it will cause problems.
2234
+ Otherwise, `import pynvml` will import the correct module.
2235
+ We take the safest approach here, to manually import the correct
2236
+ `pynvml.py` module from the `nvidia-ml-py` package.
2237
+ """
2238
+ if TYPE_CHECKING:
2239
+ import pynvml
2240
+ return pynvml
2241
+ if "pynvml" in sys.modules:
2242
+ import pynvml
2243
+ if pynvml.__file__.endswith("__init__.py"):
2244
+ # this is pynvml < 12.0
2245
+ raise RuntimeError(
2246
+ "You are using a deprecated `pynvml` package. "
2247
+ "Please uninstall `pynvml` or upgrade to at least"
2248
+ " version 12.0. See https://pypi.org/project/pynvml "
2249
+ "for more information.")
2250
+ return sys.modules["pynvml"]
2251
+ import importlib.util
2252
+ import os
2253
+ import site
2254
+ for site_dir in site.getsitepackages():
2255
+ pynvml_path = os.path.join(site_dir, "pynvml.py")
2256
+ if os.path.exists(pynvml_path):
2257
+ spec = importlib.util.spec_from_file_location(
2258
+ "pynvml", pynvml_path)
2259
+ pynvml = importlib.util.module_from_spec(spec)
2260
+ sys.modules["pynvml"] = pynvml
2261
+ spec.loader.exec_module(pynvml)
2262
+ return pynvml