A high-throughput and memory-efficient inference and serving engine for LLMs
Mirrored from https://github.com/tduncan-lum/Nvidia-Optimization-Stack---VLLM.git
Synced Nov 15, 22:15 UTC
[Misc] Warn if the vLLM version can't be retrieved (#13501)
Signed-off-by: Alex-Brooks <Alex.brooks@ibm.com>
Alex Brooks committed
Feb 19, 2025 at 23:24 UTC
96216678744559ed10ed66975e618bbb5e74ffa3
1 file changed
+20
-10
vllm/platforms/__init__.py
+20
-10
@@ -2,6 +2,7 @@
2
3
import logging
4
import traceback
5
+from contextlib import suppress
6
from itertools import chain
7
from typing import TYPE_CHECKING, Optional
8
@@ -14,6 +15,21 @@ from .interface import CpuArchEnum, Platform, PlatformEnum
15
logger = logging.getLogger(__name__)
16
17
18
+def vllm_version_matches_substr(substr: str) -> bool:
19
+ """
20
+ Check to see if the vLLM version matches a substring.
21
+ """
22
+ from importlib.metadata import PackageNotFoundError, version
23
+ try:
24
+ vllm_version = version("vllm")
25
+ except PackageNotFoundError as e:
26
+ logger.warning(
27
+ "The vLLM package was not found, so its version could not be "
28
+ "inspected. This may cause platform detection to fail.")
29
+ raise e
30
+ return substr in vllm_version
31
+
32
+
33
def tpu_platform_plugin() -> Optional[str]:
34
is_tpu = False
35
try:
@@ -33,8 +49,6 @@ def cuda_platform_plugin() -> Optional[str]:
49
is_cuda = False
50
51
try:
36
- from importlib.metadata import version
37
-
52
from vllm.utils import import_pynvml
53
pynvml = import_pynvml()
54
pynvml.nvmlInit()
@@ -45,7 +59,7 @@ def cuda_platform_plugin() -> Optional[str]:
59
# Otherwise, vllm will always activate cuda plugin
60
# on a GPU machine, even if in a cpu build.
61
is_cuda = (pynvml.nvmlDeviceGetCount() > 0
48
- and "cpu" not in version("vllm"))
62
+ and not vllm_version_matches_substr("cpu"))
63
finally:
64
pynvml.nvmlShutdown()
65
except Exception as e:
@@ -113,8 +127,7 @@ def xpu_platform_plugin() -> Optional[str]:
127
def cpu_platform_plugin() -> Optional[str]:
128
is_cpu = False
129
try:
116
- from importlib.metadata import version
117
- is_cpu = "cpu" in version("vllm")
130
+ is_cpu = vllm_version_matches_substr("cpu")
131
if not is_cpu:
132
import platform
133
is_cpu = platform.machine().lower().startswith("arm")
@@ -138,11 +151,8 @@ def neuron_platform_plugin() -> Optional[str]:
151
152
def openvino_platform_plugin() -> Optional[str]:
153
is_openvino = False
141
- try:
142
- from importlib.metadata import version
143
- is_openvino = "openvino" in version("vllm")
144
- except Exception:
145
- pass
154
+ with suppress(Exception):
155
+ is_openvino = vllm_version_matches_substr("openvino")
156
157
return "vllm.platforms.openvino.OpenVinoPlatform" if is_openvino else None
158