A high-throughput and memory-efficient inference and serving engine for LLMs
Mirrored from https://github.com/tduncan-lum/Nvidia-Optimization-Stack---VLLM.git
Synced Nov 15, 21:00 UTC
[platform] add debug logging during inferring the device type (#14195)
Signed-off-by: youkaichao <youkaichao@gmail.com>
youkaichao committed
Mar 4, 2025 at 18:39 UTC
ac65bc92dfedc6218d3b389e7cc2947faf77d902
1 file changed
+55
-9
vllm/platforms/__init__.py
+55
-9
@@ -32,6 +32,7 @@ def vllm_version_matches_substr(substr: str) -> bool:
32
33
def tpu_platform_plugin() -> Optional[str]:
34
is_tpu = False
35
+ logger.debug("Checking if TPU platform is available.")
36
try:
37
# While it's technically possible to install libtpu on a
38
# non-TPU machine, this is a very uncommon scenario. Therefore,
@@ -39,7 +40,9 @@ def tpu_platform_plugin() -> Optional[str]:
40
# has TPUs.
41
import libtpu # noqa: F401
42
is_tpu = True
42
- except Exception:
43
+ logger.debug("Confirmed TPU platform is available.")
44
+ except Exception as e:
45
+ logger.debug("TPU platform is not available because: %s", str(e))
46
pass
47
48
return "vllm.platforms.tpu.TpuPlatform" if is_tpu else None
@@ -47,7 +50,7 @@ def tpu_platform_plugin() -> Optional[str]:
50
51
def cuda_platform_plugin() -> Optional[str]:
52
is_cuda = False
50
-
53
+ logger.debug("Checking if CUDA platform is available.")
54
try:
55
from vllm.utils import import_pynvml
56
pynvml = import_pynvml()
@@ -60,9 +63,19 @@ def cuda_platform_plugin() -> Optional[str]:
63
# on a GPU machine, even if in a cpu build.
64
is_cuda = (pynvml.nvmlDeviceGetCount() > 0
65
and not vllm_version_matches_substr("cpu"))
66
+ if pynvml.nvmlDeviceGetCount() <= 0:
67
+ logger.debug(
68
+ "CUDA platform is not available because no GPU is found.")
69
+ if vllm_version_matches_substr("cpu"):
70
+ logger.debug("CUDA platform is not available because"
71
+ " vLLM is built with CPU.")
72
+ if is_cuda:
73
+ logger.debug("Confirmed CUDA platform is available.")
74
finally:
75
pynvml.nvmlShutdown()
76
except Exception as e:
77
+ logger.debug("Exception happens when checking CUDA platform: %s",
78
+ str(e))
79
if "nvml" not in e.__class__.__name__.lower():
80
# If the error is not related to NVML, re-raise it.
81
raise e
@@ -75,23 +88,28 @@ def cuda_platform_plugin() -> Optional[str]:
88
or os.path.exists("/sys/class/tegra-firmware")
89
90
if cuda_is_jetson():
91
+ logger.debug("Confirmed CUDA platform is available on Jetson.")
92
is_cuda = True
93
+ else:
94
+ logger.debug("CUDA platform is not available because: %s", str(e))
95
96
return "vllm.platforms.cuda.CudaPlatform" if is_cuda else None
97
98
99
def rocm_platform_plugin() -> Optional[str]:
100
is_rocm = False
85
-
101
+ logger.debug("Checking if ROCm platform is available.")
102
try:
103
import amdsmi
104
amdsmi.amdsmi_init()
105
try:
106
if len(amdsmi.amdsmi_get_processor_handles()) > 0:
107
is_rocm = True
108
+ logger.debug("Confirmed ROCm platform is available.")
109
finally:
110
amdsmi.amdsmi_shut_down()
94
- except Exception:
111
+ except Exception as e:
112
+ logger.debug("ROCm platform is not available because: %s", str(e))
113
pass
114
115
return "vllm.platforms.rocm.RocmPlatform" if is_rocm else None
@@ -99,10 +117,17 @@ def rocm_platform_plugin() -> Optional[str]:
117
118
def hpu_platform_plugin() -> Optional[str]:
119
is_hpu = False
120
+ logger.debug("Checking if HPU platform is available.")
121
try:
122
from importlib import util
123
is_hpu = util.find_spec('habana_frameworks') is not None
105
- except Exception:
124
+ if is_hpu:
125
+ logger.debug("Confirmed HPU platform is available.")
126
+ else:
127
+ logger.debug("HPU platform is not available because "
128
+ "habana_frameworks is not found.")
129
+ except Exception as e:
130
+ logger.debug("HPU platform is not available because: %s", str(e))
131
pass
132
133
return "vllm.platforms.hpu.HpuPlatform" if is_hpu else None
@@ -110,7 +135,7 @@ def hpu_platform_plugin() -> Optional[str]:
135
136
def xpu_platform_plugin() -> Optional[str]:
137
is_xpu = False
113
-
138
+ logger.debug("Checking if XPU platform is available.")
139
try:
140
# installed IPEX if the machine has XPUs.
141
import intel_extension_for_pytorch # noqa: F401
@@ -118,7 +143,9 @@ def xpu_platform_plugin() -> Optional[str]:
143
import torch
144
if hasattr(torch, 'xpu') and torch.xpu.is_available():
145
is_xpu = True
121
- except Exception:
146
+ logger.debug("Confirmed XPU platform is available.")
147
+ except Exception as e:
148
+ logger.debug("XPU platform is not available because: %s", str(e))
149
pass
150
151
return "vllm.platforms.xpu.XPUPlatform" if is_xpu else None
@@ -126,13 +153,21 @@ def xpu_platform_plugin() -> Optional[str]:
153
154
def cpu_platform_plugin() -> Optional[str]:
155
is_cpu = False
156
+ logger.debug("Checking if CPU platform is available.")
157
try:
158
is_cpu = vllm_version_matches_substr("cpu")
159
+ if is_cpu:
160
+ logger.debug("Confirmed CPU platform is available because"
161
+ " vLLM is built with CPU.")
162
if not is_cpu:
163
import platform
164
is_cpu = platform.machine().lower().startswith("arm")
165
+ if is_cpu:
166
+ logger.debug("Confirmed CPU platform is available"
167
+ " because the machine is ARM.")
168
135
- except Exception:
169
+ except Exception as e:
170
+ logger.debug("CPU platform is not available because: %s", str(e))
171
pass
172
173
return "vllm.platforms.cpu.CpuPlatform" if is_cpu else None
@@ -140,10 +175,14 @@ def cpu_platform_plugin() -> Optional[str]:
175
176
def neuron_platform_plugin() -> Optional[str]:
177
is_neuron = False
178
+ logger.debug("Checking if Neuron platform is available.")
179
try:
180
import transformers_neuronx # noqa: F401
181
is_neuron = True
146
- except ImportError:
182
+ logger.debug("Confirmed Neuron platform is available because"
183
+ " transformers_neuronx is found.")
184
+ except ImportError as e:
185
+ logger.debug("Neuron platform is not available because: %s", str(e))
186
pass
187
188
return "vllm.platforms.neuron.NeuronPlatform" if is_neuron else None
@@ -151,8 +190,15 @@ def neuron_platform_plugin() -> Optional[str]:
190
191
def openvino_platform_plugin() -> Optional[str]:
192
is_openvino = False
193
+ logger.debug("Checking if OpenVINO platform is available.")
194
with suppress(Exception):
195
is_openvino = vllm_version_matches_substr("openvino")
196
+ if is_openvino:
197
+ logger.debug("Confirmed OpenVINO platform is available"
198
+ " because vLLM is built with OpenVINO.")
199
+ if not is_openvino:
200
+ logger.debug("OpenVINO platform is not available because"
201
+ " vLLM is not built with OpenVINO.")
202
203
return "vllm.platforms.openvino.OpenVinoPlatform" if is_openvino else None
204