A high-throughput and memory-efficient inference and serving engine for LLMs
Mirrored from https://github.com/tduncan-lum/Nvidia-Optimization-Stack---VLLM.git
Synced Nov 14, 05:30 UTC
[Bugfix] Fix `vllm bench ...` on CPU-only head nodes (#25283)
Signed-off-by: Aydin Abiar <aydin@anyscale.com> Signed-off-by: Harry Mellor <19981378+hmellor@users.noreply.github.com> Co-authored-by: Aydin Abiar <aydin@anyscale.com> Co-authored-by: Harry Mellor <19981378+hmellor@users.noreply.github.com> Co-authored-by: Wentao Ye <44945378+yewentao256@users.noreply.github.com>
Aydin Abiar committed
Oct 8, 2025 at 09:06 UTC
76afe4edf83ae5925b00f9eabc48355be6157dc2
3 files changed
+34
vllm/entrypoints/cli/main.py
+21
@@ -8,6 +8,11 @@ to avoid certain eager import breakage."""
8
from __future__ import annotations
9
10
import importlib.metadata
11
+import sys
12
+
13
+from vllm.logger import init_logger
14
+
15
+logger = init_logger(__name__)
16
17
18
def main():
@@ -29,6 +34,22 @@ def main():
34
35
cli_env_setup()
36
37
+ # For 'vllm bench *': use CPU instead of UnspecifiedPlatform by default
38
+ if len(sys.argv) > 1 and sys.argv[1] == "bench":
39
+ logger.debug(
40
+ "Bench command detected, must ensure current platform is not "
41
+ "UnspecifiedPlatform to avoid device type inference error"
42
+ )
43
+ from vllm import platforms
44
+
45
+ if platforms.current_platform.is_unspecified():
46
+ from vllm.platforms.cpu import CpuPlatform
47
+
48
+ platforms.current_platform = CpuPlatform()
49
+ logger.info(
50
+ "Unspecified platform detected, switching to CPU Platform instead."
51
+ )
52
+
53
parser = FlexibleArgumentParser(
54
description="vLLM CLI",
55
epilog=VLLM_SUBCMD_PARSER_EPILOG.format(subcmd="[subcommand]"),
vllm/platforms/__init__.py
+10
@@ -261,4 +261,14 @@ def __getattr__(name: str):
261
raise AttributeError(f"No attribute named '{name}' exists in {__name__}.")
262
263
264
+def __setattr__(name: str, value):
265
+ if name == "current_platform":
266
+ global _current_platform
267
+ _current_platform = value
268
+ elif name in globals():
269
+ globals()[name] = value
270
+ else:
271
+ raise AttributeError(f"No attribute named '{name}' exists in {__name__}.")
272
+
273
+
274
__all__ = ["Platform", "PlatformEnum", "current_platform", "CpuArchEnum", "_init_trace"]
vllm/platforms/interface.py
+3
@@ -141,6 +141,9 @@ class Platform:
141
def is_out_of_tree(self) -> bool:
142
return self._enum == PlatformEnum.OOT
143
144
+ def is_unspecified(self) -> bool:
145
+ return self._enum == PlatformEnum.UNSPECIFIED
146
+
147
def get_max_output_tokens(self, prompt_len: int) -> int:
148
return sys.maxsize
149