A high-throughput and memory-efficient inference and serving engine for LLMs
Mirrored from https://github.com/tduncan-lum/Nvidia-Optimization-Stack---VLLM.git
Synced Nov 15, 23:30 UTC
[Build] Make pypi install work on CPU platform (#12874)
wangxiyuan committed
Feb 8, 2025 at 17:15 UTC
407b5537db02da122abf863673fc6cb76795e8bd
1 file changed
+7
-2
setup.py
+7
-2
@@ -47,6 +47,11 @@ elif not (sys.platform.startswith("linux")
47
"Building on %s, "
48
"so vLLM may not be able to run correctly", sys.platform)
49
VLLM_TARGET_DEVICE = "empty"
50
+elif (sys.platform.startswith("linux") and torch.version.cuda is None
51
+ and os.getenv("VLLM_TARGET_DEVICE") is None):
52
+ # if cuda is not available and VLLM_TARGET_DEVICE is not set,
53
+ # fallback to cpu
54
+ VLLM_TARGET_DEVICE = "cpu"
55
56
MAIN_CUDA_VERSION = "12.1"
57
@@ -482,7 +487,6 @@ def get_vllm_version() -> str:
487
version = get_version(
488
write_to="vllm/_version.py", # TODO: move this to pyproject.toml
489
)
485
-
490
sep = "+" if "+" not in version else "." # dev versions might contain +
491
492
if _no_device():
@@ -520,7 +524,8 @@ def get_vllm_version() -> str:
524
elif _is_tpu():
525
version += f"{sep}tpu"
526
elif _is_cpu():
523
- version += f"{sep}cpu"
527
+ if envs.VLLM_TARGET_DEVICE == "cpu":
528
+ version += f"{sep}cpu"
529
elif _is_xpu():
530
version += f"{sep}xpu"
531
else: