A high-throughput and memory-efficient inference and serving engine for LLMs
Mirrored from https://github.com/tduncan-lum/Nvidia-Optimization-Stack---VLLM.git
Synced Nov 15, 07:15 UTC
[Bugfix] fix v1 cpu worker fails on macOS (#19121)
Kebe committed
Jun 5, 2025 at 04:17 UTC
ef3f98b59fb971291a8cf3d0530ab7822f39d959
2 files changed
+14
-3
vllm/engine/arg_utils.py
+3
-2
@@ -1290,7 +1290,7 @@ class EngineArgs:
1290
# Skip this check if we are running on a non-GPU platform,
1291
# or if the device capability is not available
1292
# (e.g. in a Ray actor without GPUs).
1293
- from vllm.platforms import current_platform
1293
+ from vllm.platforms import CpuArchEnum, current_platform
1294
if (current_platform.is_cuda()
1295
and current_platform.get_device_capability()
1296
and current_platform.get_device_capability().major < 8):
@@ -1434,7 +1434,8 @@ class EngineArgs:
1434
# Non-[CUDA, TPU] may be supported on V1, but off by default for now.
1435
v0_hardware = not any(
1436
(current_platform.is_cuda(), current_platform.is_tpu(),
1437
- current_platform.is_cpu()))
1437
+ (current_platform.is_cpu()
1438
+ and current_platform.get_cpu_architecture() == CpuArchEnum.X86)))
1439
if v0_hardware and _warn_or_fallback( # noqa: SIM103
1440
current_platform.device_name):
1441
return False
vllm/platforms/cpu.py
+11
-1
@@ -2,6 +2,7 @@
2
# SPDX-FileCopyrightText: Copyright contributors to the vLLM project
3
4
import os
5
+import platform
6
import sys
7
from importlib.util import find_spec
8
from typing import TYPE_CHECKING, Optional
@@ -22,6 +23,15 @@ else:
23
VllmConfig = None
24
25
26
+def get_max_threads(pid=0):
27
+ if hasattr(os, 'sched_getaffinity'):
28
+ return len(os.sched_getaffinity(pid))
29
+ elif platform.system() == 'Darwin':
30
+ return os.cpu_count()
31
+ else:
32
+ raise NotImplementedError("Unsupported OS")
33
+
34
+
35
class CpuPlatform(Platform):
36
_enum = PlatformEnum.CPU
37
device_name: str = "cpu"
@@ -190,7 +200,7 @@ class CpuPlatform(Platform):
200
201
# Note: to avoid the error 'nthreads cannot be larger than environment
202
# variable "NUMEXPR_MAX_THREADS" (64)'.
193
- os.environ["NUMEXPR_MAX_THREADS"] = str(len(os.sched_getaffinity(0)))
203
+ os.environ["NUMEXPR_MAX_THREADS"] = str(get_max_threads())
204
205
# Set default threads num for OpenMP parallel
206
os.environ["OMP_NUM_THREADS"] = str(torch.get_num_threads())