A high-throughput and memory-efficient inference and serving engine for LLMs
Mirrored from https://github.com/tduncan-lum/Nvidia-Optimization-Stack---VLLM.git
Synced Nov 12, 02:15 UTC
[Bugfix][CPU] Fix CPU KV cache fallback memory allocation (#29604)
Signed-off-by: Gauri Sahnan <gauri.sahnan@arm.com> Co-authored-by: Li, Jiang <jiang1.li@intel.com>
gausah01 committed
Dec 4, 2025 at 05:01 UTC
28097d5638cc695f4644c411edac8eb05a03b39b
1 file changed
+14
-3
vllm/platforms/cpu.py
+14
-3
@@ -10,6 +10,7 @@ import sys
10
from dataclasses import dataclass
11
from typing import TYPE_CHECKING
12
13
+import psutil
14
import regex as re
15
import torch
16
@@ -147,11 +148,21 @@ class CpuPlatform(Platform):
148
from vllm.utils.mem_constants import GiB_bytes
149
150
kv_cache_space = envs.VLLM_CPU_KVCACHE_SPACE
151
+ node_dir = "/sys/devices/system/node"
152
if kv_cache_space is None:
151
- kv_cache_space = 4 * GiB_bytes # type: ignore
153
+ nodes = (
154
+ [d for d in os.listdir(node_dir) if d.startswith("node")]
155
+ if os.path.exists(node_dir)
156
+ else []
157
+ )
158
+ num_numa_nodes = len(nodes) or 1
159
+ free_cpu_memory = psutil.virtual_memory().total // num_numa_nodes
160
+ DEFAULT_CPU_MEM_UTILIZATION = 0.5
161
+ kv_cache_space = int(free_cpu_memory * DEFAULT_CPU_MEM_UTILIZATION)
162
+ kv_cache_space_gib = kv_cache_space / GiB_bytes
163
logger.warning_once(
153
- "Environment variable VLLM_CPU_KVCACHE_SPACE (GiB) "
154
- "for CPU backend is not set, using 4 by default."
164
+ "VLLM_CPU_KVCACHE_SPACE not set. Using "
165
+ f"{kv_cache_space_gib:.2f} GiB for KV cache."
166
)
167
else:
168
kv_cache_space *= GiB_bytes