A high-throughput and memory-efficient inference and serving engine for LLMs
[Bugfix][CPU] Honor cgroup memory limit when computing KV cache size (#45086)
[Bugfix][CPU] Honor cgroup memory limit when computing KV cache size (#45086) Signed-off-by: baoloongmao <baoloongmao@tencent.com> Co-authored-by: Li, Jiang <jiang1.li@intel.com>
maobaolong committed
Jun 15, 2026 at 10:26 UTC
b675cb7d0fbf52cdf768df756ab4d6a576f7f756
1 file changed
+52
vllm/utils/cpu_resource_utils.py
+52
@@ -50,6 +50,47 @@ class MemoryNodeInfo:
50
available_memory: int = -1
51
52
53
+def _read_int_file(path: str) -> int | None:
54
+ try:
55
+ with open(path) as f:
56
+ value = f.read().strip()
57
+ if not value or value == "max":
58
+ return None
59
+ return int(value)
60
+ except (OSError, ValueError):
61
+ return None
62
+
63
+
64
+@cache
65
+def get_cgroup_memory_limit() -> tuple[int | None, int | None]:
66
+ """Return (limit, usage) in bytes from cgroup, or (None, None).
67
+
68
+ Supports both cgroup v2 (unified) and v1. Returns (None, None) when
69
+ not running under a constrained cgroup (e.g. bare metal, or limit
70
+ reported as `max`/an unrealistically large value).
71
+ """
72
+ if sys.platform != "linux":
73
+ return None, None
74
+
75
+ # cgroup v2 unified hierarchy
76
+ v2_limit = _read_int_file("/sys/fs/cgroup/memory.max")
77
+ if v2_limit is not None:
78
+ v2_usage = _read_int_file("/sys/fs/cgroup/memory.current")
79
+ return v2_limit, v2_usage
80
+
81
+ # cgroup v1
82
+ v1_limit = _read_int_file("/sys/fs/cgroup/memory/memory.limit_in_bytes")
83
+ if v1_limit is not None:
84
+ # cgroup v1 reports a huge sentinel (close to PAGE_COUNTER_MAX)
85
+ # when unlimited. Treat absurdly large values as "no limit".
86
+ if v1_limit >= (1 << 62):
87
+ return None, None
88
+ v1_usage = _read_int_file("/sys/fs/cgroup/memory/memory.usage_in_bytes")
89
+ return v1_limit, v1_usage
90
+
91
+ return None, None
92
+
93
+
94
def get_memory_affinity(pid: int = 0) -> list[int]:
95
pid = os.getpid() if pid == 0 else pid
96
path = f"/proc/{pid}/status"
@@ -114,6 +155,17 @@ def get_memory_node_info(node_id: int = 0) -> MemoryNodeInfo:
155
free_memory + active_file_memory + inactive_file_memory + reclaimable_memory
156
)
157
158
+ # Honor cgroup memory limit (containers / k8s pods). NUMA meminfo
159
+ # reflects host-wide numbers; without this, gpu_memory_utilization
160
+ # would be applied to host RAM instead of the pod's limit. cgroup
161
+ # does not expose per-NUMA-node limits, so we just clamp the totals
162
+ # against the pod-wide limit here.
163
+ cgroup_limit, cgroup_usage = get_cgroup_memory_limit()
164
+ if cgroup_limit is not None and cgroup_limit < total_memory:
165
+ total_memory = cgroup_limit
166
+ cgroup_available = cgroup_limit - (cgroup_usage or 0)
167
+ available_memory = max(0, min(available_memory, cgroup_available))
168
+
169
return MemoryNodeInfo(
170
total_memory=total_memory,
171
available_memory=available_memory,