| 1 | # SPDX-License-Identifier: Apache-2.0 |
| 2 | # SPDX-FileCopyrightText: Copyright contributors to the vLLM project |
| 3 | |
| 4 | import json |
| 5 | import os |
| 6 | import platform |
| 7 | import subprocess |
| 8 | import sys |
| 9 | from dataclasses import dataclass |
| 10 | from functools import cache |
| 11 | |
| 12 | import psutil |
| 13 | import regex as re |
| 14 | |
| 15 | from vllm.logger import init_logger |
| 16 | |
| 17 | DEVICE_CONTROL_ENV_VAR = "CPU_VISIBLE_MEMORY_NODES" |
| 18 | |
| 19 | logger = init_logger(__name__) |
| 20 | |
| 21 | |
| 22 | @dataclass |
| 23 | class LogicalCPUInfo: |
| 24 | id: int = -1 |
| 25 | physical_core: int = -1 |
| 26 | numa_node: int = -1 |
| 27 | |
| 28 | @classmethod |
| 29 | def _int(cls, value: str) -> int: |
| 30 | try: |
| 31 | int_value = int(value) |
| 32 | except Exception: |
| 33 | int_value = -1 |
| 34 | return int_value |
| 35 | |
| 36 | @staticmethod |
| 37 | def json_decoder(obj_dict: dict): |
| 38 | id = obj_dict.get("cpu") |
| 39 | physical_core = obj_dict.get("core") |
| 40 | numa_node = obj_dict.get("node") |
| 41 | |
| 42 | if not (id is None or physical_core is None or numa_node is None): |
| 43 | return LogicalCPUInfo( |
| 44 | id=LogicalCPUInfo._int(id), |
| 45 | physical_core=LogicalCPUInfo._int(physical_core), |
| 46 | numa_node=LogicalCPUInfo._int(numa_node), |
| 47 | ) |
| 48 | else: |
| 49 | return obj_dict |
| 50 | |
| 51 | |
| 52 | @dataclass |
| 53 | class MemoryNodeInfo: |
| 54 | total_memory: int = -1 |
| 55 | available_memory: int = -1 |
| 56 | |
| 57 | |
| 58 | def _read_int_file(path: str) -> int | None: |
| 59 | try: |
| 60 | with open(path) as f: |
| 61 | value = f.read().strip() |
| 62 | if not value or value == "max": |
| 63 | return None |
| 64 | return int(value) |
| 65 | except (OSError, ValueError): |
| 66 | return None |
| 67 | |
| 68 | |
| 69 | @cache |
| 70 | def get_cgroup_memory_limit() -> int | None: |
| 71 | """Return the cgroup memory limit in bytes, or None. |
| 72 | |
| 73 | Supports both cgroup v2 (unified) and v1. Returns None when |
| 74 | not running under a constrained cgroup (e.g. bare metal, or limit |
| 75 | reported as `max`/an unrealistically large value). |
| 76 | """ |
| 77 | if sys.platform != "linux": |
| 78 | return None |
| 79 | |
| 80 | # cgroup v2 unified hierarchy |
| 81 | v2_limit = _read_int_file("/sys/fs/cgroup/memory.max") |
| 82 | if v2_limit is not None: |
| 83 | return v2_limit |
| 84 | |
| 85 | # cgroup v1 |
| 86 | v1_limit = _read_int_file("/sys/fs/cgroup/memory/memory.limit_in_bytes") |
| 87 | if v1_limit is not None: |
| 88 | # cgroup v1 reports a huge sentinel (close to PAGE_COUNTER_MAX) |
| 89 | # when unlimited. Treat absurdly large values as "no limit". |
| 90 | if v1_limit >= (1 << 62): |
| 91 | return None |
| 92 | return v1_limit |
| 93 | |
| 94 | return None |
| 95 | |
| 96 | |
| 97 | def get_cgroup_memory_usage() -> int | None: |
| 98 | """Return the current cgroup memory usage in bytes, or None. |
| 99 | |
| 100 | The usage value is intentionally read on every call because cgroup |
| 101 | memory usage changes while the process is running. |
| 102 | """ |
| 103 | if sys.platform != "linux": |
| 104 | return None |
| 105 | |
| 106 | # cgroup v2 unified hierarchy |
| 107 | if _read_int_file("/sys/fs/cgroup/memory.max") is not None: |
| 108 | return _read_int_file("/sys/fs/cgroup/memory.current") |
| 109 | |
| 110 | # cgroup v1 |
| 111 | v1_limit = _read_int_file("/sys/fs/cgroup/memory/memory.limit_in_bytes") |
| 112 | if v1_limit is not None and v1_limit < (1 << 62): |
| 113 | return _read_int_file("/sys/fs/cgroup/memory/memory.usage_in_bytes") |
| 114 | |
| 115 | return None |
| 116 | |
| 117 | |
| 118 | def check_cgroup_memory_available( |
| 119 | required_bytes: int, |
| 120 | allocation_name: str, |
| 121 | ) -> None: |
| 122 | """Log cgroup memory headroom for an upcoming allocation. |
| 123 | |
| 124 | Args: |
| 125 | required_bytes: Bytes required by the allocation. |
| 126 | allocation_name: Human-readable name used in log messages. |
| 127 | |
| 128 | Low headroom logs a warning, but does not reject the allocation because |
| 129 | cgroup usage can include reclaimable memory. If the cgroup limit or usage |
| 130 | cannot be read, the check is skipped. |
| 131 | |
| 132 | """ |
| 133 | cgroup_limit = get_cgroup_memory_limit() |
| 134 | cgroup_usage = get_cgroup_memory_usage() |
| 135 | if cgroup_limit is None or cgroup_usage is None: |
| 136 | return |
| 137 | |
| 138 | cgroup_available = max(0, cgroup_limit - cgroup_usage) |
| 139 | mib = 1 << 20 |
| 140 | remaining_bytes = cgroup_available - required_bytes |
| 141 | log_fn = logger.debug if remaining_bytes >= 0 else logger.warning |
| 142 | status = ( |
| 143 | "current headroom meets the requested allocation" |
| 144 | if remaining_bytes >= 0 |
| 145 | else "current headroom is below the requested allocation; allocation " |
| 146 | "will still be attempted because cgroup usage may be reclaimable" |
| 147 | ) |
| 148 | log_fn( |
| 149 | "Cgroup memory preflight for %s: %.0f MiB required, %.0f MiB current " |
| 150 | "usage, %.0f MiB available under %.0f MiB limit, %.0f MiB remaining " |
| 151 | "after allocation based on current usage; %s.", |
| 152 | allocation_name, |
| 153 | required_bytes / mib, |
| 154 | cgroup_usage / mib, |
| 155 | cgroup_available / mib, |
| 156 | cgroup_limit / mib, |
| 157 | remaining_bytes / mib, |
| 158 | status, |
| 159 | ) |
| 160 | |
| 161 | |
| 162 | def get_memory_affinity(pid: int = 0) -> list[int]: |
| 163 | pid = os.getpid() if pid == 0 else pid |
| 164 | path = f"/proc/{pid}/status" |
| 165 | with open(path) as f: |
| 166 | for line in f: |
| 167 | if line.startswith("Mems_allowed_list:"): |
| 168 | # Extract the string part (e.g., "0-1,3") |
| 169 | raw_list = line.split(":")[1].strip() |
| 170 | return parse_id_list(raw_list) |
| 171 | return [] |
| 172 | |
| 173 | |
| 174 | def parse_id_list(raw_str: str) -> list[int]: |
| 175 | """Parses strings like '0-2,4,7-8' into [0, 1, 2, 4, 7, 8].""" |
| 176 | result: list[int] = [] |
| 177 | if not raw_str: |
| 178 | return result |
| 179 | |
| 180 | for part in raw_str.split(","): |
| 181 | if "-" in part: |
| 182 | start, end = map(int, part.split("-")) |
| 183 | result.extend(range(start, end + 1)) |
| 184 | else: |
| 185 | result.append(int(part)) |
| 186 | return sorted(list(set(result))) |
| 187 | |
| 188 | |
| 189 | def get_memory_node_info(node_id: int = 0) -> MemoryNodeInfo: |
| 190 | if sys.platform == "darwin": |
| 191 | # MacOS has no memory node |
| 192 | return MemoryNodeInfo( |
| 193 | total_memory=psutil.virtual_memory().total, |
| 194 | available_memory=psutil.virtual_memory().available, |
| 195 | ) |
| 196 | |
| 197 | meminfo_path = f"/sys/devices/system/node/node{node_id}/meminfo" |
| 198 | if not os.path.exists(meminfo_path): |
| 199 | # Non-NUMA systems (e.g. many RISC-V boards) don't expose per-node |
| 200 | # meminfo. Fall back to system-wide numbers from psutil. |
| 201 | vm = psutil.virtual_memory() |
| 202 | return MemoryNodeInfo( |
| 203 | total_memory=vm.total, |
| 204 | available_memory=vm.available, |
| 205 | ) |
| 206 | |
| 207 | meminfo = {} |
| 208 | with open(meminfo_path) as f: |
| 209 | for line in f: |
| 210 | # Each line looks like: "Node 0 MemTotal: 97421888 kB" |
| 211 | parts = line.split() |
| 212 | key = parts[2].rstrip(":") |
| 213 | # convert to Bytes |
| 214 | value = int(parts[3]) * 1024 |
| 215 | meminfo[key] = value |
| 216 | |
| 217 | total_memory = meminfo["MemTotal"] |
| 218 | free_memory = meminfo["MemFree"] |
| 219 | active_file_memory = meminfo["Active(file)"] |
| 220 | inactive_file_memory = meminfo["Inactive(file)"] |
| 221 | reclaimable_memory = meminfo["SReclaimable"] |
| 222 | available_memory = ( |
| 223 | free_memory + active_file_memory + inactive_file_memory + reclaimable_memory |
| 224 | ) |
| 225 | |
| 226 | # Honor cgroup memory limit (containers / k8s pods). NUMA meminfo |
| 227 | # reflects host-wide numbers; without this, gpu_memory_utilization |
| 228 | # would be applied to host RAM instead of the pod's limit. cgroup |
| 229 | # does not expose per-NUMA-node limits, so we just clamp the totals |
| 230 | # against the pod-wide limit here. |
| 231 | cgroup_limit = get_cgroup_memory_limit() |
| 232 | cgroup_usage = get_cgroup_memory_usage() |
| 233 | if cgroup_limit is not None and cgroup_limit < total_memory: |
| 234 | total_memory = cgroup_limit |
| 235 | cgroup_available = cgroup_limit - (cgroup_usage or 0) |
| 236 | available_memory = max(0, min(available_memory, cgroup_available)) |
| 237 | |
| 238 | return MemoryNodeInfo( |
| 239 | total_memory=total_memory, |
| 240 | available_memory=available_memory, |
| 241 | ) |
| 242 | |
| 243 | |
| 244 | def get_allowed_cpu_list() -> list[LogicalCPUInfo]: |
| 245 | cpu_list = _get_cpu_list() |
| 246 | if sys.platform == "linux": |
| 247 | allowed = os.sched_getaffinity(0) |
| 248 | return [x for x in cpu_list if x.id in allowed] |
| 249 | return cpu_list |
| 250 | |
| 251 | |
| 252 | def get_visible_memory_node() -> list[int]: |
| 253 | if sys.platform == "darwin": |
| 254 | return [0] |
| 255 | |
| 256 | allowed_memory_node_list = get_memory_affinity() |
| 257 | |
| 258 | env_key = DEVICE_CONTROL_ENV_VAR |
| 259 | if ( |
| 260 | ("VLLM_CPU_SIM_MULTI_NUMA" not in os.environ) |
| 261 | and env_key in os.environ |
| 262 | and os.environ[env_key] != "" |
| 263 | ): |
| 264 | visible_nodes = [int(s) for s in os.environ[env_key].split(",")] |
| 265 | visible_nodes = [ |
| 266 | node for node in visible_nodes if node in allowed_memory_node_list |
| 267 | ] |
| 268 | return visible_nodes |
| 269 | |
| 270 | return allowed_memory_node_list |
| 271 | |
| 272 | |
| 273 | @cache |
| 274 | def _synthesize_cpu_list() -> list[LogicalCPUInfo]: |
| 275 | """Synthesize a flat CPU list: each logical CPU is its own core on |
| 276 | NUMA node 0. Used when lscpu output is unavailable or unparsable |
| 277 | (e.g. macOS, RISC-V).""" |
| 278 | cpu_count = os.cpu_count() |
| 279 | assert cpu_count |
| 280 | return [LogicalCPUInfo(i, i, 0) for i in range(cpu_count)] |
| 281 | |
| 282 | |
| 283 | def _get_cpu_list() -> list[LogicalCPUInfo]: |
| 284 | if sys.platform == "darwin": |
| 285 | # For MacOS, no user-level CPU affinity and SMT, return all CPUs |
| 286 | return _synthesize_cpu_list() |
| 287 | |
| 288 | if platform.machine() == "s390x": |
| 289 | lscpu_output = subprocess.check_output( |
| 290 | "lscpu -J -e=CPU,CORE,NODE,SOCKET,BOOK", shell=True, text=True |
| 291 | ) |
| 292 | else: |
| 293 | lscpu_output = subprocess.check_output( |
| 294 | "lscpu --json --extended=CPU,CORE,NODE --online", shell=True, text=True |
| 295 | ) |
| 296 | |
| 297 | # For platforms without NUMA, map bare `-` node to 0 so non-NUMA |
| 298 | # systems keep the existing behavior from #39781. |
| 299 | lscpu_output = re.sub(r'"node":\s*-\s*(,|\n|\})', r'"node": 0\1', lscpu_output) |
| 300 | |
| 301 | # On some architectures (notably RISC-V), lscpu also emits bare `-` |
| 302 | # for cpu/core. Quote them so the JSON parses; they will decode to |
| 303 | # -1 and be filtered out below, triggering the synthesized fallback. |
| 304 | lscpu_output = re.sub( |
| 305 | r'("(?:cpu|core)":\s*)-\s*(,|\n|\})', |
| 306 | r'\1"-"\2', |
| 307 | lscpu_output, |
| 308 | ) |
| 309 | |
| 310 | if platform.machine() == "s390x": |
| 311 | # On s390x, use the best topology level as the abstract CPU group |
| 312 | # key via numa_node. The hierarchy is book > socket > core; prefer |
| 313 | # book, fall back to socket if only one book exists. |
| 314 | lscpu_output = re.sub(r'"book":\s*-\s*(,|\n|\})', r'"book": 0\1', lscpu_output) |
| 315 | lscpu_output = re.sub( |
| 316 | r'"socket":\s*-\s*(,|\n|\})', r'"socket": 0\1', lscpu_output |
| 317 | ) |
| 318 | |
| 319 | raw_cpus = json.loads(lscpu_output)["cpus"] |
| 320 | book_values = set() |
| 321 | socket_values = set() |
| 322 | for entry in raw_cpus: |
| 323 | book_values.add(LogicalCPUInfo._int(str(entry.get("book", "-1")))) |
| 324 | socket_values.add(LogicalCPUInfo._int(str(entry.get("socket", "-1")))) |
| 325 | |
| 326 | # Pick the best partitioning level: book first, then socket |
| 327 | if len(book_values - {-1}) > 1: |
| 328 | group_key = "book" |
| 329 | elif len(socket_values - {-1}) > 1: |
| 330 | group_key = "socket" |
| 331 | else: |
| 332 | group_key = None |
| 333 | |
| 334 | if group_key is not None: |
| 335 | logical_book_socket_list: list[LogicalCPUInfo] = [] |
| 336 | for entry in raw_cpus: |
| 337 | cpu_id = LogicalCPUInfo._int(str(entry.get("cpu", "-1"))) |
| 338 | core = LogicalCPUInfo._int(str(entry.get("core", "-1"))) |
| 339 | group = LogicalCPUInfo._int(str(entry.get(group_key, "-1"))) |
| 340 | if -1 not in (cpu_id, core, group): |
| 341 | logical_book_socket_list.append( |
| 342 | LogicalCPUInfo(id=cpu_id, physical_core=core, numa_node=group) |
| 343 | ) |
| 344 | |
| 345 | if not logical_book_socket_list: |
| 346 | return _synthesize_cpu_list() |
| 347 | return logical_book_socket_list |
| 348 | |
| 349 | logical_cpu_list: list[LogicalCPUInfo] = json.loads( |
| 350 | lscpu_output, object_hook=LogicalCPUInfo.json_decoder |
| 351 | )["cpus"] |
| 352 | |
| 353 | # Filter CPUs with invalid attributes (only require id, core, node) |
| 354 | logical_cpu_list = [ |
| 355 | x for x in logical_cpu_list if -1 not in (x.id, x.physical_core, x.numa_node) |
| 356 | ] |
| 357 | |
| 358 | # If lscpu returned no valid entries (e.g. RISC-V where all fields |
| 359 | # are bare `-`), fall back to synthesized topology. |
| 360 | if not logical_cpu_list: |
| 361 | logical_cpu_list = _synthesize_cpu_list() |
| 362 | |
| 363 | return logical_cpu_list |