[Core] Fail fast when /dev/shm is too small for the shm ring buffer (#48879)
Signed-off-by: Dr Andrea Tassi <andrea@verticular.uk> Co-authored-by: Dr Andrea Tassi <andrea@verticular.uk> Co-authored-by: Claude Opus 4.8 <noreply@anthropic.com>
Andrea Tassi committed
Jul 27, 2026 at 20:48 UTC
831d3848f16e21e652bf1c377d010c2b5fe76f37
2 files changed
+72
-1
tests/distributed/test_shm_broadcast.py
+43
-1
@@ -4,6 +4,7 @@
4
import random
5
import threading
6
import time
7
+from types import SimpleNamespace
8
from unittest import mock
9
10
import multiprocess as mp
@@ -11,7 +12,12 @@ import numpy as np
12
import pytest
13
import torch.distributed as dist
14
14
-from vllm.distributed.device_communicators.shm_broadcast import MessageQueue
15
+from vllm.distributed.device_communicators import shm_broadcast
16
+from vllm.distributed.device_communicators.shm_broadcast import (
17
+ MessageQueue,
18
+ ShmRingBuffer,
19
+ check_shm_free_space,
20
+)
21
from vllm.distributed.utils import StatelessProcessGroup
22
from vllm.utils.network_utils import get_open_port
23
from vllm.utils.system_utils import update_environment_variables
@@ -522,3 +528,39 @@ def test_warning_logs(caplog_vllm):
528
# Clean up when done
529
writer.shutdown()
530
reader.shutdown()
531
+
532
+
533
+def _fake_disk_usage(free_bytes: int):
534
+ return SimpleNamespace(total=free_bytes, used=0, free=free_bytes)
535
+
536
+
537
+def test_check_shm_free_space_raises_when_insufficient(tmp_path):
538
+ with (
539
+ mock.patch.object(
540
+ shm_broadcast.shutil, "disk_usage", return_value=_fake_disk_usage(32 << 20)
541
+ ),
542
+ pytest.raises(RuntimeError, match="Insufficient space"),
543
+ ):
544
+ check_shm_free_space(240 << 20, shm_path=str(tmp_path))
545
+
546
+
547
+def test_check_shm_free_space_passes_when_sufficient(tmp_path):
548
+ with mock.patch.object(
549
+ shm_broadcast.shutil, "disk_usage", return_value=_fake_disk_usage(512 << 20)
550
+ ):
551
+ check_shm_free_space(240 << 20, shm_path=str(tmp_path))
552
+
553
+
554
+def test_check_shm_free_space_skipped_when_path_missing(tmp_path):
555
+ check_shm_free_space(1 << 60, shm_path=str(tmp_path / "does-not-exist"))
556
+
557
+
558
+def test_shm_ring_buffer_creation_checks_free_space():
559
+ with (
560
+ mock.patch.object(
561
+ shm_broadcast.shutil, "disk_usage", return_value=_fake_disk_usage(1 << 20)
562
+ ),
563
+ mock.patch.object(shm_broadcast.os.path, "isdir", return_value=True),
564
+ pytest.raises(RuntimeError, match="Insufficient space"),
565
+ ):
566
+ ShmRingBuffer(n_reader=1, max_chunk_bytes=24 * 1024 * 1024, max_chunks=10)
vllm/distributed/device_communicators/shm_broadcast.py
+29
@@ -1,7 +1,9 @@
1
# SPDX-License-Identifier: Apache-2.0
2
# SPDX-FileCopyrightText: Copyright contributors to the vLLM project
3
import functools
4
+import os
5
import pickle
6
+import shutil
7
import sys
8
import threading
9
import time
@@ -218,6 +220,32 @@ class SpinCondition:
220
self.local_notify_socket.send(b"\x00")
221
222
223
+SHM_PATH = "/dev/shm"
224
+
225
+
226
+def check_shm_free_space(required_bytes: int, shm_path: str = SHM_PATH) -> None:
227
+ """Raise if ``shm_path`` cannot fit a ``required_bytes`` shared segment.
228
+
229
+ Args:
230
+ required_bytes: Size of the shared-memory segment to be created.
231
+ shm_path: Mount point backing POSIX shared memory; skipped if absent.
232
+
233
+ Raises:
234
+ RuntimeError: If ``required_bytes`` exceeds the free space.
235
+ """
236
+ if not os.path.isdir(shm_path):
237
+ return
238
+ free_bytes = shutil.disk_usage(shm_path).free
239
+ if required_bytes <= free_bytes:
240
+ return
241
+ mib = 1 << 20
242
+ raise RuntimeError(
243
+ f"Insufficient space in {shm_path}: {required_bytes / mib:.0f} MiB "
244
+ f"required, {free_bytes / mib:.0f} MiB free. Increase {shm_path} "
245
+ "(e.g. --shm-size or --ipc=host)."
246
+ )
247
+
248
+
249
class ShmRingBuffer:
250
def __init__(
251
self,
@@ -288,6 +316,7 @@ class ShmRingBuffer:
316
if name is None:
317
# we are creating a buffer
318
self.is_creator = True
319
+ check_shm_free_space(self.total_bytes_of_buffer)
320
self.shared_memory = shared_memory.SharedMemory(
321
create=True, size=self.total_bytes_of_buffer
322
)