A high-throughput and memory-efficient inference and serving engine for LLMs
Mirrored from https://github.com/tduncan-lum/Nvidia-Optimization-Stack---VLLM.git
Synced Nov 11, 16:15 UTC
fix(shm): Add memory barriers for cross-process shared memory visibility (#30407)
Signed-off-by: Christina Holland <hey@christinaholland.com> Signed-off-by: Christina <truffle@gmail.com>
Christina Norman committed
Dec 10, 2025 at 17:01 UTC
166ac3c94d6ee845d4d8dc1a6dced4d9033fa4e3
1 file changed
+44
vllm/distributed/device_communicators/shm_broadcast.py
+44
@@ -2,6 +2,7 @@
2
# SPDX-FileCopyrightText: Copyright contributors to the vLLM project
3
import functools
4
import pickle
5
+import threading
6
import time
7
from contextlib import contextmanager
8
from dataclasses import dataclass, field
@@ -43,6 +44,33 @@ VLLM_RINGBUFFER_WARNING_INTERVAL = envs.VLLM_RINGBUFFER_WARNING_INTERVAL
44
from_bytes_big = functools.partial(int.from_bytes, byteorder="big")
45
46
47
+# Memory fence for cross-process shared memory visibility.
48
+# Required for correct producer-consumer synchronization when using
49
+# shared memory without locks.
50
+_memory_fence_lock = threading.Lock()
51
+
52
+
53
+def memory_fence():
54
+ """
55
+ Full memory barrier for shared memory synchronization.
56
+
57
+ Ensures all prior memory writes are visible to other processes before
58
+ any subsequent reads. This is critical for lock-free producer-consumer
59
+ patterns using shared memory.
60
+
61
+ Implementation acquires and immediately releases a lock. Python's
62
+ threading.Lock provides sequentially consistent memory barrier semantics
63
+ across all major platforms (POSIX, Windows). This is a lightweight
64
+ operation (~20ns) that guarantees:
65
+ - All stores before the barrier are visible to other threads/processes
66
+ - All loads after the barrier see the latest values
67
+ """
68
+ # Lock acquire/release provides full memory barrier semantics.
69
+ # Using context manager ensures lock release even on exceptions.
70
+ with _memory_fence_lock:
71
+ pass
72
+
73
+
74
def to_bytes_big(value: int, size: int) -> bytes:
75
return value.to_bytes(size, byteorder="big")
76
@@ -414,6 +442,10 @@ class MessageQueue:
442
n_warning = 1
443
while True:
444
with self.buffer.get_metadata(self.current_idx) as metadata_buffer:
445
+ # Memory fence ensures we see the latest read flags from readers.
446
+ # Without this, we may read stale flags from our CPU cache and
447
+ # spin indefinitely even though readers have completed.
448
+ memory_fence()
449
read_count = sum(metadata_buffer[1:])
450
written_flag = metadata_buffer[0]
451
if written_flag and read_count != self.buffer.n_reader:
@@ -458,6 +490,10 @@ class MessageQueue:
490
metadata_buffer[i] = 0
491
# mark the block as written
492
metadata_buffer[0] = 1
493
+ # Memory fence ensures the write is visible to readers on other cores
494
+ # before we proceed. Without this, readers may spin indefinitely
495
+ # waiting for a write that's stuck in our CPU's store buffer.
496
+ memory_fence()
497
self.current_idx = (self.current_idx + 1) % self.buffer.max_chunks
498
break
499
@@ -473,6 +509,10 @@ class MessageQueue:
509
n_warning = 1
510
while True:
511
with self.buffer.get_metadata(self.current_idx) as metadata_buffer:
512
+ # Memory fence ensures we see the latest writes from the writer.
513
+ # Without this, we may read stale flags from our CPU cache
514
+ # and spin indefinitely even though writer has updated them.
515
+ memory_fence()
516
read_flag = metadata_buffer[self.local_reader_rank + 1]
517
written_flag = metadata_buffer[0]
518
if not written_flag or read_flag:
@@ -513,6 +553,10 @@ class MessageQueue:
553
# caller has read from the buffer
554
# set the read flag
555
metadata_buffer[self.local_reader_rank + 1] = 1
556
+ # Memory fence ensures the read flag is visible to the writer.
557
+ # Without this, writer may not see our read completion and
558
+ # could wait indefinitely for all readers to finish.
559
+ memory_fence()
560
self.current_idx = (self.current_idx + 1) % self.buffer.max_chunks
561
562
self._read_spin_timer.record_activity()