@tduncan / Nvidia-Optimization-Stack-VLLM Mirror · read-only / commits / 07eb6f19f3
Fork 0

A high-throughput and memory-efficient inference and serving engine for LLMs

Mirrored from https://github.com/tduncan-lum/Nvidia-Optimization-Stack---VLLM.git

Synced Nov 16, 20:45 UTC

[bugfix][distributed] fix multi-node bug for shared memory (#6597)

youkaichao committed Jul 19, 2024 at 15:34 UTC 07eb6f19f3b0ee9f7adf6eb689607028aa40bfd5
1 file changed +8 -2
vllm/distributed/device_communicators/shm_broadcast.py
+8 -2
@@ -108,8 +108,14 @@ class ShmRingBuffer:
108 # created by the process. The following patch is a workaround.
109 with patch("multiprocessing.resource_tracker.register",
110 lambda *args, **kwargs: None):
111 - self.shared_memory = shared_memory.SharedMemory(name=name)
112 - assert self.shared_memory.size == self.total_bytes_of_buffer
111 + try:
112 + self.shared_memory = shared_memory.SharedMemory(name=name)
113 + assert self.shared_memory.size == self.total_bytes_of_buffer # noqa
114 + except FileNotFoundError:
115 + # we might deserialize the object in a different node
116 + # in this case, this object is not used,
117 + # and we should suppress the error
118 + pass
119
120 def __reduce__(self):
121 return (