A high-throughput and memory-efficient inference and serving engine for LLMs
Mirrored from https://github.com/tduncan-lum/Nvidia-Optimization-Stack---VLLM.git
Synced Nov 16, 23:15 UTC
[misc][distributed] fix benign error in `is_in_the_same_node` (#5512)
youkaichao committed
Jun 14, 2024 at 10:59 UTC
d1c3d7d1398c26fa5afd4583a58fceca76555c2a
1 file changed
+10
-10
vllm/distributed/parallel_state.py
+10
-10
@@ -23,8 +23,9 @@ import contextlib
23
from collections import namedtuple
24
from contextlib import contextmanager, nullcontext
25
from dataclasses import dataclass
26
-from multiprocessing import resource_tracker, shared_memory
26
+from multiprocessing import shared_memory
27
from typing import Any, Dict, List, Optional, Tuple, Union
28
+from unittest.mock import patch
29
30
import torch
31
from torch.distributed import Backend, ProcessGroup
@@ -744,7 +745,12 @@ def is_in_the_same_node(pg: ProcessGroup):
745
src=ranks[0],
746
group=pg)
747
name = recv[0]
747
- shm = shared_memory.SharedMemory(name=name)
748
+ # fix to https://stackoverflow.com/q/62748654/9191338
749
+ # Python incorrectly tracks shared memory even if it is not
750
+ # created by the process. The following patch is a workaround.
751
+ with patch("multiprocessing.resource_tracker.register",
752
+ lambda *args, **kwargs: None):
753
+ shm = shared_memory.SharedMemory(name=name)
754
if shm.buf[:len(magic_message)] == magic_message:
755
is_in_the_same_node[rank] = 1
756
except Exception as e:
@@ -757,14 +763,8 @@ def is_in_the_same_node(pg: ProcessGroup):
763
764
# clean up the shared memory segment
765
with contextlib.suppress(OSError):
760
- if rank == 0:
761
- if shm:
762
- shm.unlink()
763
- else:
764
- if shm:
765
- # fix to https://stackoverflow.com/q/62748654/9191338
766
- resource_tracker.unregister(
767
- shm._name, "shared_memory") # type: ignore[attr-defined]
766
+ if rank == 0 and shm:
767
+ shm.unlink()
768
torch.distributed.all_reduce(is_in_the_same_node, group=pg)
769
770
return is_in_the_same_node.sum().item() == world_size