@tduncan / Nvidia-Optimization-Stack-VLLM Mirror · read-only / commits / d1c3d7d139
Fork 0

A high-throughput and memory-efficient inference and serving engine for LLMs

Mirrored from https://github.com/tduncan-lum/Nvidia-Optimization-Stack---VLLM.git

Synced Nov 16, 23:15 UTC

[misc][distributed] fix benign error in `is_in_the_same_node` (#5512)

youkaichao committed Jun 14, 2024 at 10:59 UTC d1c3d7d1398c26fa5afd4583a58fceca76555c2a
1 file changed +10 -10
vllm/distributed/parallel_state.py
+10 -10
@@ -23,8 +23,9 @@ import contextlib
23 from collections import namedtuple
24 from contextlib import contextmanager, nullcontext
25 from dataclasses import dataclass
26 -from multiprocessing import resource_tracker, shared_memory
26 +from multiprocessing import shared_memory
27 from typing import Any, Dict, List, Optional, Tuple, Union
28 +from unittest.mock import patch
29
30 import torch
31 from torch.distributed import Backend, ProcessGroup
@@ -744,7 +745,12 @@ def is_in_the_same_node(pg: ProcessGroup):
745 src=ranks[0],
746 group=pg)
747 name = recv[0]
747 - shm = shared_memory.SharedMemory(name=name)
748 + # fix to https://stackoverflow.com/q/62748654/9191338
749 + # Python incorrectly tracks shared memory even if it is not
750 + # created by the process. The following patch is a workaround.
751 + with patch("multiprocessing.resource_tracker.register",
752 + lambda *args, **kwargs: None):
753 + shm = shared_memory.SharedMemory(name=name)
754 if shm.buf[:len(magic_message)] == magic_message:
755 is_in_the_same_node[rank] = 1
756 except Exception as e:
@@ -757,14 +763,8 @@ def is_in_the_same_node(pg: ProcessGroup):
763
764 # clean up the shared memory segment
765 with contextlib.suppress(OSError):
760 - if rank == 0:
761 - if shm:
762 - shm.unlink()
763 - else:
764 - if shm:
765 - # fix to https://stackoverflow.com/q/62748654/9191338
766 - resource_tracker.unregister(
767 - shm._name, "shared_memory") # type: ignore[attr-defined]
766 + if rank == 0 and shm:
767 + shm.unlink()
768 torch.distributed.all_reduce(is_in_the_same_node, group=pg)
769
770 return is_in_the_same_node.sum().item() == world_size