A high-throughput and memory-efficient inference and serving engine for LLMs
Mirrored from https://github.com/tduncan-lum/Nvidia-Optimization-Stack---VLLM.git
Synced Nov 16, 23:15 UTC
[Core][Distributed] add same-node detection (#5369)
youkaichao committed
Jun 11, 2024 at 10:53 UTC
c4bd03c7c5672b6a5d3d6839339853e04fe15127
4 files changed
+87
-1
.buildkite/test-pipeline.yaml
+1
@@ -37,6 +37,7 @@ steps:
37
working_dir: "/vllm-workspace/tests"
38
num_gpus: 2
39
commands:
40
+ - VLLM_TEST_SAME_HOST=1 torchrun --nproc-per-node=4 distributed/test_same_node.py
41
- TEST_DIST_MODEL=facebook/opt-125m DISTRIBUTED_EXECUTOR_BACKEND=ray pytest -v -s distributed/test_basic_distributed_correctness.py
42
- TEST_DIST_MODEL=meta-llama/Llama-2-7b-hf DISTRIBUTED_EXECUTOR_BACKEND=ray pytest -v -s distributed/test_basic_distributed_correctness.py
43
- TEST_DIST_MODEL=facebook/opt-125m DISTRIBUTED_EXECUTOR_BACKEND=ray pytest -v -s distributed/test_chunked_prefill_distributed.py
tests/distributed/test_same_node.py
new
+11
@@ -0,0 +1,11 @@
1
+import os
2
+
3
+import torch
4
+
5
+from vllm.distributed.parallel_state import is_in_the_same_node
6
+
7
+torch.distributed.init_process_group(backend="gloo")
8
+test_result = is_in_the_same_node(torch.distributed.group.WORLD)
9
+
10
+expected = os.environ.get("VLLM_TEST_SAME_HOST", "1") == "1"
11
+assert test_result == expected, f"Expected {expected}, got {test_result}"
vllm/distributed/device_communicators/custom_all_reduce.py
+8
-1
@@ -10,7 +10,7 @@ from vllm import _custom_ops as ops
10
from vllm.distributed.device_communicators.custom_all_reduce_utils import (
11
gpu_p2p_access_check)
12
from vllm.distributed.parallel_state import (
13
- get_local_rank, get_tensor_model_parallel_cpu_group)
13
+ get_local_rank, get_tensor_model_parallel_cpu_group, is_in_the_same_node)
14
from vllm.logger import init_logger
15
16
try:
@@ -113,6 +113,13 @@ class CustomAllreduce:
113
assert dist.get_backend(group) != dist.Backend.NCCL, (
114
"CustomAllreduce should be attached to a non-NCCL group.")
115
116
+ if not is_in_the_same_node(group):
117
+ # No need to initialize custom allreduce for multi-node case.
118
+ logger.warning(
119
+ "Custom allreduce is disabled because this process group"
120
+ " spans across nodes.")
121
+ return
122
+
123
rank = dist.get_rank(group=self.group)
124
world_size = dist.get_world_size(group=self.group)
125
if world_size == 1:
vllm/distributed/parallel_state.py
+67
@@ -3,6 +3,8 @@
3
# https://github.com/NVIDIA/Megatron-LM/blob/main/megatron/core/parallel_state.py
4
# Copyright (c) 2022, NVIDIA CORPORATION. All rights reserved.
5
"""Tensor and pipeline parallel groups."""
6
+import contextlib
7
+from multiprocessing import resource_tracker, shared_memory
8
from typing import List, Optional
9
10
import torch
@@ -376,3 +378,68 @@ def destroy_model_parallel():
378
_PP_DEVICE_GROUP = None
379
global _PP_GLOBAL_RANKS
380
_PP_GLOBAL_RANKS = None
381
+
382
+
383
+def is_in_the_same_node(pg: ProcessGroup):
384
+ """
385
+ This is a collective operation that checks if all processes in the group
386
+ are in the same node. It tests if all processes are attached to the same
387
+ memory system (shared access to shared memory).
388
+ """
389
+ assert torch.distributed.get_backend(
390
+ pg) != torch.distributed.Backend.NCCL, (
391
+ "is_in_the_same_node should be tested with a non-NCCL group.")
392
+ # local rank inside the group
393
+ rank = torch.distributed.get_rank(group=pg)
394
+ world_size = torch.distributed.get_world_size(group=pg)
395
+
396
+ # local tensor in each process to store the result
397
+ is_in_the_same_node = torch.tensor([0] * world_size, dtype=torch.int32)
398
+
399
+ # global ranks of the processes in the group
400
+ ranks = torch.distributed.get_process_group_ranks(pg)
401
+
402
+ magic_message = b"magic_message"
403
+ shm = None
404
+
405
+ try:
406
+ with contextlib.suppress(OSError):
407
+ if rank == 0:
408
+ # create a shared memory segment
409
+ shm = shared_memory.SharedMemory(create=True, size=128)
410
+ shm.buf[:len(magic_message)] = magic_message
411
+ torch.distributed.broadcast_object_list([shm.name],
412
+ src=ranks[0],
413
+ group=pg)
414
+ is_in_the_same_node[0] = 1
415
+ else:
416
+ # try to open the shared memory segment
417
+ recv = [None]
418
+ torch.distributed.broadcast_object_list(recv,
419
+ src=ranks[0],
420
+ group=pg)
421
+ name = recv[0]
422
+ shm = shared_memory.SharedMemory(name=name)
423
+ if shm.buf[:len(magic_message)] == magic_message:
424
+ is_in_the_same_node[rank] = 1
425
+ except Exception as e:
426
+ logger.error("Error ignored in is_in_the_same_node: %s", e)
427
+ finally:
428
+ if shm:
429
+ shm.close()
430
+
431
+ torch.distributed.barrier(group=pg)
432
+
433
+ # clean up the shared memory segment
434
+ with contextlib.suppress(OSError):
435
+ if rank == 0:
436
+ if shm:
437
+ shm.unlink()
438
+ else:
439
+ if shm:
440
+ # fix to https://stackoverflow.com/q/62748654/9191338
441
+ resource_tracker.unregister(
442
+ shm._name, "shared_memory") # type: ignore[attr-defined]
443
+ torch.distributed.all_reduce(is_in_the_same_node, group=pg)
444
+
445
+ return is_in_the_same_node.sum().item() == world_size