@tduncan / Nvidia-Optimization-Stack-VLLM Mirror · read-only / commits / d571ca0108
Fork 0

A high-throughput and memory-efficient inference and serving engine for LLMs

Mirrored from https://github.com/tduncan-lum/Nvidia-Optimization-Stack---VLLM.git

Synced Nov 16, 23:15 UTC

[ci][distributed] add tests for custom allreduce (#5689)

youkaichao committed Jun 19, 2024 at 13:16 UTC d571ca010813c5532c646ee74c1a2c9e1e78e12a
2 files changed +10 -5
.buildkite/test-pipeline.yaml
+6 -2
@@ -182,7 +182,11 @@ steps:
182 - pip install -r requirements-docs.txt
183 - SPHINXOPTS=\"-W\" make html
184
185 -- label: A100 status
185 +- label: Distributed Tests (A100)
186 gpu: a100
187 commands:
188 - - nvidia-smi
188 + # NOTE: don't test llama model here, it seems hf implementation is buggy
189 + # see https://github.com/vllm-project/vllm/pull/5689 for details
190 + - pytest -v -s distributed/test_custom_all_reduce.py
191 + - TEST_DIST_MODEL=facebook/opt-125m DISTRIBUTED_EXECUTOR_BACKEND=ray pytest -v -s distributed/test_basic_distributed_correctness.py
192 + - TEST_DIST_MODEL=facebook/opt-125m DISTRIBUTED_EXECUTOR_BACKEND=mp pytest -v -s distributed/test_basic_distributed_correctness.py
tests/distributed/test_custom_all_reduce.py
+4 -3
@@ -11,7 +11,8 @@ from vllm.distributed.communication_op import ( # noqa
11 from vllm.distributed.parallel_state import (get_tensor_model_parallel_group,
12 get_tp_group, graph_capture)
13
14 -from ..utils import (init_test_distributed_environment,
14 +from ..utils import (ensure_model_parallel_initialized,
15 + init_test_distributed_environment,
16 multi_process_tensor_parallel)
17
18 random.seed(42)
@@ -27,8 +28,8 @@ def graph_allreduce(tp_size, pp_size, rank, distributed_init_port):
28 torch.cuda.set_device(device)
29 init_test_distributed_environment(tp_size, pp_size, rank,
30 distributed_init_port)
30 -
31 - group = get_tensor_model_parallel_group()
31 + ensure_model_parallel_initialized(tp_size, pp_size)
32 + group = get_tensor_model_parallel_group().device_group
33
34 # A small all_reduce for warmup.
35 # this is needed because device communicators might be created lazily