A high-throughput and memory-efficient inference and serving engine for LLMs
Mirrored from https://github.com/tduncan-lum/Nvidia-Optimization-Stack---VLLM.git
Synced Nov 16, 23:15 UTC
[ci][distributed] add tests for custom allreduce (#5689)
youkaichao committed
Jun 19, 2024 at 13:16 UTC
d571ca010813c5532c646ee74c1a2c9e1e78e12a
2 files changed
+10
-5
.buildkite/test-pipeline.yaml
+6
-2
@@ -182,7 +182,11 @@ steps:
182
- pip install -r requirements-docs.txt
183
- SPHINXOPTS=\"-W\" make html
184
185
-- label: A100 status
185
+- label: Distributed Tests (A100)
186
gpu: a100
187
commands:
188
- - nvidia-smi
188
+ # NOTE: don't test llama model here, it seems hf implementation is buggy
189
+ # see https://github.com/vllm-project/vllm/pull/5689 for details
190
+ - pytest -v -s distributed/test_custom_all_reduce.py
191
+ - TEST_DIST_MODEL=facebook/opt-125m DISTRIBUTED_EXECUTOR_BACKEND=ray pytest -v -s distributed/test_basic_distributed_correctness.py
192
+ - TEST_DIST_MODEL=facebook/opt-125m DISTRIBUTED_EXECUTOR_BACKEND=mp pytest -v -s distributed/test_basic_distributed_correctness.py
tests/distributed/test_custom_all_reduce.py
+4
-3
@@ -11,7 +11,8 @@ from vllm.distributed.communication_op import ( # noqa
11
from vllm.distributed.parallel_state import (get_tensor_model_parallel_group,
12
get_tp_group, graph_capture)
13
14
-from ..utils import (init_test_distributed_environment,
14
+from ..utils import (ensure_model_parallel_initialized,
15
+ init_test_distributed_environment,
16
multi_process_tensor_parallel)
17
18
random.seed(42)
@@ -27,8 +28,8 @@ def graph_allreduce(tp_size, pp_size, rank, distributed_init_port):
28
torch.cuda.set_device(device)
29
init_test_distributed_environment(tp_size, pp_size, rank,
30
distributed_init_port)
30
-
31
- group = get_tensor_model_parallel_group()
31
+ ensure_model_parallel_initialized(tp_size, pp_size)
32
+ group = get_tensor_model_parallel_group().device_group
33
34
# A small all_reduce for warmup.
35
# this is needed because device communicators might be created lazily