A high-throughput and memory-efficient inference and serving engine for LLMs
Mirrored from https://github.com/tduncan-lum/Nvidia-Optimization-Stack---VLLM.git
Synced Nov 16, 19:30 UTC
[ci][distributed] merge distributed test commands (#7097)
Co-authored-by: Cyrus Leung <cyrus.tl.leung@gmail.com>
youkaichao committed
Aug 2, 2024 at 21:33 UTC
04e55834254bf11770d544bbeebdbdb7731d9bbd
4 files changed
+78
-91
.buildkite/test-pipeline.yaml
+4
-23
@@ -82,20 +82,9 @@ steps:
82
num_gpus: 2
83
commands:
84
- VLLM_TEST_SAME_HOST=1 torchrun --nproc-per-node=4 distributed/test_same_node.py
85
- - TEST_DIST_MODEL=facebook/opt-125m DISTRIBUTED_EXECUTOR_BACKEND=ray pytest -v -s distributed/test_basic_distributed_correctness.py
86
- - TEST_DIST_MODEL=meta-llama/Llama-2-7b-hf DISTRIBUTED_EXECUTOR_BACKEND=ray pytest -v -s distributed/test_basic_distributed_correctness.py
87
- - TEST_DIST_MODEL=facebook/opt-125m DISTRIBUTED_EXECUTOR_BACKEND=ray VLLM_USE_RAY_SPMD_WORKER=1 VLLM_USE_RAY_COMPILED_DAG=1 pytest -v -s distributed/test_basic_distributed_correctness.py
88
- - TEST_DIST_MODEL=meta-llama/Llama-2-7b-hf DISTRIBUTED_EXECUTOR_BACKEND=ray VLLM_USE_RAY_SPMD_WORKER=1 VLLM_USE_RAY_COMPILED_DAG=1 pytest -v -s distributed/test_basic_distributed_correctness.py
89
- - TEST_DIST_MODEL=facebook/opt-125m DISTRIBUTED_EXECUTOR_BACKEND=ray pytest -v -s distributed/test_chunked_prefill_distributed.py
90
- - TEST_DIST_MODEL=meta-llama/Llama-2-7b-hf DISTRIBUTED_EXECUTOR_BACKEND=ray pytest -v -s distributed/test_chunked_prefill_distributed.py
91
- - TEST_DIST_MODEL=llava-hf/llava-1.5-7b-hf DISTRIBUTED_EXECUTOR_BACKEND=ray pytest -v -s distributed/test_multimodal_broadcast.py
92
- - TEST_DIST_MODEL=llava-hf/llava-v1.6-mistral-7b-hf DISTRIBUTED_EXECUTOR_BACKEND=ray pytest -v -s distributed/test_multimodal_broadcast.py
93
- - TEST_DIST_MODEL=facebook/opt-125m DISTRIBUTED_EXECUTOR_BACKEND=mp pytest -v -s distributed/test_basic_distributed_correctness.py
94
- - TEST_DIST_MODEL=meta-llama/Llama-2-7b-hf DISTRIBUTED_EXECUTOR_BACKEND=mp pytest -v -s distributed/test_basic_distributed_correctness.py
95
- - TEST_DIST_MODEL=facebook/opt-125m DISTRIBUTED_EXECUTOR_BACKEND=mp pytest -v -s distributed/test_chunked_prefill_distributed.py
96
- - TEST_DIST_MODEL=meta-llama/Llama-2-7b-hf DISTRIBUTED_EXECUTOR_BACKEND=mp pytest -v -s distributed/test_chunked_prefill_distributed.py
97
- - TEST_DIST_MODEL=llava-hf/llava-1.5-7b-hf DISTRIBUTED_EXECUTOR_BACKEND=mp pytest -v -s distributed/test_multimodal_broadcast.py
98
- - TEST_DIST_MODEL=llava-hf/llava-v1.6-mistral-7b-hf DISTRIBUTED_EXECUTOR_BACKEND=mp pytest -v -s distributed/test_multimodal_broadcast.py
85
+ - TARGET_TEST_SUITE=L4 pytest -v -s distributed/test_basic_distributed_correctness.py
86
+ - pytest -v -s distributed/test_chunked_prefill_distributed.py
87
+ - pytest -v -s distributed/test_multimodal_broadcast.py
88
- pytest -v -s spec_decode/e2e/test_integration_dist_tp2.py
89
- CUDA_VISIBLE_DEVICES=0,1 pytest -v -s test_sharded_state_loader.py
90
- CUDA_VISIBLE_DEVICES=0,1 pytest -v -s distributed/test_utils.py
@@ -107,11 +96,6 @@ steps:
96
fast_check: true
97
commands:
98
- pytest -v -s distributed/test_pynccl.py
110
- # We want to test that models which use 2 GPUs work with 4 GPUs, which is why we duplicate them here.
111
- # See https://github.com/vllm-project/vllm/pull/5473#issuecomment-2166601837 for context.
112
- - TEST_DIST_MODEL=facebook/opt-125m DISTRIBUTED_EXECUTOR_BACKEND=ray pytest -v -s distributed/test_basic_distributed_correctness.py
113
- - TEST_DIST_MODEL=facebook/opt-125m DISTRIBUTED_EXECUTOR_BACKEND=ray VLLM_USE_RAY_SPMD_WORKER=1 VLLM_USE_RAY_COMPILED_DAG=1 pytest -v -s distributed/test_basic_distributed_correctness.py
114
- - TEST_DIST_MODEL=facebook/opt-125m DISTRIBUTED_EXECUTOR_BACKEND=mp pytest -v -s distributed/test_basic_distributed_correctness.py
99
- pytest -v -s spec_decode/e2e/test_integration_dist_tp4.py
100
101
- label: Pipeline Parallelism Test
@@ -279,9 +263,6 @@ steps:
263
# NOTE: don't test llama model here, it seems hf implementation is buggy
264
# see https://github.com/vllm-project/vllm/pull/5689 for details
265
- pytest -v -s distributed/test_custom_all_reduce.py
282
- - TEST_DIST_MODEL=facebook/opt-125m DISTRIBUTED_EXECUTOR_BACKEND=ray pytest -v -s distributed/test_basic_distributed_correctness.py
283
- - TEST_DIST_MODEL=facebook/opt-125m DISTRIBUTED_EXECUTOR_BACKEND=mp pytest -v -s distributed/test_basic_distributed_correctness.py
266
- pip install https://github.com/flashinfer-ai/flashinfer/releases/download/v0.1.2/flashinfer-0.1.2+cu121torch2.4-cp310-cp310-linux_x86_64.whl
285
- - VLLM_ATTENTION_BACKEND=FLASHINFER TEST_DIST_MODEL=facebook/opt-125m DISTRIBUTED_EXECUTOR_BACKEND=ray pytest -v -s distributed/test_basic_distributed_correctness.py
286
- - VLLM_ATTENTION_BACKEND=FLASHINFER TEST_DIST_MODEL=meta-llama/Meta-Llama-3-8B DISTRIBUTED_EXECUTOR_BACKEND=ray pytest -v -s distributed/test_basic_distributed_correctness.py
267
+ - TARGET_TEST_SUITE=A100 pytest -v -s distributed/test_basic_distributed_correctness.py
268
- pytest -v -s -x lora/test_mixtral.py
tests/distributed/test_basic_distributed_correctness.py
+33
-17
@@ -1,15 +1,10 @@
1
"""Compare the outputs of HF and distributed vLLM when using greedy sampling.
2
-vLLM will allocate all the available memory, so we need to run the tests one
3
-by one. The solution is to pass arguments (model name) by environment
4
-variables.
2
+
3
Run:
4
```sh
5
cd $VLLM_PATH/tests
6
9
-TEST_DIST_MODEL=facebook/opt-125m pytest \
10
- distributed/test_basic_distributed_correctness.py
11
-TEST_DIST_MODEL=meta-llama/Llama-2-7b-hf \
12
- distributed/test_basic_distributed_correctness.py
7
+pytest distributed/test_basic_distributed_correctness.py
8
```
9
"""
10
import os
@@ -19,27 +14,48 @@ import pytest
14
from vllm.utils import cuda_device_count_stateless
15
16
from ..models.utils import check_outputs_equal
17
+from ..utils import fork_new_process_for_each_test
18
23
-MODELS = [
24
- os.environ["TEST_DIST_MODEL"],
25
-]
26
-DISTRIBUTED_EXECUTOR_BACKEND = "DISTRIBUTED_EXECUTOR_BACKEND"
19
+TARGET_TEST_SUITE = os.environ.get("TARGET_TEST_SUITE", "L4")
20
21
22
@pytest.mark.skipif(cuda_device_count_stateless() < 2,
23
reason="Need at least 2 GPUs to run the test.")
31
-@pytest.mark.parametrize("model", MODELS)
32
-@pytest.mark.parametrize("dtype", ["half"])
33
-@pytest.mark.parametrize("max_tokens", [5])
24
+@pytest.mark.parametrize(
25
+ "model, distributed_executor_backend, attention_backend, test_suite", [
26
+ ("facebook/opt-125m", "ray", "", "L4"),
27
+ ("facebook/opt-125m", "mp", "", "L4"),
28
+ ("meta-llama/Llama-2-7b-hf", "ray", "", "L4"),
29
+ ("meta-llama/Llama-2-7b-hf", "mp", "", "L4"),
30
+ ("facebook/opt-125m", "ray", "", "A100"),
31
+ ("facebook/opt-125m", "mp", "", "A100"),
32
+ ("facebook/opt-125m", "mp", "FLASHINFER", "A100"),
33
+ ("meta-llama/Meta-Llama-3-8B", "ray", "FLASHINFER", "A100"),
34
+ ])
35
+@fork_new_process_for_each_test
36
def test_models(
37
hf_runner,
38
vllm_runner,
39
example_prompts,
40
model: str,
39
- dtype: str,
40
- max_tokens: int,
41
+ distributed_executor_backend: str,
42
+ attention_backend: str,
43
+ test_suite: str,
44
) -> None:
42
- distributed_executor_backend = os.getenv(DISTRIBUTED_EXECUTOR_BACKEND)
45
+
46
+ if test_suite != TARGET_TEST_SUITE:
47
+ pytest.skip(f"Skip test for {test_suite}")
48
+
49
+ if model == "meta-llama/Llama-2-7b-hf" and distributed_executor_backend == "ray" and attention_backend == "" and test_suite == "L4": # noqa
50
+ # test ray adag
51
+ os.environ['VLLM_USE_RAY_SPMD_WORKER'] = "1"
52
+ os.environ['VLLM_USE_RAY_COMPILED_DAG'] = "1"
53
+
54
+ if attention_backend:
55
+ os.environ["VLLM_ATTENTION_BACKEND"] = attention_backend
56
+
57
+ dtype = "half"
58
+ max_tokens = 5
59
60
# NOTE: take care of the order. run vLLM first, and then run HF.
61
# vLLM needs a fresh new process without cuda initialization.
tests/distributed/test_chunked_prefill_distributed.py
+14
-21
@@ -1,46 +1,39 @@
1
"""Compare the outputs of HF and distributed vLLM when using greedy sampling.
2
-vLLM will allocate all the available memory, so we need to run the tests one
3
-by one. The solution is to pass arguments (model name) by environment
4
-variables.
2
3
Run:
4
```sh
8
-TEST_DIST_MODEL=facebook/opt-125m pytest \
9
- test_chunked_prefill_distributed.py
10
-TEST_DIST_MODEL=meta-llama/Llama-2-7b-hf \
11
- test_chunked_prefill_distributed.py
5
+pytest test_chunked_prefill_distributed.py
6
```
7
"""
14
-import os
8
9
import pytest
10
11
from vllm.utils import cuda_device_count_stateless
12
13
from ..models.utils import check_outputs_equal
21
-
22
-MODELS = [
23
- os.environ["TEST_DIST_MODEL"],
24
-]
25
-DISTRIBUTED_EXECUTOR_BACKEND = "DISTRIBUTED_EXECUTOR_BACKEND"
14
+from ..utils import fork_new_process_for_each_test
15
16
17
@pytest.mark.skipif(cuda_device_count_stateless() < 2,
18
reason="Need at least 2 GPUs to run the test.")
30
-@pytest.mark.parametrize("model", MODELS)
31
-@pytest.mark.parametrize("dtype", ["half"])
32
-@pytest.mark.parametrize("max_tokens", [5])
33
-@pytest.mark.parametrize("chunked_prefill_token_size", [16])
19
+@pytest.mark.parametrize("model, distributed_executor_backend", [
20
+ ("facebook/opt-125m", "ray"),
21
+ ("meta-llama/Llama-2-7b-hf", "ray"),
22
+ ("facebook/opt-125m", "mp"),
23
+ ("meta-llama/Llama-2-7b-hf", "mp"),
24
+])
25
+@fork_new_process_for_each_test
26
def test_models(
27
hf_runner,
28
vllm_runner,
29
example_prompts,
30
model: str,
39
- dtype: str,
40
- max_tokens: int,
41
- chunked_prefill_token_size: int,
31
+ distributed_executor_backend: str,
32
) -> None:
43
- distributed_executor_backend = os.getenv(DISTRIBUTED_EXECUTOR_BACKEND)
33
+
34
+ dtype = "half"
35
+ max_tokens = 5
36
+ chunked_prefill_token_size = 16
37
38
# Add a chunked prefill config.
39
max_num_seqs = min(chunked_prefill_token_size, 256)
tests/distributed/test_multimodal_broadcast.py
+27
-30
@@ -1,44 +1,41 @@
1
"""Compare the outputs of HF and distributed vLLM when using greedy sampling.
2
-The second test will hang if more than one test is run per command, so we need
3
-to run the tests one by one. The solution is to pass arguments (model name) by
4
-environment variables.
2
3
Run:
4
```sh
8
-TEST_DIST_MODEL=llava-hf/llava-1.5-7b-hf \
9
- test_multimodal_broadcast.py
10
-TEST_DIST_MODEL=microsoft/Phi-3-vision-128k-instruct \
11
- test_multimodal_broadcast.py
5
+pytest -s -v test_multimodal_broadcast.py
6
```
7
"""
14
-import os
8
9
import pytest
10
11
from vllm.utils import cuda_device_count_stateless
12
20
-model = os.environ["TEST_DIST_MODEL"]
21
-
22
-if model.startswith("llava-hf/llava-1.5"):
23
- from ..models.test_llava import models, run_test
24
-elif model.startswith("llava-hf/llava-v1.6"):
25
- from ..models.test_llava_next import models, run_test
26
-else:
27
- raise NotImplementedError(f"Unsupported model: {model}")
28
-
29
-
30
-@pytest.mark.parametrize("tensor_parallel_size", [2])
31
-@pytest.mark.parametrize("dtype", ["half"])
32
-@pytest.mark.parametrize("max_tokens", [128])
33
-@pytest.mark.parametrize("num_logprobs", [5])
34
-def test_models(hf_runner, vllm_runner, image_assets,
35
- tensor_parallel_size: int, dtype: str, max_tokens: int,
36
- num_logprobs: int) -> None:
37
- if cuda_device_count_stateless() < tensor_parallel_size:
38
- pytest.skip(
39
- f"Need at least {tensor_parallel_size} GPUs to run the test.")
40
-
41
- distributed_executor_backend = os.getenv("DISTRIBUTED_EXECUTOR_BACKEND")
13
+from ..utils import fork_new_process_for_each_test
14
+
15
+
16
+@pytest.mark.skipif(cuda_device_count_stateless() < 2,
17
+ reason="Need at least 2 GPUs to run the test.")
18
+@pytest.mark.parametrize("model, distributed_executor_backend", [
19
+ ("llava-hf/llava-1.5-7b-hf", "ray"),
20
+ ("llava-hf/llava-v1.6-mistral-7b-hf", "ray"),
21
+ ("llava-hf/llava-1.5-7b-hf", "mp"),
22
+ ("llava-hf/llava-v1.6-mistral-7b-hf", "mp"),
23
+])
24
+@fork_new_process_for_each_test
25
+def test_models(hf_runner, vllm_runner, image_assets, model: str,
26
+ distributed_executor_backend: str) -> None:
27
+
28
+ dtype = "half"
29
+ max_tokens = 5
30
+ num_logprobs = 5
31
+ tensor_parallel_size = 2
32
+
33
+ if model.startswith("llava-hf/llava-1.5"):
34
+ from ..models.test_llava import models, run_test
35
+ elif model.startswith("llava-hf/llava-v1.6"):
36
+ from ..models.test_llava_next import models, run_test
37
+ else:
38
+ raise NotImplementedError(f"Unsupported model: {model}")
39
40
run_test(
41
hf_runner,