@tduncan / Nvidia-Optimization-Stack-VLLM Mirror · read-only / commits / 04e5583425
Fork 0

A high-throughput and memory-efficient inference and serving engine for LLMs

Mirrored from https://github.com/tduncan-lum/Nvidia-Optimization-Stack---VLLM.git

Synced Nov 16, 19:30 UTC

[ci][distributed] merge distributed test commands (#7097)

Co-authored-by: Cyrus Leung <cyrus.tl.leung@gmail.com>

youkaichao committed Aug 2, 2024 at 21:33 UTC 04e55834254bf11770d544bbeebdbdb7731d9bbd
4 files changed +78 -91
.buildkite/test-pipeline.yaml
+4 -23
@@ -82,20 +82,9 @@ steps:
82 num_gpus: 2
83 commands:
84 - VLLM_TEST_SAME_HOST=1 torchrun --nproc-per-node=4 distributed/test_same_node.py
85 - - TEST_DIST_MODEL=facebook/opt-125m DISTRIBUTED_EXECUTOR_BACKEND=ray pytest -v -s distributed/test_basic_distributed_correctness.py
86 - - TEST_DIST_MODEL=meta-llama/Llama-2-7b-hf DISTRIBUTED_EXECUTOR_BACKEND=ray pytest -v -s distributed/test_basic_distributed_correctness.py
87 - - TEST_DIST_MODEL=facebook/opt-125m DISTRIBUTED_EXECUTOR_BACKEND=ray VLLM_USE_RAY_SPMD_WORKER=1 VLLM_USE_RAY_COMPILED_DAG=1 pytest -v -s distributed/test_basic_distributed_correctness.py
88 - - TEST_DIST_MODEL=meta-llama/Llama-2-7b-hf DISTRIBUTED_EXECUTOR_BACKEND=ray VLLM_USE_RAY_SPMD_WORKER=1 VLLM_USE_RAY_COMPILED_DAG=1 pytest -v -s distributed/test_basic_distributed_correctness.py
89 - - TEST_DIST_MODEL=facebook/opt-125m DISTRIBUTED_EXECUTOR_BACKEND=ray pytest -v -s distributed/test_chunked_prefill_distributed.py
90 - - TEST_DIST_MODEL=meta-llama/Llama-2-7b-hf DISTRIBUTED_EXECUTOR_BACKEND=ray pytest -v -s distributed/test_chunked_prefill_distributed.py
91 - - TEST_DIST_MODEL=llava-hf/llava-1.5-7b-hf DISTRIBUTED_EXECUTOR_BACKEND=ray pytest -v -s distributed/test_multimodal_broadcast.py
92 - - TEST_DIST_MODEL=llava-hf/llava-v1.6-mistral-7b-hf DISTRIBUTED_EXECUTOR_BACKEND=ray pytest -v -s distributed/test_multimodal_broadcast.py
93 - - TEST_DIST_MODEL=facebook/opt-125m DISTRIBUTED_EXECUTOR_BACKEND=mp pytest -v -s distributed/test_basic_distributed_correctness.py
94 - - TEST_DIST_MODEL=meta-llama/Llama-2-7b-hf DISTRIBUTED_EXECUTOR_BACKEND=mp pytest -v -s distributed/test_basic_distributed_correctness.py
95 - - TEST_DIST_MODEL=facebook/opt-125m DISTRIBUTED_EXECUTOR_BACKEND=mp pytest -v -s distributed/test_chunked_prefill_distributed.py
96 - - TEST_DIST_MODEL=meta-llama/Llama-2-7b-hf DISTRIBUTED_EXECUTOR_BACKEND=mp pytest -v -s distributed/test_chunked_prefill_distributed.py
97 - - TEST_DIST_MODEL=llava-hf/llava-1.5-7b-hf DISTRIBUTED_EXECUTOR_BACKEND=mp pytest -v -s distributed/test_multimodal_broadcast.py
98 - - TEST_DIST_MODEL=llava-hf/llava-v1.6-mistral-7b-hf DISTRIBUTED_EXECUTOR_BACKEND=mp pytest -v -s distributed/test_multimodal_broadcast.py
85 + - TARGET_TEST_SUITE=L4 pytest -v -s distributed/test_basic_distributed_correctness.py
86 + - pytest -v -s distributed/test_chunked_prefill_distributed.py
87 + - pytest -v -s distributed/test_multimodal_broadcast.py
88 - pytest -v -s spec_decode/e2e/test_integration_dist_tp2.py
89 - CUDA_VISIBLE_DEVICES=0,1 pytest -v -s test_sharded_state_loader.py
90 - CUDA_VISIBLE_DEVICES=0,1 pytest -v -s distributed/test_utils.py
@@ -107,11 +96,6 @@ steps:
96 fast_check: true
97 commands:
98 - pytest -v -s distributed/test_pynccl.py
110 - # We want to test that models which use 2 GPUs work with 4 GPUs, which is why we duplicate them here.
111 - # See https://github.com/vllm-project/vllm/pull/5473#issuecomment-2166601837 for context.
112 - - TEST_DIST_MODEL=facebook/opt-125m DISTRIBUTED_EXECUTOR_BACKEND=ray pytest -v -s distributed/test_basic_distributed_correctness.py
113 - - TEST_DIST_MODEL=facebook/opt-125m DISTRIBUTED_EXECUTOR_BACKEND=ray VLLM_USE_RAY_SPMD_WORKER=1 VLLM_USE_RAY_COMPILED_DAG=1 pytest -v -s distributed/test_basic_distributed_correctness.py
114 - - TEST_DIST_MODEL=facebook/opt-125m DISTRIBUTED_EXECUTOR_BACKEND=mp pytest -v -s distributed/test_basic_distributed_correctness.py
99 - pytest -v -s spec_decode/e2e/test_integration_dist_tp4.py
100
101 - label: Pipeline Parallelism Test
@@ -279,9 +263,6 @@ steps:
263 # NOTE: don't test llama model here, it seems hf implementation is buggy
264 # see https://github.com/vllm-project/vllm/pull/5689 for details
265 - pytest -v -s distributed/test_custom_all_reduce.py
282 - - TEST_DIST_MODEL=facebook/opt-125m DISTRIBUTED_EXECUTOR_BACKEND=ray pytest -v -s distributed/test_basic_distributed_correctness.py
283 - - TEST_DIST_MODEL=facebook/opt-125m DISTRIBUTED_EXECUTOR_BACKEND=mp pytest -v -s distributed/test_basic_distributed_correctness.py
266 - pip install https://github.com/flashinfer-ai/flashinfer/releases/download/v0.1.2/flashinfer-0.1.2+cu121torch2.4-cp310-cp310-linux_x86_64.whl
285 - - VLLM_ATTENTION_BACKEND=FLASHINFER TEST_DIST_MODEL=facebook/opt-125m DISTRIBUTED_EXECUTOR_BACKEND=ray pytest -v -s distributed/test_basic_distributed_correctness.py
286 - - VLLM_ATTENTION_BACKEND=FLASHINFER TEST_DIST_MODEL=meta-llama/Meta-Llama-3-8B DISTRIBUTED_EXECUTOR_BACKEND=ray pytest -v -s distributed/test_basic_distributed_correctness.py
267 + - TARGET_TEST_SUITE=A100 pytest -v -s distributed/test_basic_distributed_correctness.py
268 - pytest -v -s -x lora/test_mixtral.py
tests/distributed/test_basic_distributed_correctness.py
+33 -17
@@ -1,15 +1,10 @@
1 """Compare the outputs of HF and distributed vLLM when using greedy sampling.
2 -vLLM will allocate all the available memory, so we need to run the tests one
3 -by one. The solution is to pass arguments (model name) by environment
4 -variables.
2 +
3 Run:
4 ```sh
5 cd $VLLM_PATH/tests
6
9 -TEST_DIST_MODEL=facebook/opt-125m pytest \
10 - distributed/test_basic_distributed_correctness.py
11 -TEST_DIST_MODEL=meta-llama/Llama-2-7b-hf \
12 - distributed/test_basic_distributed_correctness.py
7 +pytest distributed/test_basic_distributed_correctness.py
8 ```
9 """
10 import os
@@ -19,27 +14,48 @@ import pytest
14 from vllm.utils import cuda_device_count_stateless
15
16 from ..models.utils import check_outputs_equal
17 +from ..utils import fork_new_process_for_each_test
18
23 -MODELS = [
24 - os.environ["TEST_DIST_MODEL"],
25 -]
26 -DISTRIBUTED_EXECUTOR_BACKEND = "DISTRIBUTED_EXECUTOR_BACKEND"
19 +TARGET_TEST_SUITE = os.environ.get("TARGET_TEST_SUITE", "L4")
20
21
22 @pytest.mark.skipif(cuda_device_count_stateless() < 2,
23 reason="Need at least 2 GPUs to run the test.")
31 -@pytest.mark.parametrize("model", MODELS)
32 -@pytest.mark.parametrize("dtype", ["half"])
33 -@pytest.mark.parametrize("max_tokens", [5])
24 +@pytest.mark.parametrize(
25 + "model, distributed_executor_backend, attention_backend, test_suite", [
26 + ("facebook/opt-125m", "ray", "", "L4"),
27 + ("facebook/opt-125m", "mp", "", "L4"),
28 + ("meta-llama/Llama-2-7b-hf", "ray", "", "L4"),
29 + ("meta-llama/Llama-2-7b-hf", "mp", "", "L4"),
30 + ("facebook/opt-125m", "ray", "", "A100"),
31 + ("facebook/opt-125m", "mp", "", "A100"),
32 + ("facebook/opt-125m", "mp", "FLASHINFER", "A100"),
33 + ("meta-llama/Meta-Llama-3-8B", "ray", "FLASHINFER", "A100"),
34 + ])
35 +@fork_new_process_for_each_test
36 def test_models(
37 hf_runner,
38 vllm_runner,
39 example_prompts,
40 model: str,
39 - dtype: str,
40 - max_tokens: int,
41 + distributed_executor_backend: str,
42 + attention_backend: str,
43 + test_suite: str,
44 ) -> None:
42 - distributed_executor_backend = os.getenv(DISTRIBUTED_EXECUTOR_BACKEND)
45 +
46 + if test_suite != TARGET_TEST_SUITE:
47 + pytest.skip(f"Skip test for {test_suite}")
48 +
49 + if model == "meta-llama/Llama-2-7b-hf" and distributed_executor_backend == "ray" and attention_backend == "" and test_suite == "L4": # noqa
50 + # test ray adag
51 + os.environ['VLLM_USE_RAY_SPMD_WORKER'] = "1"
52 + os.environ['VLLM_USE_RAY_COMPILED_DAG'] = "1"
53 +
54 + if attention_backend:
55 + os.environ["VLLM_ATTENTION_BACKEND"] = attention_backend
56 +
57 + dtype = "half"
58 + max_tokens = 5
59
60 # NOTE: take care of the order. run vLLM first, and then run HF.
61 # vLLM needs a fresh new process without cuda initialization.
tests/distributed/test_chunked_prefill_distributed.py
+14 -21
@@ -1,46 +1,39 @@
1 """Compare the outputs of HF and distributed vLLM when using greedy sampling.
2 -vLLM will allocate all the available memory, so we need to run the tests one
3 -by one. The solution is to pass arguments (model name) by environment
4 -variables.
2
3 Run:
4 ```sh
8 -TEST_DIST_MODEL=facebook/opt-125m pytest \
9 - test_chunked_prefill_distributed.py
10 -TEST_DIST_MODEL=meta-llama/Llama-2-7b-hf \
11 - test_chunked_prefill_distributed.py
5 +pytest test_chunked_prefill_distributed.py
6 ```
7 """
14 -import os
8
9 import pytest
10
11 from vllm.utils import cuda_device_count_stateless
12
13 from ..models.utils import check_outputs_equal
21 -
22 -MODELS = [
23 - os.environ["TEST_DIST_MODEL"],
24 -]
25 -DISTRIBUTED_EXECUTOR_BACKEND = "DISTRIBUTED_EXECUTOR_BACKEND"
14 +from ..utils import fork_new_process_for_each_test
15
16
17 @pytest.mark.skipif(cuda_device_count_stateless() < 2,
18 reason="Need at least 2 GPUs to run the test.")
30 -@pytest.mark.parametrize("model", MODELS)
31 -@pytest.mark.parametrize("dtype", ["half"])
32 -@pytest.mark.parametrize("max_tokens", [5])
33 -@pytest.mark.parametrize("chunked_prefill_token_size", [16])
19 +@pytest.mark.parametrize("model, distributed_executor_backend", [
20 + ("facebook/opt-125m", "ray"),
21 + ("meta-llama/Llama-2-7b-hf", "ray"),
22 + ("facebook/opt-125m", "mp"),
23 + ("meta-llama/Llama-2-7b-hf", "mp"),
24 +])
25 +@fork_new_process_for_each_test
26 def test_models(
27 hf_runner,
28 vllm_runner,
29 example_prompts,
30 model: str,
39 - dtype: str,
40 - max_tokens: int,
41 - chunked_prefill_token_size: int,
31 + distributed_executor_backend: str,
32 ) -> None:
43 - distributed_executor_backend = os.getenv(DISTRIBUTED_EXECUTOR_BACKEND)
33 +
34 + dtype = "half"
35 + max_tokens = 5
36 + chunked_prefill_token_size = 16
37
38 # Add a chunked prefill config.
39 max_num_seqs = min(chunked_prefill_token_size, 256)
tests/distributed/test_multimodal_broadcast.py
+27 -30
@@ -1,44 +1,41 @@
1 """Compare the outputs of HF and distributed vLLM when using greedy sampling.
2 -The second test will hang if more than one test is run per command, so we need
3 -to run the tests one by one. The solution is to pass arguments (model name) by
4 -environment variables.
2
3 Run:
4 ```sh
8 -TEST_DIST_MODEL=llava-hf/llava-1.5-7b-hf \
9 - test_multimodal_broadcast.py
10 -TEST_DIST_MODEL=microsoft/Phi-3-vision-128k-instruct \
11 - test_multimodal_broadcast.py
5 +pytest -s -v test_multimodal_broadcast.py
6 ```
7 """
14 -import os
8
9 import pytest
10
11 from vllm.utils import cuda_device_count_stateless
12
20 -model = os.environ["TEST_DIST_MODEL"]
21 -
22 -if model.startswith("llava-hf/llava-1.5"):
23 - from ..models.test_llava import models, run_test
24 -elif model.startswith("llava-hf/llava-v1.6"):
25 - from ..models.test_llava_next import models, run_test
26 -else:
27 - raise NotImplementedError(f"Unsupported model: {model}")
28 -
29 -
30 -@pytest.mark.parametrize("tensor_parallel_size", [2])
31 -@pytest.mark.parametrize("dtype", ["half"])
32 -@pytest.mark.parametrize("max_tokens", [128])
33 -@pytest.mark.parametrize("num_logprobs", [5])
34 -def test_models(hf_runner, vllm_runner, image_assets,
35 - tensor_parallel_size: int, dtype: str, max_tokens: int,
36 - num_logprobs: int) -> None:
37 - if cuda_device_count_stateless() < tensor_parallel_size:
38 - pytest.skip(
39 - f"Need at least {tensor_parallel_size} GPUs to run the test.")
40 -
41 - distributed_executor_backend = os.getenv("DISTRIBUTED_EXECUTOR_BACKEND")
13 +from ..utils import fork_new_process_for_each_test
14 +
15 +
16 +@pytest.mark.skipif(cuda_device_count_stateless() < 2,
17 + reason="Need at least 2 GPUs to run the test.")
18 +@pytest.mark.parametrize("model, distributed_executor_backend", [
19 + ("llava-hf/llava-1.5-7b-hf", "ray"),
20 + ("llava-hf/llava-v1.6-mistral-7b-hf", "ray"),
21 + ("llava-hf/llava-1.5-7b-hf", "mp"),
22 + ("llava-hf/llava-v1.6-mistral-7b-hf", "mp"),
23 +])
24 +@fork_new_process_for_each_test
25 +def test_models(hf_runner, vllm_runner, image_assets, model: str,
26 + distributed_executor_backend: str) -> None:
27 +
28 + dtype = "half"
29 + max_tokens = 5
30 + num_logprobs = 5
31 + tensor_parallel_size = 2
32 +
33 + if model.startswith("llava-hf/llava-1.5"):
34 + from ..models.test_llava import models, run_test
35 + elif model.startswith("llava-hf/llava-v1.6"):
36 + from ..models.test_llava_next import models, run_test
37 + else:
38 + raise NotImplementedError(f"Unsupported model: {model}")
39
40 run_test(
41 hf_runner,