A high-throughput and memory-efficient inference and serving engine for LLMs
Mirrored from https://github.com/tduncan-lum/Nvidia-Optimization-Stack---VLLM.git
Synced Nov 17, 04:15 UTC
[Test] Add basic correctness test (#2908)
Zhuohan Li committed
Feb 18, 2024 at 16:44 UTC
a61f0521b8d0d53a91951bb56789ead397d5cd83
4 files changed
+91
-2
.buildkite/test-pipeline.yaml
+10
-2
@@ -11,8 +11,16 @@ steps:
11
- label: AsyncEngine Test
12
command: pytest -v -s async_engine
13
14
-- label: Distributed Test
15
- command: pytest -v -s test_comm_ops.py
14
+- label: Basic Correctness Test
15
+ command: pytest -v -s --forked basic_correctness
16
+
17
+- label: Distributed Comm Ops Test
18
+ command: pytest -v -s --forked test_comm_ops.py
19
+ working_dir: "/vllm-workspace/tests/distributed"
20
+ num_gpus: 2 # only support 1 or 2 for now.
21
+
22
+- label: Distributed Correctness Test
23
+ command: pytest -v -s --forked test_basic_distributed_correctness.py
24
working_dir: "/vllm-workspace/tests/distributed"
25
num_gpus: 2 # only support 1 or 2 for now.
26
tests/basic_correctness/test_basic_correctness.py
new
+38
@@ -0,0 +1,38 @@
1
+"""Compare the short outputs of HF and vLLM when using greedy sampling.
2
+
3
+Run `pytest tests/basic_correctness/test_basic_correctness.py --forked`.
4
+"""
5
+import pytest
6
+
7
+MODELS = [
8
+ "facebook/opt-125m",
9
+ "meta-llama/Llama-2-7b-hf",
10
+]
11
+
12
+
13
+@pytest.mark.parametrize("model", MODELS)
14
+@pytest.mark.parametrize("dtype", ["half"])
15
+@pytest.mark.parametrize("max_tokens", [5])
16
+def test_models(
17
+ hf_runner,
18
+ vllm_runner,
19
+ example_prompts,
20
+ model: str,
21
+ dtype: str,
22
+ max_tokens: int,
23
+) -> None:
24
+ hf_model = hf_runner(model, dtype=dtype)
25
+ hf_outputs = hf_model.generate_greedy(example_prompts, max_tokens)
26
+ del hf_model
27
+
28
+ vllm_model = vllm_runner(model, dtype=dtype)
29
+ vllm_outputs = vllm_model.generate_greedy(example_prompts, max_tokens)
30
+ del vllm_model
31
+
32
+ for i in range(len(example_prompts)):
33
+ hf_output_ids, hf_output_str = hf_outputs[i]
34
+ vllm_output_ids, vllm_output_str = vllm_outputs[i]
35
+ assert hf_output_str == vllm_output_str, (
36
+ f"Test{i}:\nHF: {hf_output_str!r}\nvLLM: {vllm_output_str!r}")
37
+ assert hf_output_ids == vllm_output_ids, (
38
+ f"Test{i}:\nHF: {hf_output_ids}\nvLLM: {vllm_output_ids}")
tests/conftest.py
+2
@@ -165,6 +165,7 @@ class VllmRunner:
165
model_name: str,
166
tokenizer_name: Optional[str] = None,
167
dtype: str = "half",
168
+ tensor_parallel_size: int = 1,
169
) -> None:
170
self.model = LLM(
171
model=model_name,
@@ -172,6 +173,7 @@ class VllmRunner:
173
trust_remote_code=True,
174
dtype=dtype,
175
swap_space=0,
176
+ tensor_parallel_size=tensor_parallel_size,
177
)
178
179
def generate(
tests/distributed/test_basic_distributed_correctness.py
new
+41
@@ -0,0 +1,41 @@
1
+"""Compare the outputs of HF and distributed vLLM when using greedy sampling.
2
+
3
+Run `pytest tests/distributed/test_basic_distributed_correctness.py --forked`.
4
+"""
5
+import pytest
6
+import torch
7
+
8
+MODELS = [
9
+ "facebook/opt-125m",
10
+ "meta-llama/Llama-2-7b-hf",
11
+]
12
+
13
+
14
+@pytest.mark.skipif(torch.cuda.device_count() < 2,
15
+ reason="Need at least 2 GPUs to run the test.")
16
+@pytest.mark.parametrize("model", MODELS)
17
+@pytest.mark.parametrize("dtype", ["half"])
18
+@pytest.mark.parametrize("max_tokens", [5])
19
+def test_models(
20
+ hf_runner,
21
+ vllm_runner,
22
+ example_prompts,
23
+ model: str,
24
+ dtype: str,
25
+ max_tokens: int,
26
+) -> None:
27
+ hf_model = hf_runner(model, dtype=dtype)
28
+ hf_outputs = hf_model.generate_greedy(example_prompts, max_tokens)
29
+ del hf_model
30
+
31
+ vllm_model = vllm_runner(model, dtype=dtype, tensor_parallel_size=2)
32
+ vllm_outputs = vllm_model.generate_greedy(example_prompts, max_tokens)
33
+ del vllm_model
34
+
35
+ for i in range(len(example_prompts)):
36
+ hf_output_ids, hf_output_str = hf_outputs[i]
37
+ vllm_output_ids, vllm_output_str = vllm_outputs[i]
38
+ assert hf_output_str == vllm_output_str, (
39
+ f"Test{i}:\nHF: {hf_output_str!r}\nvLLM: {vllm_output_str!r}")
40
+ assert hf_output_ids == vllm_output_ids, (
41
+ f"Test{i}:\nHF: {hf_output_ids}\nvLLM: {vllm_output_ids}")