@tduncan / Nvidia-Optimization-Stack-VLLM Mirror · read-only / commits / a61f0521b8
Fork 0

A high-throughput and memory-efficient inference and serving engine for LLMs

Mirrored from https://github.com/tduncan-lum/Nvidia-Optimization-Stack---VLLM.git

Synced Nov 17, 04:15 UTC

[Test] Add basic correctness test (#2908)

Zhuohan Li committed Feb 18, 2024 at 16:44 UTC a61f0521b8d0d53a91951bb56789ead397d5cd83
4 files changed +91 -2
.buildkite/test-pipeline.yaml
+10 -2
@@ -11,8 +11,16 @@ steps:
11 - label: AsyncEngine Test
12 command: pytest -v -s async_engine
13
14 -- label: Distributed Test
15 - command: pytest -v -s test_comm_ops.py
14 +- label: Basic Correctness Test
15 + command: pytest -v -s --forked basic_correctness
16 +
17 +- label: Distributed Comm Ops Test
18 + command: pytest -v -s --forked test_comm_ops.py
19 + working_dir: "/vllm-workspace/tests/distributed"
20 + num_gpus: 2 # only support 1 or 2 for now.
21 +
22 +- label: Distributed Correctness Test
23 + command: pytest -v -s --forked test_basic_distributed_correctness.py
24 working_dir: "/vllm-workspace/tests/distributed"
25 num_gpus: 2 # only support 1 or 2 for now.
26
tests/basic_correctness/test_basic_correctness.py new
+38
@@ -0,0 +1,38 @@
1 +"""Compare the short outputs of HF and vLLM when using greedy sampling.
2 +
3 +Run `pytest tests/basic_correctness/test_basic_correctness.py --forked`.
4 +"""
5 +import pytest
6 +
7 +MODELS = [
8 + "facebook/opt-125m",
9 + "meta-llama/Llama-2-7b-hf",
10 +]
11 +
12 +
13 +@pytest.mark.parametrize("model", MODELS)
14 +@pytest.mark.parametrize("dtype", ["half"])
15 +@pytest.mark.parametrize("max_tokens", [5])
16 +def test_models(
17 + hf_runner,
18 + vllm_runner,
19 + example_prompts,
20 + model: str,
21 + dtype: str,
22 + max_tokens: int,
23 +) -> None:
24 + hf_model = hf_runner(model, dtype=dtype)
25 + hf_outputs = hf_model.generate_greedy(example_prompts, max_tokens)
26 + del hf_model
27 +
28 + vllm_model = vllm_runner(model, dtype=dtype)
29 + vllm_outputs = vllm_model.generate_greedy(example_prompts, max_tokens)
30 + del vllm_model
31 +
32 + for i in range(len(example_prompts)):
33 + hf_output_ids, hf_output_str = hf_outputs[i]
34 + vllm_output_ids, vllm_output_str = vllm_outputs[i]
35 + assert hf_output_str == vllm_output_str, (
36 + f"Test{i}:\nHF: {hf_output_str!r}\nvLLM: {vllm_output_str!r}")
37 + assert hf_output_ids == vllm_output_ids, (
38 + f"Test{i}:\nHF: {hf_output_ids}\nvLLM: {vllm_output_ids}")
tests/conftest.py
+2
@@ -165,6 +165,7 @@ class VllmRunner:
165 model_name: str,
166 tokenizer_name: Optional[str] = None,
167 dtype: str = "half",
168 + tensor_parallel_size: int = 1,
169 ) -> None:
170 self.model = LLM(
171 model=model_name,
@@ -172,6 +173,7 @@ class VllmRunner:
173 trust_remote_code=True,
174 dtype=dtype,
175 swap_space=0,
176 + tensor_parallel_size=tensor_parallel_size,
177 )
178
179 def generate(
tests/distributed/test_basic_distributed_correctness.py new
+41
@@ -0,0 +1,41 @@
1 +"""Compare the outputs of HF and distributed vLLM when using greedy sampling.
2 +
3 +Run `pytest tests/distributed/test_basic_distributed_correctness.py --forked`.
4 +"""
5 +import pytest
6 +import torch
7 +
8 +MODELS = [
9 + "facebook/opt-125m",
10 + "meta-llama/Llama-2-7b-hf",
11 +]
12 +
13 +
14 +@pytest.mark.skipif(torch.cuda.device_count() < 2,
15 + reason="Need at least 2 GPUs to run the test.")
16 +@pytest.mark.parametrize("model", MODELS)
17 +@pytest.mark.parametrize("dtype", ["half"])
18 +@pytest.mark.parametrize("max_tokens", [5])
19 +def test_models(
20 + hf_runner,
21 + vllm_runner,
22 + example_prompts,
23 + model: str,
24 + dtype: str,
25 + max_tokens: int,
26 +) -> None:
27 + hf_model = hf_runner(model, dtype=dtype)
28 + hf_outputs = hf_model.generate_greedy(example_prompts, max_tokens)
29 + del hf_model
30 +
31 + vllm_model = vllm_runner(model, dtype=dtype, tensor_parallel_size=2)
32 + vllm_outputs = vllm_model.generate_greedy(example_prompts, max_tokens)
33 + del vllm_model
34 +
35 + for i in range(len(example_prompts)):
36 + hf_output_ids, hf_output_str = hf_outputs[i]
37 + vllm_output_ids, vllm_output_str = vllm_outputs[i]
38 + assert hf_output_str == vllm_output_str, (
39 + f"Test{i}:\nHF: {hf_output_str!r}\nvLLM: {vllm_output_str!r}")
40 + assert hf_output_ids == vllm_output_ids, (
41 + f"Test{i}:\nHF: {hf_output_ids}\nvLLM: {vllm_output_ids}")