A high-throughput and memory-efficient inference and serving engine for LLMs
Mirrored from https://github.com/tduncan-lum/Nvidia-Optimization-Stack---VLLM.git
Synced Nov 16, 22:00 UTC
[CI/Build] Reuse code for checking output consistency (#5988)
Cyrus Leung committed
Jun 30, 2024 at 11:44 UTC
cff6a1fec15dba524c162d20a4e8de4df2b0a3d5
11 files changed
+125
-75
tests/basic_correctness/test_basic_correctness.py
+8
-7
@@ -8,6 +8,8 @@ import pytest
8
9
from vllm import LLM
10
11
+from ..models.utils import check_outputs_equal
12
+
13
MODELS = [
14
"facebook/opt-125m",
15
"meta-llama/Llama-2-7b-hf",
@@ -46,10 +48,9 @@ def test_models(
48
gpu_memory_utilization=0.7) as vllm_model:
49
vllm_outputs = vllm_model.generate_greedy(example_prompts, max_tokens)
50
49
- for i in range(len(example_prompts)):
50
- hf_output_ids, hf_output_str = hf_outputs[i]
51
- vllm_output_ids, vllm_output_str = vllm_outputs[i]
52
- assert hf_output_str == vllm_output_str, (
53
- f"Test{i}:\nHF: {hf_output_str!r}\nvLLM: {vllm_output_str!r}")
54
- assert hf_output_ids == vllm_output_ids, (
55
- f"Test{i}:\nHF: {hf_output_ids}\nvLLM: {vllm_output_ids}")
51
+ check_outputs_equal(
52
+ outputs_0_lst=hf_outputs,
53
+ outputs_1_lst=vllm_outputs,
54
+ name_0="hf",
55
+ name_1="vllm",
56
+ )
tests/basic_correctness/test_chunked_prefill.py
+8
-7
@@ -8,6 +8,8 @@ Run `pytest tests/models/test_chunked_prefill.py`.
8
"""
9
import pytest
10
11
+from ..models.utils import check_outputs_equal
12
+
13
MODELS = [
14
"facebook/opt-125m",
15
"meta-llama/Llama-2-7b-hf",
@@ -54,10 +56,9 @@ def test_models(
56
) as vllm_model:
57
vllm_outputs = vllm_model.generate_greedy(example_prompts, max_tokens)
58
57
- for i in range(len(example_prompts)):
58
- hf_output_ids, hf_output_str = hf_outputs[i]
59
- vllm_output_ids, vllm_output_str = vllm_outputs[i]
60
- assert hf_output_str == vllm_output_str, (
61
- f"Test{i}:\nHF: {hf_output_str!r}\nvLLM: {vllm_output_str!r}")
62
- assert hf_output_ids == vllm_output_ids, (
63
- f"Test{i}:\nHF: {hf_output_ids}\nvLLM: {vllm_output_ids}")
59
+ check_outputs_equal(
60
+ outputs_0_lst=hf_outputs,
61
+ outputs_1_lst=vllm_outputs,
62
+ name_0="hf",
63
+ name_1="vllm",
64
+ )
tests/basic_correctness/test_preemption.py
+9
-7
@@ -12,6 +12,8 @@ from vllm import SamplingParams
12
from vllm.core.scheduler import (ARTIFICIAL_PREEMPTION_MAX_CNT,
13
ENABLE_ARTIFICIAL_PREEMPT)
14
15
+from ..models.utils import check_outputs_equal
16
+
17
MODELS = [
18
"facebook/opt-125m",
19
]
@@ -94,13 +96,13 @@ def test_preemption(
96
total_preemption = (
97
vllm_model.model.llm_engine.scheduler.num_cumulative_preemption)
98
97
- for i in range(len(example_prompts)):
98
- hf_output_ids, hf_output_str = hf_outputs[i]
99
- vllm_output_ids, vllm_output_str = vllm_outputs[i]
100
- assert hf_output_str == vllm_output_str, (
101
- f"Test{i}:\nHF: {hf_output_str!r}\nvLLM: {vllm_output_str!r}")
102
- assert hf_output_ids == vllm_output_ids, (
103
- f"Test{i}:\nHF: {hf_output_ids}\nvLLM: {vllm_output_ids}")
99
+ check_outputs_equal(
100
+ outputs_0_lst=hf_outputs,
101
+ outputs_1_lst=vllm_outputs,
102
+ name_0="hf",
103
+ name_1="vllm",
104
+ )
105
+
106
assert ("is preempted by PreemptionMode.RECOMPUTE mode because there "
107
"is not enough KV cache space." in caplog_vllm.text)
108
# Ensure the count bucket of request-level histogram metrics matches
tests/distributed/test_basic_distributed_correctness.py
+8
-7
@@ -17,6 +17,8 @@ import os
17
import pytest
18
import torch
19
20
+from ..models.utils import check_outputs_equal
21
+
22
MODELS = [
23
os.environ["TEST_DIST_MODEL"],
24
]
@@ -48,10 +50,9 @@ def test_models(
50
) as vllm_model:
51
vllm_outputs = vllm_model.generate_greedy(example_prompts, max_tokens)
52
51
- for i in range(len(example_prompts)):
52
- hf_output_ids, hf_output_str = hf_outputs[i]
53
- vllm_output_ids, vllm_output_str = vllm_outputs[i]
54
- assert hf_output_str == vllm_output_str, (
55
- f"Test{i}:\nHF: {hf_output_str!r}\nvLLM: {vllm_output_str!r}")
56
- assert hf_output_ids == vllm_output_ids, (
57
- f"Test{i}:\nHF: {hf_output_ids}\nvLLM: {vllm_output_ids}")
53
+ check_outputs_equal(
54
+ outputs_0_lst=hf_outputs,
55
+ outputs_1_lst=vllm_outputs,
56
+ name_0="hf",
57
+ name_1="vllm",
58
+ )
tests/distributed/test_chunked_prefill_distributed.py
+8
-7
@@ -16,6 +16,8 @@ import os
16
import pytest
17
import torch
18
19
+from ..models.utils import check_outputs_equal
20
+
21
MODELS = [
22
os.environ["TEST_DIST_MODEL"],
23
]
@@ -59,10 +61,9 @@ def test_models(
61
) as vllm_model:
62
vllm_outputs = vllm_model.generate_greedy(example_prompts, max_tokens)
63
62
- for i in range(len(example_prompts)):
63
- hf_output_ids, hf_output_str = hf_outputs[i]
64
- vllm_output_ids, vllm_output_str = vllm_outputs[i]
65
- assert hf_output_str == vllm_output_str, (
66
- f"Test{i}:\nHF: {hf_output_str!r}\nvLLM: {vllm_output_str!r}")
67
- assert hf_output_ids == vllm_output_ids, (
68
- f"Test{i}:\nHF: {hf_output_ids}\nvLLM: {vllm_output_ids}")
64
+ check_outputs_equal(
65
+ outputs_0_lst=hf_outputs,
66
+ outputs_1_lst=vllm_outputs,
67
+ name_0="hf",
68
+ name_1="vllm",
69
+ )
tests/models/test_big_models.py
+8
-7
@@ -7,6 +7,8 @@ Run `pytest tests/models/test_big_models.py`.
7
import pytest
8
import torch
9
10
+from .utils import check_outputs_equal
11
+
12
MODELS = [
13
"meta-llama/Llama-2-7b-hf",
14
# "mistralai/Mistral-7B-v0.1", # Tested by test_mistral.py
@@ -40,13 +42,12 @@ def test_models(
42
with vllm_runner(model, dtype=dtype) as vllm_model:
43
vllm_outputs = vllm_model.generate_greedy(example_prompts, max_tokens)
44
43
- for i in range(len(example_prompts)):
44
- hf_output_ids, hf_output_str = hf_outputs[i]
45
- vllm_output_ids, vllm_output_str = vllm_outputs[i]
46
- assert hf_output_str == vllm_output_str, (
47
- f"Test{i}:\nHF: {hf_output_str!r}\nvLLM: {vllm_output_str!r}")
48
- assert hf_output_ids == vllm_output_ids, (
49
- f"Test{i}:\nHF: {hf_output_ids}\nvLLM: {vllm_output_ids}")
45
+ check_outputs_equal(
46
+ outputs_0_lst=hf_outputs,
47
+ outputs_1_lst=vllm_outputs,
48
+ name_0="hf",
49
+ name_1="vllm",
50
+ )
51
52
53
@pytest.mark.parametrize("model", MODELS)
tests/models/test_llava.py
+10
-8
@@ -6,6 +6,7 @@ from transformers import AutoTokenizer
6
from vllm.config import VisionLanguageConfig
7
8
from ..conftest import IMAGE_ASSETS, HfRunner, VllmRunner, _ImageAssets
9
+from .utils import check_outputs_equal
10
11
pytestmark = pytest.mark.vlm
12
@@ -109,14 +110,15 @@ def run_test(
110
max_tokens,
111
images=vllm_images)
112
112
- for i in range(len(HF_IMAGE_PROMPTS)):
113
- hf_output_ids, hf_output_str = hf_outputs[i]
114
- vllm_output_ids, vllm_output_str = vllm_to_hf_output(
115
- vllm_outputs[i], vlm_config, model_id)
116
- assert hf_output_str == vllm_output_str, (
117
- f"Test{i}:\nHF: {hf_output_str!r}\nvLLM: {vllm_output_str!r}")
118
- assert hf_output_ids == vllm_output_ids, (
119
- f"Test{i}:\nHF: {hf_output_ids}\nvLLM: {vllm_output_ids}")
113
+ check_outputs_equal(
114
+ hf_outputs,
115
+ [
116
+ vllm_to_hf_output(vllm_output, vlm_config, model_id)
117
+ for vllm_output in vllm_outputs
118
+ ],
119
+ name_0="hf",
120
+ name_1="vllm",
121
+ )
122
123
124
@pytest.mark.parametrize("model_and_config", model_and_vl_config)
tests/models/test_llava_next.py
+10
-8
@@ -6,6 +6,7 @@ from transformers import AutoTokenizer
6
from vllm.config import VisionLanguageConfig
7
8
from ..conftest import IMAGE_ASSETS
9
+from .utils import check_outputs_equal
10
11
pytestmark = pytest.mark.vlm
12
@@ -115,11 +116,12 @@ def test_models(hf_runner, vllm_runner, image_assets, model_and_config,
116
max_tokens,
117
images=vllm_images)
118
118
- for i in range(len(HF_IMAGE_PROMPTS)):
119
- hf_output_ids, hf_output_str = hf_outputs[i]
120
- vllm_output_ids, vllm_output_str = vllm_to_hf_output(
121
- vllm_outputs[i], vlm_config, model_id)
122
- assert hf_output_str == vllm_output_str, (
123
- f"Test{i}:\nHF: {hf_output_str!r}\nvLLM: {vllm_output_str!r}")
124
- assert hf_output_ids == vllm_output_ids, (
125
- f"Test{i}:\nHF: {hf_output_ids}\nvLLM: {vllm_output_ids}")
119
+ check_outputs_equal(
120
+ hf_outputs,
121
+ [
122
+ vllm_to_hf_output(vllm_output, vlm_config, model_id)
123
+ for vllm_output in vllm_outputs
124
+ ],
125
+ name_0="hf",
126
+ name_1="vllm",
127
+ )
tests/models/test_models.py
+8
-7
@@ -7,6 +7,8 @@ Run `pytest tests/models/test_models.py`.
7
"""
8
import pytest
9
10
+from .utils import check_outputs_equal
11
+
12
MODELS = [
13
"facebook/opt-125m",
14
"gpt2",
@@ -41,13 +43,12 @@ def test_models(
43
with vllm_runner(model, dtype=dtype) as vllm_model:
44
vllm_outputs = vllm_model.generate_greedy(example_prompts, max_tokens)
45
44
- for i in range(len(example_prompts)):
45
- hf_output_ids, hf_output_str = hf_outputs[i]
46
- vllm_output_ids, vllm_output_str = vllm_outputs[i]
47
- assert hf_output_str == vllm_output_str, (
48
- f"Test{i}:\nHF: {hf_output_str!r}\nvLLM: {vllm_output_str!r}")
49
- assert hf_output_ids == vllm_output_ids, (
50
- f"Test{i}:\nHF: {hf_output_ids}\nvLLM: {vllm_output_ids}")
46
+ check_outputs_equal(
47
+ outputs_0_lst=hf_outputs,
48
+ outputs_1_lst=vllm_outputs,
49
+ name_0="hf",
50
+ name_1="vllm",
51
+ )
52
53
54
@pytest.mark.parametrize("model", MODELS)
tests/models/test_phi3v.py
+10
-8
@@ -7,6 +7,7 @@ from vllm.config import VisionLanguageConfig
7
from vllm.utils import is_cpu
8
9
from ..conftest import IMAGE_ASSETS, HfRunner, VllmRunner, _ImageAssets
10
+from .utils import check_outputs_equal
11
12
pytestmark = pytest.mark.vlm
13
@@ -124,14 +125,15 @@ def run_test(
125
max_tokens,
126
images=vllm_images)
127
127
- for i in range(len(HF_IMAGE_PROMPTS)):
128
- hf_output_ids, hf_output_str = hf_outputs[i]
129
- vllm_output_ids, vllm_output_str = vllm_to_hf_output(
130
- vllm_outputs[i], vlm_config, model_id)
131
- assert hf_output_str == vllm_output_str, (
132
- f"Test{i}:\nHF: {hf_output_str!r}\nvLLM: {vllm_output_str!r}")
133
- assert hf_output_ids == vllm_output_ids, (
134
- f"Test{i}:\nHF: {hf_output_ids}\nvLLM: {vllm_output_ids}")
128
+ check_outputs_equal(
129
+ hf_outputs,
130
+ [
131
+ vllm_to_hf_output(vllm_output, vlm_config, model_id)
132
+ for vllm_output in vllm_outputs
133
+ ],
134
+ name_0="hf",
135
+ name_1="vllm",
136
+ )
137
138
139
# Since we use _attn_implementation="eager" for hf_runner, here is
tests/models/utils.py
+38
-2
@@ -1,7 +1,43 @@
1
-def check_logprobs_close(outputs_0_lst, outputs_1_lst, name_0, name_1):
2
- """Compare the logprobs of two sequences generated by different models,
1
+from typing import Dict, List, Tuple
2
+
3
+TokensText = Tuple[List[int], str]
4
+
5
+
6
+def check_outputs_equal(outputs_0_lst: List[TokensText],
7
+ outputs_1_lst: List[TokensText], name_0: str,
8
+ name_1: str):
9
+ """
10
+ Compare the two sequences generated by different models,
11
+ which should be equal.
12
+ """
13
+ assert len(outputs_0_lst) == len(outputs_1_lst)
14
+
15
+ for prompt_idx, (outputs_0,
16
+ outputs_1) in enumerate(zip(outputs_0_lst,
17
+ outputs_1_lst)):
18
+ output_ids_0, output_str_0 = outputs_0
19
+ output_ids_1, output_str_1 = outputs_1
20
+
21
+ assert output_str_0 == output_str_1, (f"Test{prompt_idx}:"
22
+ f"\n{name_0}:\t{output_str_0!r}"
23
+ f"\n{name_1}:\t{output_str_1!r}")
24
+ assert output_ids_0 == output_ids_1, (f"Test{prompt_idx}:"
25
+ f"\n{name_0}:\t{output_str_0!r}"
26
+ f"\n{name_1}:\t{output_str_1!r}")
27
+
28
+
29
+TokensTextLogprobs = Tuple[List[int], str, List[Dict[int, float]]]
30
+
31
+
32
+def check_logprobs_close(outputs_0_lst: List[TokensTextLogprobs],
33
+ outputs_1_lst: List[TokensTextLogprobs], name_0: str,
34
+ name_1: str):
35
+ """
36
+ Compare the logprobs of two sequences generated by different models,
37
which should be similar but not necessarily equal.
38
"""
39
+ assert len(outputs_0_lst) == len(outputs_1_lst)
40
+
41
# Loop through responses to each prompt.
42
for prompt_idx, (outputs_0,
43
outputs_1) in enumerate(zip(outputs_0_lst,