@tduncan / Nvidia-Optimization-Stack-VLLM Mirror · read-only / commits / cff6a1fec1
Fork 0

A high-throughput and memory-efficient inference and serving engine for LLMs

Mirrored from https://github.com/tduncan-lum/Nvidia-Optimization-Stack---VLLM.git

Synced Nov 16, 22:00 UTC

[CI/Build] Reuse code for checking output consistency (#5988)

Cyrus Leung committed Jun 30, 2024 at 11:44 UTC cff6a1fec15dba524c162d20a4e8de4df2b0a3d5
11 files changed +125 -75
tests/basic_correctness/test_basic_correctness.py
+8 -7
@@ -8,6 +8,8 @@ import pytest
8
9 from vllm import LLM
10
11 +from ..models.utils import check_outputs_equal
12 +
13 MODELS = [
14 "facebook/opt-125m",
15 "meta-llama/Llama-2-7b-hf",
@@ -46,10 +48,9 @@ def test_models(
48 gpu_memory_utilization=0.7) as vllm_model:
49 vllm_outputs = vllm_model.generate_greedy(example_prompts, max_tokens)
50
49 - for i in range(len(example_prompts)):
50 - hf_output_ids, hf_output_str = hf_outputs[i]
51 - vllm_output_ids, vllm_output_str = vllm_outputs[i]
52 - assert hf_output_str == vllm_output_str, (
53 - f"Test{i}:\nHF: {hf_output_str!r}\nvLLM: {vllm_output_str!r}")
54 - assert hf_output_ids == vllm_output_ids, (
55 - f"Test{i}:\nHF: {hf_output_ids}\nvLLM: {vllm_output_ids}")
51 + check_outputs_equal(
52 + outputs_0_lst=hf_outputs,
53 + outputs_1_lst=vllm_outputs,
54 + name_0="hf",
55 + name_1="vllm",
56 + )
tests/basic_correctness/test_chunked_prefill.py
+8 -7
@@ -8,6 +8,8 @@ Run `pytest tests/models/test_chunked_prefill.py`.
8 """
9 import pytest
10
11 +from ..models.utils import check_outputs_equal
12 +
13 MODELS = [
14 "facebook/opt-125m",
15 "meta-llama/Llama-2-7b-hf",
@@ -54,10 +56,9 @@ def test_models(
56 ) as vllm_model:
57 vllm_outputs = vllm_model.generate_greedy(example_prompts, max_tokens)
58
57 - for i in range(len(example_prompts)):
58 - hf_output_ids, hf_output_str = hf_outputs[i]
59 - vllm_output_ids, vllm_output_str = vllm_outputs[i]
60 - assert hf_output_str == vllm_output_str, (
61 - f"Test{i}:\nHF: {hf_output_str!r}\nvLLM: {vllm_output_str!r}")
62 - assert hf_output_ids == vllm_output_ids, (
63 - f"Test{i}:\nHF: {hf_output_ids}\nvLLM: {vllm_output_ids}")
59 + check_outputs_equal(
60 + outputs_0_lst=hf_outputs,
61 + outputs_1_lst=vllm_outputs,
62 + name_0="hf",
63 + name_1="vllm",
64 + )
tests/basic_correctness/test_preemption.py
+9 -7
@@ -12,6 +12,8 @@ from vllm import SamplingParams
12 from vllm.core.scheduler import (ARTIFICIAL_PREEMPTION_MAX_CNT,
13 ENABLE_ARTIFICIAL_PREEMPT)
14
15 +from ..models.utils import check_outputs_equal
16 +
17 MODELS = [
18 "facebook/opt-125m",
19 ]
@@ -94,13 +96,13 @@ def test_preemption(
96 total_preemption = (
97 vllm_model.model.llm_engine.scheduler.num_cumulative_preemption)
98
97 - for i in range(len(example_prompts)):
98 - hf_output_ids, hf_output_str = hf_outputs[i]
99 - vllm_output_ids, vllm_output_str = vllm_outputs[i]
100 - assert hf_output_str == vllm_output_str, (
101 - f"Test{i}:\nHF: {hf_output_str!r}\nvLLM: {vllm_output_str!r}")
102 - assert hf_output_ids == vllm_output_ids, (
103 - f"Test{i}:\nHF: {hf_output_ids}\nvLLM: {vllm_output_ids}")
99 + check_outputs_equal(
100 + outputs_0_lst=hf_outputs,
101 + outputs_1_lst=vllm_outputs,
102 + name_0="hf",
103 + name_1="vllm",
104 + )
105 +
106 assert ("is preempted by PreemptionMode.RECOMPUTE mode because there "
107 "is not enough KV cache space." in caplog_vllm.text)
108 # Ensure the count bucket of request-level histogram metrics matches
tests/distributed/test_basic_distributed_correctness.py
+8 -7
@@ -17,6 +17,8 @@ import os
17 import pytest
18 import torch
19
20 +from ..models.utils import check_outputs_equal
21 +
22 MODELS = [
23 os.environ["TEST_DIST_MODEL"],
24 ]
@@ -48,10 +50,9 @@ def test_models(
50 ) as vllm_model:
51 vllm_outputs = vllm_model.generate_greedy(example_prompts, max_tokens)
52
51 - for i in range(len(example_prompts)):
52 - hf_output_ids, hf_output_str = hf_outputs[i]
53 - vllm_output_ids, vllm_output_str = vllm_outputs[i]
54 - assert hf_output_str == vllm_output_str, (
55 - f"Test{i}:\nHF: {hf_output_str!r}\nvLLM: {vllm_output_str!r}")
56 - assert hf_output_ids == vllm_output_ids, (
57 - f"Test{i}:\nHF: {hf_output_ids}\nvLLM: {vllm_output_ids}")
53 + check_outputs_equal(
54 + outputs_0_lst=hf_outputs,
55 + outputs_1_lst=vllm_outputs,
56 + name_0="hf",
57 + name_1="vllm",
58 + )
tests/distributed/test_chunked_prefill_distributed.py
+8 -7
@@ -16,6 +16,8 @@ import os
16 import pytest
17 import torch
18
19 +from ..models.utils import check_outputs_equal
20 +
21 MODELS = [
22 os.environ["TEST_DIST_MODEL"],
23 ]
@@ -59,10 +61,9 @@ def test_models(
61 ) as vllm_model:
62 vllm_outputs = vllm_model.generate_greedy(example_prompts, max_tokens)
63
62 - for i in range(len(example_prompts)):
63 - hf_output_ids, hf_output_str = hf_outputs[i]
64 - vllm_output_ids, vllm_output_str = vllm_outputs[i]
65 - assert hf_output_str == vllm_output_str, (
66 - f"Test{i}:\nHF: {hf_output_str!r}\nvLLM: {vllm_output_str!r}")
67 - assert hf_output_ids == vllm_output_ids, (
68 - f"Test{i}:\nHF: {hf_output_ids}\nvLLM: {vllm_output_ids}")
64 + check_outputs_equal(
65 + outputs_0_lst=hf_outputs,
66 + outputs_1_lst=vllm_outputs,
67 + name_0="hf",
68 + name_1="vllm",
69 + )
tests/models/test_big_models.py
+8 -7
@@ -7,6 +7,8 @@ Run `pytest tests/models/test_big_models.py`.
7 import pytest
8 import torch
9
10 +from .utils import check_outputs_equal
11 +
12 MODELS = [
13 "meta-llama/Llama-2-7b-hf",
14 # "mistralai/Mistral-7B-v0.1", # Tested by test_mistral.py
@@ -40,13 +42,12 @@ def test_models(
42 with vllm_runner(model, dtype=dtype) as vllm_model:
43 vllm_outputs = vllm_model.generate_greedy(example_prompts, max_tokens)
44
43 - for i in range(len(example_prompts)):
44 - hf_output_ids, hf_output_str = hf_outputs[i]
45 - vllm_output_ids, vllm_output_str = vllm_outputs[i]
46 - assert hf_output_str == vllm_output_str, (
47 - f"Test{i}:\nHF: {hf_output_str!r}\nvLLM: {vllm_output_str!r}")
48 - assert hf_output_ids == vllm_output_ids, (
49 - f"Test{i}:\nHF: {hf_output_ids}\nvLLM: {vllm_output_ids}")
45 + check_outputs_equal(
46 + outputs_0_lst=hf_outputs,
47 + outputs_1_lst=vllm_outputs,
48 + name_0="hf",
49 + name_1="vllm",
50 + )
51
52
53 @pytest.mark.parametrize("model", MODELS)
tests/models/test_llava.py
+10 -8
@@ -6,6 +6,7 @@ from transformers import AutoTokenizer
6 from vllm.config import VisionLanguageConfig
7
8 from ..conftest import IMAGE_ASSETS, HfRunner, VllmRunner, _ImageAssets
9 +from .utils import check_outputs_equal
10
11 pytestmark = pytest.mark.vlm
12
@@ -109,14 +110,15 @@ def run_test(
110 max_tokens,
111 images=vllm_images)
112
112 - for i in range(len(HF_IMAGE_PROMPTS)):
113 - hf_output_ids, hf_output_str = hf_outputs[i]
114 - vllm_output_ids, vllm_output_str = vllm_to_hf_output(
115 - vllm_outputs[i], vlm_config, model_id)
116 - assert hf_output_str == vllm_output_str, (
117 - f"Test{i}:\nHF: {hf_output_str!r}\nvLLM: {vllm_output_str!r}")
118 - assert hf_output_ids == vllm_output_ids, (
119 - f"Test{i}:\nHF: {hf_output_ids}\nvLLM: {vllm_output_ids}")
113 + check_outputs_equal(
114 + hf_outputs,
115 + [
116 + vllm_to_hf_output(vllm_output, vlm_config, model_id)
117 + for vllm_output in vllm_outputs
118 + ],
119 + name_0="hf",
120 + name_1="vllm",
121 + )
122
123
124 @pytest.mark.parametrize("model_and_config", model_and_vl_config)
tests/models/test_llava_next.py
+10 -8
@@ -6,6 +6,7 @@ from transformers import AutoTokenizer
6 from vllm.config import VisionLanguageConfig
7
8 from ..conftest import IMAGE_ASSETS
9 +from .utils import check_outputs_equal
10
11 pytestmark = pytest.mark.vlm
12
@@ -115,11 +116,12 @@ def test_models(hf_runner, vllm_runner, image_assets, model_and_config,
116 max_tokens,
117 images=vllm_images)
118
118 - for i in range(len(HF_IMAGE_PROMPTS)):
119 - hf_output_ids, hf_output_str = hf_outputs[i]
120 - vllm_output_ids, vllm_output_str = vllm_to_hf_output(
121 - vllm_outputs[i], vlm_config, model_id)
122 - assert hf_output_str == vllm_output_str, (
123 - f"Test{i}:\nHF: {hf_output_str!r}\nvLLM: {vllm_output_str!r}")
124 - assert hf_output_ids == vllm_output_ids, (
125 - f"Test{i}:\nHF: {hf_output_ids}\nvLLM: {vllm_output_ids}")
119 + check_outputs_equal(
120 + hf_outputs,
121 + [
122 + vllm_to_hf_output(vllm_output, vlm_config, model_id)
123 + for vllm_output in vllm_outputs
124 + ],
125 + name_0="hf",
126 + name_1="vllm",
127 + )
tests/models/test_models.py
+8 -7
@@ -7,6 +7,8 @@ Run `pytest tests/models/test_models.py`.
7 """
8 import pytest
9
10 +from .utils import check_outputs_equal
11 +
12 MODELS = [
13 "facebook/opt-125m",
14 "gpt2",
@@ -41,13 +43,12 @@ def test_models(
43 with vllm_runner(model, dtype=dtype) as vllm_model:
44 vllm_outputs = vllm_model.generate_greedy(example_prompts, max_tokens)
45
44 - for i in range(len(example_prompts)):
45 - hf_output_ids, hf_output_str = hf_outputs[i]
46 - vllm_output_ids, vllm_output_str = vllm_outputs[i]
47 - assert hf_output_str == vllm_output_str, (
48 - f"Test{i}:\nHF: {hf_output_str!r}\nvLLM: {vllm_output_str!r}")
49 - assert hf_output_ids == vllm_output_ids, (
50 - f"Test{i}:\nHF: {hf_output_ids}\nvLLM: {vllm_output_ids}")
46 + check_outputs_equal(
47 + outputs_0_lst=hf_outputs,
48 + outputs_1_lst=vllm_outputs,
49 + name_0="hf",
50 + name_1="vllm",
51 + )
52
53
54 @pytest.mark.parametrize("model", MODELS)
tests/models/test_phi3v.py
+10 -8
@@ -7,6 +7,7 @@ from vllm.config import VisionLanguageConfig
7 from vllm.utils import is_cpu
8
9 from ..conftest import IMAGE_ASSETS, HfRunner, VllmRunner, _ImageAssets
10 +from .utils import check_outputs_equal
11
12 pytestmark = pytest.mark.vlm
13
@@ -124,14 +125,15 @@ def run_test(
125 max_tokens,
126 images=vllm_images)
127
127 - for i in range(len(HF_IMAGE_PROMPTS)):
128 - hf_output_ids, hf_output_str = hf_outputs[i]
129 - vllm_output_ids, vllm_output_str = vllm_to_hf_output(
130 - vllm_outputs[i], vlm_config, model_id)
131 - assert hf_output_str == vllm_output_str, (
132 - f"Test{i}:\nHF: {hf_output_str!r}\nvLLM: {vllm_output_str!r}")
133 - assert hf_output_ids == vllm_output_ids, (
134 - f"Test{i}:\nHF: {hf_output_ids}\nvLLM: {vllm_output_ids}")
128 + check_outputs_equal(
129 + hf_outputs,
130 + [
131 + vllm_to_hf_output(vllm_output, vlm_config, model_id)
132 + for vllm_output in vllm_outputs
133 + ],
134 + name_0="hf",
135 + name_1="vllm",
136 + )
137
138
139 # Since we use _attn_implementation="eager" for hf_runner, here is
tests/models/utils.py
+38 -2
@@ -1,7 +1,43 @@
1 -def check_logprobs_close(outputs_0_lst, outputs_1_lst, name_0, name_1):
2 - """Compare the logprobs of two sequences generated by different models,
1 +from typing import Dict, List, Tuple
2 +
3 +TokensText = Tuple[List[int], str]
4 +
5 +
6 +def check_outputs_equal(outputs_0_lst: List[TokensText],
7 + outputs_1_lst: List[TokensText], name_0: str,
8 + name_1: str):
9 + """
10 + Compare the two sequences generated by different models,
11 + which should be equal.
12 + """
13 + assert len(outputs_0_lst) == len(outputs_1_lst)
14 +
15 + for prompt_idx, (outputs_0,
16 + outputs_1) in enumerate(zip(outputs_0_lst,
17 + outputs_1_lst)):
18 + output_ids_0, output_str_0 = outputs_0
19 + output_ids_1, output_str_1 = outputs_1
20 +
21 + assert output_str_0 == output_str_1, (f"Test{prompt_idx}:"
22 + f"\n{name_0}:\t{output_str_0!r}"
23 + f"\n{name_1}:\t{output_str_1!r}")
24 + assert output_ids_0 == output_ids_1, (f"Test{prompt_idx}:"
25 + f"\n{name_0}:\t{output_str_0!r}"
26 + f"\n{name_1}:\t{output_str_1!r}")
27 +
28 +
29 +TokensTextLogprobs = Tuple[List[int], str, List[Dict[int, float]]]
30 +
31 +
32 +def check_logprobs_close(outputs_0_lst: List[TokensTextLogprobs],
33 + outputs_1_lst: List[TokensTextLogprobs], name_0: str,
34 + name_1: str):
35 + """
36 + Compare the logprobs of two sequences generated by different models,
37 which should be similar but not necessarily equal.
38 """
39 + assert len(outputs_0_lst) == len(outputs_1_lst)
40 +
41 # Loop through responses to each prompt.
42 for prompt_idx, (outputs_0,
43 outputs_1) in enumerate(zip(outputs_0_lst,