@tduncan / Nvidia-Optimization-Stack-VLLM Mirror · read-only / commits / c2a8ac75e0
Fork 0

A high-throughput and memory-efficient inference and serving engine for LLMs

Mirrored from https://github.com/tduncan-lum/Nvidia-Optimization-Stack---VLLM.git

Synced Nov 16, 23:15 UTC

[CI/Build] Add E2E tests for MLPSpeculator (#5791)

Signed-off-by: Thomas Parnell <tpa@zurich.ibm.com>

Thomas Parnell committed Jun 26, 2024 at 01:04 UTC c2a8ac75e03aec19dad397a8e64377d37c67239a
1 file changed +216
tests/spec_decode/e2e/test_mlp_correctness.py new
+216
@@ -0,0 +1,216 @@
1 +"""This docstring details important information on the testing methodology.
2 +
3 +Most of the tests rely on "greedy equality", where we expect the output of
4 +speculative decoding on a sequence to exactly match the output of normal non-
5 +speculative decoding.
6 +
7 +Since speculative decoding with rejection sampling guarantees that the output
8 +distribution matches the target model's output distribution (up to hardware
9 +numerics, see https://arxiv.org/pdf/2302.01318.pdf), we can expect greedy
10 +equality.
11 +
12 +However, we still need to verify below scenario could be passed:
13 + * Batch size 1 greedy equality
14 + * Batch size >1 greedy equality
15 + * Test greedy equality under preemption
16 + * Test greedy equality under various number of speculative tokens.
17 +
18 +With those tests, we can say at least, MLPSpeculator would not break the
19 +correctess for the target model outputs.
20 +"""
21 +
22 +import pytest
23 +
24 +from .conftest import run_greedy_equality_correctness_test
25 +
26 +# main model
27 +MAIN_MODEL = "ibm-granite/granite-3b-code-instruct"
28 +
29 +# speculative model
30 +SPEC_MODEL = "ibm-granite/granite-3b-code-instruct-accelerator"
31 +
32 +# max. number of speculative tokens: this corresponds to
33 +# n_predict in the config.json of the speculator model.
34 +MAX_SPEC_TOKENS = 5
35 +
36 +# precision
37 +PRECISION = "float16"
38 +
39 +
40 +@pytest.mark.parametrize(
41 + "common_llm_kwargs",
42 + [{
43 + # Skip cuda graph recording for fast test.
44 + "enforce_eager": True,
45 +
46 + # Required for spec decode.
47 + "use_v2_block_manager": True,
48 +
49 + # Print spec metrics.
50 + "disable_log_stats": False,
51 +
52 + # Precision
53 + "dtype": PRECISION,
54 +
55 + # Main model
56 + "model": MAIN_MODEL,
57 + }])
58 +@pytest.mark.parametrize("per_test_common_llm_kwargs", [{}])
59 +@pytest.mark.parametrize("baseline_llm_kwargs", [{}])
60 +@pytest.mark.parametrize("test_llm_kwargs", [
61 + {
62 + "speculative_model": SPEC_MODEL,
63 + },
64 +])
65 +@pytest.mark.parametrize("output_len", [
66 + 128,
67 +])
68 +@pytest.mark.parametrize("batch_size", [1, 32])
69 +@pytest.mark.parametrize("seed", [1])
70 +def test_mlp_e2e_greedy_correctness(baseline_llm_generator, test_llm_generator,
71 + batch_size: int, output_len: int):
72 + """Verify greedy equality with different batch size."""
73 + run_greedy_equality_correctness_test(baseline_llm_generator,
74 + test_llm_generator,
75 + batch_size,
76 + max_output_len=output_len,
77 + force_output_len=True)
78 +
79 +
80 +@pytest.mark.parametrize(
81 + "common_llm_kwargs",
82 + [{
83 + "block_size": 8,
84 + # 2 for small prompt, 256//8 for generated.
85 + "num_gpu_blocks_override": 2 + 256 // 8,
86 + "max_model_len": (2 + 256 // 8) * 8,
87 +
88 + # Skip cuda graph recording for fast test.
89 + "enforce_eager": True,
90 +
91 + # Required for spec decode.
92 + "use_v2_block_manager": True,
93 +
94 + # Precision
95 + "dtype": PRECISION,
96 +
97 + # Main model
98 + "model": MAIN_MODEL,
99 + }])
100 +@pytest.mark.parametrize("per_test_common_llm_kwargs", [{}])
101 +@pytest.mark.parametrize("baseline_llm_kwargs", [{}])
102 +@pytest.mark.parametrize("test_llm_kwargs", [
103 + {
104 + "speculative_model": SPEC_MODEL,
105 + },
106 +])
107 +@pytest.mark.parametrize(
108 + "output_len",
109 + [
110 + # Use small output len for fast test.
111 + 128,
112 + ])
113 +@pytest.mark.parametrize("batch_size", [4])
114 +@pytest.mark.parametrize("seed", [1])
115 +def test_mlp_e2e_greedy_correctness_with_preemption(baseline_llm_generator,
116 + test_llm_generator,
117 + batch_size: int,
118 + output_len: int):
119 + """Verify greedy equality, even when some sequences are preempted mid-
120 + generation.
121 + """
122 + run_greedy_equality_correctness_test(baseline_llm_generator,
123 + test_llm_generator,
124 + batch_size,
125 + max_output_len=output_len,
126 + force_output_len=True)
127 +
128 +
129 +@pytest.mark.parametrize(
130 + "common_llm_kwargs",
131 + [{
132 + # Skip cuda graph recording for fast test.
133 + "enforce_eager": True,
134 +
135 + # Required for spec decode.
136 + "use_v2_block_manager": True,
137 +
138 + # Precision
139 + "dtype": PRECISION,
140 +
141 + # Main model
142 + "model": MAIN_MODEL,
143 + }])
144 +@pytest.mark.parametrize("per_test_common_llm_kwargs", [{}])
145 +@pytest.mark.parametrize("baseline_llm_kwargs", [{}])
146 +@pytest.mark.parametrize(
147 + "test_llm_kwargs",
148 + [
149 + {
150 + "speculative_model": SPEC_MODEL,
151 + "num_speculative_tokens": k,
152 + }
153 + # Try a range of num. speculative tokens
154 + for k in range(1, 1 + MAX_SPEC_TOKENS)
155 + ])
156 +@pytest.mark.parametrize("batch_size", [2])
157 +@pytest.mark.parametrize(
158 + "output_len",
159 + [
160 + # Use smaller output len for fast test.
161 + 32,
162 + ])
163 +@pytest.mark.parametrize("seed", [1])
164 +def test_mlp_different_k(baseline_llm_generator, test_llm_generator,
165 + batch_size: int, output_len: int):
166 + """Verify that mlp speculative decoding produces exact equality
167 + to without spec decode with different values of num_speculative_tokens.
168 + """
169 + run_greedy_equality_correctness_test(baseline_llm_generator,
170 + test_llm_generator,
171 + batch_size,
172 + max_output_len=output_len,
173 + force_output_len=True)
174 +
175 +
176 +@pytest.mark.parametrize(
177 + "common_llm_kwargs",
178 + [{
179 + # Skip cuda graph recording for fast test.
180 + "enforce_eager": True,
181 +
182 + # Required for spec decode.
183 + "use_v2_block_manager": True,
184 +
185 + # Precision
186 + "dtype": PRECISION,
187 +
188 + # Main model
189 + "model": MAIN_MODEL,
190 + }])
191 +@pytest.mark.parametrize("per_test_common_llm_kwargs", [{}])
192 +@pytest.mark.parametrize("baseline_llm_kwargs", [{}])
193 +@pytest.mark.parametrize("test_llm_kwargs",
194 + [{
195 + "speculative_model": SPEC_MODEL,
196 + "speculative_disable_by_batch_size": 4
197 + }])
198 +@pytest.mark.parametrize("batch_size", [1, 5])
199 +@pytest.mark.parametrize(
200 + "output_len",
201 + [
202 + # Use smaller output len for fast test.
203 + 32,
204 + ])
205 +@pytest.mark.parametrize("seed", [1])
206 +def test_mlp_disable_queue(baseline_llm_generator, test_llm_generator,
207 + batch_size: int, output_len: int):
208 + """Verify that mlp speculative decoding produces exact equality
209 + to without spec decode when speculation is disabled for large
210 + batch sizes.
211 + """
212 + run_greedy_equality_correctness_test(baseline_llm_generator,
213 + test_llm_generator,
214 + batch_size,
215 + max_output_len=output_len,
216 + force_output_len=True)