@tduncan / Nvidia-Optimization-Stack-VLLM Mirror · read-only / commits / 775f00f81e
Fork 0

A high-throughput and memory-efficient inference and serving engine for LLMs

Mirrored from https://github.com/tduncan-lum/Nvidia-Optimization-Stack---VLLM.git

Synced Nov 16, 17:00 UTC

[Speculative Decoding] Test refactor (#8317)

Co-authored-by: youkaichao <youkaichao@126.com>

Lily Liu committed Sep 11, 2024 at 14:07 UTC 775f00f81e4f5a12b17816d39261c628e2f36683
12 files changed +929 -1044
.buildkite/test-pipeline.yaml
+2 -1
@@ -217,7 +217,8 @@ steps:
217 commands:
218 # See https://github.com/vllm-project/vllm/issues/5152
219 - export VLLM_ATTENTION_BACKEND=XFORMERS
220 - - pytest -v -s spec_decode
220 + - pytest -v -s spec_decode/e2e/test_multistep_correctness.py
221 + - pytest -v -s spec_decode --ignore=spec_decode/e2e/test_multistep_correctness.py
222
223 - label: LoRA Test %N # 30min each
224 mirror_hardwares: [amd]
tests/spec_decode/e2e/conftest.py
+176 -299
@@ -1,224 +1,54 @@
1 -import asyncio
2 -import os
1 from itertools import cycle
4 -from typing import Dict, List, Optional, Sequence, Tuple, Union
2 +from typing import List, Optional, Tuple
3
4 import pytest
7 -import ray
8 -import torch
5
10 -from vllm import LLM
11 -from vllm.engine.arg_utils import AsyncEngineArgs
12 -from vllm.engine.async_llm_engine import AsyncLLMEngine
13 -from vllm.lora.request import LoRARequest
6 +from vllm import LLM, SamplingParams
7 from vllm.model_executor.utils import set_random_seed
15 -from vllm.multimodal import MultiModalDataDict
16 -from vllm.outputs import RequestOutput
17 -from vllm.prompt_adapter.request import PromptAdapterRequest
18 -from vllm.sampling_params import SamplingParams
19 -from vllm.sequence import Logprob
20 -from vllm.usage.usage_lib import UsageContext
21 -from vllm.utils import Counter, random_uuid
8
9 from ...conftest import cleanup
24 -from ...utils import wait_for_gpu_memory_to_clear
10 +from ...models.utils import check_logprobs_close, check_outputs_equal
11 +from ...utils import RemoteOpenAIServer
12
26 -
27 -class AsyncLLM:
28 - """AsyncLLM
29 -
30 - Note: Current LLM class in vllm don't support async mode, for test purpose,
31 - we implement async one in here. Maybe we could move to
32 - vllm/entrypoints/llm.py in future.
33 -
34 - Below AsyncLLM is directly borrow from vllm/entrypoints/llm.py with changes
35 - to make to work in async mode.
36 - """
37 -
38 - def __init__(
39 - self,
40 - model: str,
41 - tokenizer: Optional[str] = None,
42 - tokenizer_mode: str = "auto",
43 - skip_tokenizer_init: bool = False,
44 - trust_remote_code: bool = False,
45 - tensor_parallel_size: int = 1,
46 - dtype: str = "auto",
47 - quantization: Optional[str] = None,
48 - revision: Optional[str] = None,
49 - tokenizer_revision: Optional[str] = None,
50 - seed: int = 0,
51 - gpu_memory_utilization: float = 0.9,
52 - swap_space: int = 4,
53 - enforce_eager: bool = False,
54 - max_seq_len_to_capture: int = 8192,
55 - disable_custom_all_reduce: bool = False,
56 - **kwargs,
57 - ) -> None:
58 - if "disable_log_stats" not in kwargs:
59 - kwargs["disable_log_stats"] = True
60 -
61 - # Needed to engine_use_ray works as a deprecated feature,
62 - # otherwise the following constructor will raise an exception
63 - os.environ["VLLM_ALLOW_ENGINE_USE_RAY"] = "1"
64 -
65 - engine_args = AsyncEngineArgs(
66 - model=model,
67 - tokenizer=tokenizer,
68 - tokenizer_mode=tokenizer_mode,
69 - skip_tokenizer_init=skip_tokenizer_init,
70 - trust_remote_code=trust_remote_code,
71 - tensor_parallel_size=tensor_parallel_size,
72 - dtype=dtype,
73 - quantization=quantization,
74 - revision=revision,
75 - tokenizer_revision=tokenizer_revision,
76 - seed=seed,
77 - gpu_memory_utilization=gpu_memory_utilization,
78 - swap_space=swap_space,
79 - enforce_eager=enforce_eager,
80 - max_seq_len_to_capture=max_seq_len_to_capture,
81 - # For now use ray for the distributed back-end, since
82 - # we rely on the use of engine_use_ray=True to avoid
83 - # reinitializing CUDA in the same process (driver worker)
84 - engine_use_ray=True,
85 - distributed_executor_backend="ray",
86 - disable_custom_all_reduce=disable_custom_all_reduce,
87 - **kwargs,
88 - )
89 - self.request_counter = Counter()
90 - self.llm_engine = AsyncLLMEngine.from_engine_args(
91 - engine_args, usage_context=UsageContext.LLM_CLASS)
92 -
93 - def generate(
94 - self,
95 - prompts: Optional[Union[str, List[str]]] = None,
96 - sampling_params: Optional[Union[SamplingParams,
97 - List[SamplingParams]]] = None,
98 - prompt_token_ids: Optional[List[List[int]]] = None,
99 - use_tqdm: bool = True,
100 - lora_request: Optional[LoRARequest] = None,
101 - multi_modal_data: Optional[MultiModalDataDict] = None,
102 - prompt_adapter_request: Optional[PromptAdapterRequest] = None
103 - ) -> List[RequestOutput]:
104 -
105 - if prompts is None:
106 - raise ValueError("prompts must be provided.")
107 - if isinstance(prompts, str):
108 - # Convert a single prompt to a list.
109 - prompts = [prompts]
110 -
111 - if prompts is not None:
112 - num_requests = len(prompts)
113 -
114 - if sampling_params is None:
115 - # Use default sampling params.
116 - sampling_params = SamplingParams()
117 -
118 - elif isinstance(sampling_params,
119 - list) and len(sampling_params) != num_requests:
120 - raise ValueError("The lengths of prompts and "
121 - "sampling_params must be the same.")
122 -
123 - async def get_output(prompt, sampling_param) -> RequestOutput:
124 - request_id = random_uuid()
125 - results_generator = self.llm_engine.generate(
126 - prompt, sampling_param, request_id)
127 - final_output = None
128 - async for request_output in results_generator:
129 - final_output = request_output
130 - assert final_output is not None
131 - return final_output
132 -
133 - outputs: List[RequestOutput] = []
134 - try:
135 - for i in range(num_requests):
136 - prompt = prompts[i] if prompts is not None else None
137 - params = sampling_params[i] if isinstance(
138 - sampling_params, Sequence) else sampling_params
139 - res = asyncio.run(get_output(prompt, params))
140 - outputs.append(res)
141 - finally:
142 - ray.shutdown()
143 - return outputs
13 +PROMPTS = [
14 + "Hello, my name is",
15 + "The president of the United States is",
16 + "The capital of France is",
17 + "The future of AI is",
18 + "San Francisco is know for its",
19 + "Facebook was created in 2004 by",
20 + "Curious George is a",
21 + "Python 3.11 brings improvements to its",
22 +]
23
24
25 @pytest.fixture
147 -def baseline_llm_generator(request, common_llm_kwargs,
148 - per_test_common_llm_kwargs, baseline_llm_kwargs,
149 - seed):
150 - return create_llm_generator("baseline", request, common_llm_kwargs,
151 - per_test_common_llm_kwargs,
152 - baseline_llm_kwargs, seed)
153 -
154 -
155 -@pytest.fixture
156 -def test_llm_generator(request, common_llm_kwargs, per_test_common_llm_kwargs,
26 +def test_llm_generator(common_llm_kwargs, per_test_common_llm_kwargs,
27 test_llm_kwargs, seed):
158 - return create_llm_generator("test", request, common_llm_kwargs,
159 - per_test_common_llm_kwargs, test_llm_kwargs,
160 - seed)
28
29 + def generate():
30 + kwargs = {
31 + **common_llm_kwargs,
32 + **per_test_common_llm_kwargs,
33 + **test_llm_kwargs,
34 + }
35 +
36 + llm = LLM(**kwargs)
37
163 -def create_llm_generator(baseline_or_test, request, common_llm_kwargs,
164 - per_test_common_llm_kwargs, distinct_llm_kwargs,
165 - seed):
166 - kwargs = {
167 - **common_llm_kwargs,
168 - **per_test_common_llm_kwargs,
169 - **distinct_llm_kwargs,
170 - }
171 - test_name = request.node.name
172 -
173 - model = kwargs["model"]
174 - draft_model = kwargs.get("speculative_model", None)
175 - same_draft_target_model = (draft_model is not None
176 - and draft_model == model)
177 -
178 - def generator_inner():
179 -
180 - wait_for_gpu_memory_to_clear(
181 - devices=list(range(torch.cuda.device_count())),
182 - threshold_bytes=2 * 2**30,
183 - timeout_s=60,
184 - )
185 -
186 - use_async = False
187 - if "use_async" in kwargs:
188 - use_async = kwargs.pop("use_async")
189 - print(f'{use_async=}')
190 -
191 - print(f'Creating {baseline_or_test=} LLM for {test_name=}. {kwargs=}')
192 - llm = AsyncLLM(**kwargs) if use_async else LLM(**kwargs)
193 -
194 - # Override logging interval to 0 for spec decode test run to
195 - # log all metrics in time.
196 - if (baseline_or_test == "test" and not use_async
197 - and llm.llm_engine.log_stats):
198 - for sate_logger in llm.llm_engine.stat_loggers.values():
199 - sate_logger.local_interval = 0
38 if seed is not None:
39 set_random_seed(seed)
40
41 yield llm
42 +
43 del llm
44 cleanup()
45
207 - def generator_outer():
208 - for llm in generator_inner():
209 - yield llm
210 - del llm
211 -
212 - # Set an attribute to the generator_outer function to allow us to
213 - # determine whether to further check the acceptance rate in tests.
214 - generator_outer.same_draft_target_model = same_draft_target_model # type: ignore
215 - return generator_outer
46 + return generate
47
48
49 def maybe_assert_ngram_worker(llm):
50 # Verify the proposer worker is ngram if ngram is specified.
220 - if (not isinstance(llm, AsyncLLM)
221 - and llm.llm_engine.speculative_config is not None
51 + if (llm.llm_engine.speculative_config is not None
52 and llm.llm_engine.speculative_config.ngram_prompt_lookup_max > 0):
53 from vllm.spec_decode.ngram_worker import NGramWorker
54 assert isinstance(
@@ -251,118 +81,165 @@ def get_output_from_llm_generator(
81 return tokens, token_ids, acceptance_rate
82
83
254 -def get_logprobs_from_llm_generator(
255 - llm_generator, prompts,
256 - sampling_params) -> List[List[Dict[int, Logprob]]]:
257 - """Returns a dict of (token_id: Logprob) for each generated position, for
258 - each sequence in the batch.
259 - """
260 - for llm in llm_generator():
261 - outputs = llm.generate(prompts, sampling_params, use_tqdm=True)
262 - logprobs = [output.outputs[0].logprobs[:] for output in outputs]
263 - del llm
84 +def run_logprob_correctness_test(vllm_runner,
85 + common_llm_kwargs,
86 + per_test_common_llm_kwargs,
87 + baseline_llm_kwargs,
88 + test_llm_kwargs,
89 + batch_size: int,
90 + max_output_len: int,
91 + seed: Optional[int] = 0,
92 + temperature: float = 0.0,
93 + logprobs: int = 1):
94 + org_args = {
95 + **common_llm_kwargs,
96 + **per_test_common_llm_kwargs,
97 + **baseline_llm_kwargs,
98 + }
99
265 - return logprobs
100 + sd_args = {
101 + **common_llm_kwargs,
102 + **per_test_common_llm_kwargs,
103 + **test_llm_kwargs,
104 + }
105
106 + prompts = [prompt for prompt, _ in zip(cycle(PROMPTS), range(batch_size))]
107
268 -def run_greedy_equality_correctness_test(baseline_llm_generator,
269 - test_llm_generator,
270 - batch_size,
271 - max_output_len,
272 - force_output_len: bool,
273 - print_tokens: bool = False,
274 - ensure_all_accepted: bool = False):
275 - """Helper method that compares the outputs of both the baseline LLM and
276 - the test LLM. It asserts greedy equality, e.g. that the outputs are exactly
277 - the same when temperature is zero.
278 - """
108 + sampling_params = SamplingParams(temperature=temperature,
109 + max_tokens=max_output_len,
110 + seed=seed,
111 + logprobs=logprobs)
112 +
113 + with vllm_runner(**org_args) as vllm_model:
114 + org_outputs = vllm_model.generate_w_logprobs(prompts, sampling_params)
115
280 - run_equality_correctness_test(baseline_llm_generator,
281 - test_llm_generator,
282 - batch_size,
283 - max_output_len,
284 - force_output_len,
285 - temperature=0.0,
286 - seeded=False,
287 - print_tokens=print_tokens,
288 - ensure_all_accepted=ensure_all_accepted)
116 + with vllm_runner(**sd_args) as vllm_model:
117 + sd_outputs = vllm_model.generate_w_logprobs(prompts, sampling_params)
118 +
119 + check_logprobs_close(outputs_0_lst=org_outputs,
120 + outputs_1_lst=sd_outputs,
121 + name_0="org",
122 + name_1="sd")
123
124
125 def run_equality_correctness_test(
292 - baseline_llm_generator,
293 - test_llm_generator,
294 - batch_size,
295 - max_output_len,
296 - force_output_len: bool,
297 - temperature: float,
298 - seeded: bool,
299 - print_tokens: bool = False,
126 + vllm_runner,
127 + common_llm_kwargs,
128 + per_test_common_llm_kwargs,
129 + baseline_llm_kwargs,
130 + test_llm_kwargs,
131 + batch_size: int,
132 + max_output_len: int,
133 + seed: Optional[int] = 0,
134 + temperature: float = 0.0,
135 + disable_seed: bool = False,
136 + ignore_eos: bool = True,
137 ensure_all_accepted: bool = False,
138 expected_acceptance_rate: Optional[float] = None):
139 +
140 + org_args = {
141 + **common_llm_kwargs,
142 + **per_test_common_llm_kwargs,
143 + **baseline_llm_kwargs,
144 + }
145 +
146 + sd_args = {
147 + **common_llm_kwargs,
148 + **per_test_common_llm_kwargs,
149 + **test_llm_kwargs,
150 + }
151 +
152 + prompts = [prompt for prompt, _ in zip(cycle(PROMPTS), range(batch_size))]
153 +
154 + if disable_seed:
155 + seed = None
156 +
157 + sampling_params = SamplingParams(temperature=temperature,
158 + max_tokens=max_output_len,
159 + seed=seed,
160 + ignore_eos=ignore_eos)
161 +
162 + with vllm_runner(**org_args) as vllm_model:
163 + org_outputs = vllm_model.generate(prompts, sampling_params)
164 +
165 + with vllm_runner(**sd_args) as vllm_model:
166 + if ensure_all_accepted or expected_acceptance_rate is not None:
167 + # Force log interval to be 0 to catch all metrics.
168 + stat_logger = vllm_model.model.llm_engine.stat_loggers[
169 + 'prometheus']
170 + stat_logger.local_interval = -100
171 +
172 + sd_outputs = vllm_model.generate(prompts, sampling_params)
173 +
174 + if ensure_all_accepted or expected_acceptance_rate is not None:
175 + acceptance_rate = (stat_logger.metrics.
176 + gauge_spec_decode_draft_acceptance_rate.labels(
177 + **stat_logger.labels)._value.get())
178 +
179 + if ensure_all_accepted:
180 + assert True
181 + # FIXME: ci fails to log acceptance rate.
182 + # It works locally.
183 + # assert acceptance_rate == 1.0
184 +
185 + if expected_acceptance_rate is not None:
186 + assert acceptance_rate >= expected_acceptance_rate - 1e-2
187 +
188 + check_outputs_equal(outputs_0_lst=org_outputs,
189 + outputs_1_lst=sd_outputs,
190 + name_0="org",
191 + name_1="sd")
192 +
193 +
194 +def run_equality_correctness_test_tp(model,
195 + common_llm_kwargs,
196 + per_test_common_llm_kwargs,
197 + baseline_llm_kwargs,
198 + test_llm_kwargs,
199 + batch_size: int,
200 + max_output_len: int,
201 + seed: int = 0,
202 + temperature: float = 0.0):
203 """Helper method that compares the outputs of both the baseline LLM and
204 the test LLM. It asserts greedy equality, e.g. that the outputs are exactly
304 - the same when temperature is zero (or when temperature is > 0 and seeded).
205 + the same when temperature is zero.
206 """
306 -
307 - prompts = [
308 - "Hello, my name is",
309 - "The president of the United States is",
310 - "The capital of France is",
311 - "The future of AI is",
312 - "San Francisco is know for its",
313 - "Facebook was created in 2004 by",
314 - "Curious George is a",
315 - "Python 3.11 brings improvements to its",
316 - ]
317 -
318 - prompts = [prompt for prompt, _ in zip(cycle(prompts), range(batch_size))]
319 -
320 - # If the test requires that we generated max_output_len tokens, then set the
321 - # sampling params to ignore eos token.
322 - ignore_eos = force_output_len
323 -
324 - if seeded:
325 - sampling_params = [
326 - SamplingParams(
327 - max_tokens=max_output_len,
328 - ignore_eos=ignore_eos,
329 - temperature=temperature,
330 - seed=i,
331 - ) for i in range(len(prompts))
332 - ]
333 - else:
334 - sampling_params = SamplingParams(
335 - max_tokens=max_output_len,
336 - ignore_eos=ignore_eos,
337 - temperature=temperature,
338 - )
339 -
340 - (spec_batch_tokens, spec_batch_token_ids,
341 - acceptance_rate) = get_output_from_llm_generator(test_llm_generator,
342 - prompts, sampling_params)
343 -
344 - (baseline_batch_tokens, baseline_batch_token_ids,
345 - _) = get_output_from_llm_generator(baseline_llm_generator, prompts,
346 - sampling_params)
347 -
348 - assert len(baseline_batch_token_ids) == len(prompts)
349 - assert len(spec_batch_token_ids) == len(prompts)
350 -
351 - for i, (baseline_token_ids, baseline_tokens, spec_token_ids,
352 - spec_tokens) in enumerate(
353 - zip(baseline_batch_token_ids, baseline_batch_tokens,
354 - spec_batch_token_ids, spec_batch_tokens)):
355 - if print_tokens:
356 - print(f'{i=} {baseline_tokens=}')
357 - print(f'{i=} {spec_tokens=}')
358 - print(f'{i=} {baseline_token_ids=}')
359 - print(f'{i=} {spec_token_ids=}')
360 - assert baseline_token_ids == spec_token_ids
361 -
362 - print(f'{acceptance_rate=}')
363 -
364 - if ensure_all_accepted:
365 - assert acceptance_rate == 1.0
366 -
367 - if expected_acceptance_rate is not None:
368 - assert acceptance_rate >= expected_acceptance_rate - 1e-2
207 + arg1 = common_llm_kwargs + per_test_common_llm_kwargs + baseline_llm_kwargs
208 + arg2 = common_llm_kwargs + per_test_common_llm_kwargs + test_llm_kwargs
209 + env1 = env2 = None
210 +
211 + max_wait_seconds = 240
212 + results = []
213 +
214 + prompts = [prompt for prompt, _ in zip(cycle(PROMPTS), range(batch_size))]
215 +
216 + for args, env in ((arg1, env1), (arg2, env2)):
217 + with RemoteOpenAIServer(model,
218 + args,
219 + env_dict=env,
220 + max_wait_seconds=max_wait_seconds) as server:
221 + client = server.get_client()
222 +
223 + completion = client.completions.create(model=model,
224 + prompt=prompts,
225 + max_tokens=max_output_len,
226 + seed=seed,
227 + temperature=temperature)
228 +
229 + results.append({
230 + "test":
231 + "seeded_sampling",
232 + "text": [choice.text for choice in completion.choices],
233 + "finish_reason":
234 + [choice.finish_reason for choice in completion.choices],
235 + "usage":
236 + completion.usage,
237 + })
238 +
239 + n = len(results) // 2
240 + arg1_results = results[:n]
241 + arg2_results = results[n:]
242 + for arg1_result, arg2_result in zip(arg1_results, arg2_results):
243 + assert arg1_result == arg2_result, (
244 + f"Results for {model=} are not the same with {arg1=} and {arg2=}. "
245 + f"{arg1_result=} != {arg2_result=}")
tests/spec_decode/e2e/test_eagle_correctness.py
+49 -48
@@ -21,7 +21,7 @@ correctess for the target model outputs.
21
22 import pytest
23
24 -from .conftest import run_greedy_equality_correctness_test
24 +from .conftest import run_equality_correctness_test
25
26 # main model
27 MAIN_MODEL = "JackFram/llama-68m"
@@ -53,7 +53,7 @@ PRECISION = "float32"
53 "dtype": PRECISION,
54
55 # Main model
56 - "model": MAIN_MODEL,
56 + "model_name": MAIN_MODEL,
57 }])
58 @pytest.mark.parametrize("per_test_common_llm_kwargs", [{}])
59 @pytest.mark.parametrize("baseline_llm_kwargs", [{}])
@@ -68,15 +68,16 @@ PRECISION = "float32"
68 ])
69 @pytest.mark.parametrize("batch_size", [1, 32])
70 @pytest.mark.parametrize("seed", [1])
71 -def test_eagle_e2e_greedy_correctness(baseline_llm_generator,
72 - test_llm_generator, batch_size: int,
73 - output_len: int):
74 - """Verify greedy equality with different batch size."""
75 - run_greedy_equality_correctness_test(baseline_llm_generator,
76 - test_llm_generator,
77 - batch_size,
78 - max_output_len=output_len,
79 - force_output_len=True)
71 +def test_eagle_e2e_greedy_correctness(vllm_runner, common_llm_kwargs,
72 + per_test_common_llm_kwargs,
73 + baseline_llm_kwargs, test_llm_kwargs,
74 + batch_size: int, output_len: int,
75 + seed: int):
76 +
77 + run_equality_correctness_test(vllm_runner, common_llm_kwargs,
78 + per_test_common_llm_kwargs,
79 + baseline_llm_kwargs, test_llm_kwargs,
80 + batch_size, output_len, seed)
81
82
83 @pytest.mark.parametrize(
@@ -94,7 +95,7 @@ def test_eagle_e2e_greedy_correctness(baseline_llm_generator,
95 "dtype": PRECISION,
96
97 # Main model
97 - "model": MAIN_MODEL,
98 + "model_name": MAIN_MODEL,
99 }])
100 @pytest.mark.parametrize("per_test_common_llm_kwargs", [{}])
101 @pytest.mark.parametrize("baseline_llm_kwargs", [{}])
@@ -109,17 +110,16 @@ def test_eagle_e2e_greedy_correctness(baseline_llm_generator,
110 ])
111 @pytest.mark.parametrize("batch_size", [1, 32])
112 @pytest.mark.parametrize("seed", [1])
112 -def test_eagle_e2e_greedy_correctness_cuda_graph(baseline_llm_generator,
113 - test_llm_generator,
114 - batch_size: int,
115 - output_len: int):
116 - """Verify greedy equality with cuda graph enabled and different
113 +def test_eagle_e2e_greedy_correctness_cuda_graph(
114 + vllm_runner, common_llm_kwargs, per_test_common_llm_kwargs,
115 + baseline_llm_kwargs, test_llm_kwargs, batch_size: int, output_len: int,
116 + seed: int):
117 + """Verify greedy equality with cuda graph enabled and different
118 batch sizes."""
118 - run_greedy_equality_correctness_test(baseline_llm_generator,
119 - test_llm_generator,
120 - batch_size,
121 - max_output_len=output_len,
122 - force_output_len=True)
119 + run_equality_correctness_test(vllm_runner, common_llm_kwargs,
120 + per_test_common_llm_kwargs,
121 + baseline_llm_kwargs, test_llm_kwargs,
122 + batch_size, output_len, seed)
123
124
125 @pytest.mark.parametrize(
@@ -140,7 +140,7 @@ def test_eagle_e2e_greedy_correctness_cuda_graph(baseline_llm_generator,
140 "dtype": PRECISION,
141
142 # Main model
143 - "model": MAIN_MODEL,
143 + "model_name": MAIN_MODEL,
144 }])
145 @pytest.mark.parametrize("per_test_common_llm_kwargs", [{}])
146 @pytest.mark.parametrize("baseline_llm_kwargs", [{}])
@@ -158,18 +158,17 @@ def test_eagle_e2e_greedy_correctness_cuda_graph(baseline_llm_generator,
158 ])
159 @pytest.mark.parametrize("batch_size", [4])
160 @pytest.mark.parametrize("seed", [1])
161 -def test_eagle_e2e_greedy_correctness_with_preemption(baseline_llm_generator,
162 - test_llm_generator,
163 - batch_size: int,
164 - output_len: int):
161 +def test_eagle_e2e_greedy_correctness_with_preemption(
162 + vllm_runner, common_llm_kwargs, per_test_common_llm_kwargs,
163 + baseline_llm_kwargs, test_llm_kwargs, batch_size: int, output_len: int,
164 + seed: int):
165 """Verify greedy equality, even when some sequences are preempted mid-
166 generation.
167 """
168 - run_greedy_equality_correctness_test(baseline_llm_generator,
169 - test_llm_generator,
170 - batch_size,
171 - max_output_len=output_len,
172 - force_output_len=True)
168 + run_equality_correctness_test(vllm_runner, common_llm_kwargs,
169 + per_test_common_llm_kwargs,
170 + baseline_llm_kwargs, test_llm_kwargs,
171 + batch_size, output_len, seed)
172
173
174 @pytest.mark.parametrize(
@@ -185,7 +184,7 @@ def test_eagle_e2e_greedy_correctness_with_preemption(baseline_llm_generator,
184 "dtype": PRECISION,
185
186 # Main model
188 - "model": MAIN_MODEL,
187 + "model_name": MAIN_MODEL,
188 }])
189 @pytest.mark.parametrize("per_test_common_llm_kwargs", [{}])
190 @pytest.mark.parametrize("baseline_llm_kwargs", [{}])
@@ -207,16 +206,17 @@ def test_eagle_e2e_greedy_correctness_with_preemption(baseline_llm_generator,
206 32,
207 ])
208 @pytest.mark.parametrize("seed", [1])
210 -def test_eagle_different_k(baseline_llm_generator, test_llm_generator,
211 - batch_size: int, output_len: int):
209 +def test_eagle_different_k(vllm_runner, common_llm_kwargs,
210 + per_test_common_llm_kwargs, baseline_llm_kwargs,
211 + test_llm_kwargs, batch_size: int, output_len: int,
212 + seed: int):
213 """Verify that eagle speculative decoding produces exact equality
214 to without spec decode with different values of num_speculative_tokens.
215 """
215 - run_greedy_equality_correctness_test(baseline_llm_generator,
216 - test_llm_generator,
217 - batch_size,
218 - max_output_len=output_len,
219 - force_output_len=True)
216 + run_equality_correctness_test(vllm_runner, common_llm_kwargs,
217 + per_test_common_llm_kwargs,
218 + baseline_llm_kwargs, test_llm_kwargs,
219 + batch_size, output_len, seed)
220
221
222 @pytest.mark.parametrize(
@@ -232,7 +232,7 @@ def test_eagle_different_k(baseline_llm_generator, test_llm_generator,
232 "dtype": PRECISION,
233
234 # Main model
235 - "model": MAIN_MODEL,
235 + "model_name": MAIN_MODEL,
236 }])
237 @pytest.mark.parametrize("per_test_common_llm_kwargs", [{}])
238 @pytest.mark.parametrize("baseline_llm_kwargs", [{}])
@@ -250,17 +250,18 @@ def test_eagle_different_k(baseline_llm_generator, test_llm_generator,
250 32,
251 ])
252 @pytest.mark.parametrize("seed", [1])
253 -def test_eagle_disable_queue(baseline_llm_generator, test_llm_generator,
254 - batch_size: int, output_len: int):
253 +def test_eagle_disable_queue(vllm_runner, common_llm_kwargs,
254 + per_test_common_llm_kwargs, baseline_llm_kwargs,
255 + test_llm_kwargs, batch_size: int, output_len: int,
256 + seed: int):
257 """Verify that eagle speculative decoding produces exact equality
258 to without spec decode when speculation is disabled for large
259 batch sizes.
260 """
259 - run_greedy_equality_correctness_test(baseline_llm_generator,
260 - test_llm_generator,
261 - batch_size,
262 - max_output_len=output_len,
263 - force_output_len=True)
261 + run_equality_correctness_test(vllm_runner, common_llm_kwargs,
262 + per_test_common_llm_kwargs,
263 + baseline_llm_kwargs, test_llm_kwargs,
264 + batch_size, output_len, seed)
265
266
267 if __name__ == "__main__":
tests/spec_decode/e2e/test_integration.py
+32 -20
@@ -4,7 +4,9 @@ other features, e.g. cuda graphs.
4
5 import pytest
6
7 -from .conftest import run_greedy_equality_correctness_test
7 +from .conftest import run_equality_correctness_test
8 +
9 +MAIN_MODEL = "JackFram/llama-68m"
10
11
12 @pytest.mark.parametrize(
@@ -15,7 +17,7 @@ from .conftest import run_greedy_equality_correctness_test
17
18 # Verify equality when cuda graphs allowed.
19 "enforce_eager": False,
18 - "model": "JackFram/llama-68m",
20 + "model_name": "JackFram/llama-68m",
21 }])
22 @pytest.mark.parametrize(
23 "per_test_common_llm_kwargs",
@@ -31,23 +33,27 @@ from .conftest import run_greedy_equality_correctness_test
33 @pytest.mark.parametrize("batch_size", [8])
34 @pytest.mark.parametrize("output_len", [32])
35 @pytest.mark.parametrize("seed", [1])
34 -def test_spec_decode_cuda_graph(baseline_llm_generator, test_llm_generator,
35 - batch_size, output_len):
36 +def test_spec_decode_cuda_graph(vllm_runner, common_llm_kwargs,
37 + per_test_common_llm_kwargs,
38 + baseline_llm_kwargs, test_llm_kwargs,
39 + batch_size: int, output_len: int, seed: int):
40 """Verify spec decode equality when cuda graphs are enabled.
41 """
38 - run_greedy_equality_correctness_test(
39 - baseline_llm_generator,
40 - test_llm_generator,
41 - batch_size,
42 - max_output_len=output_len,
43 - force_output_len=True,
44 - )
42 + run_equality_correctness_test(vllm_runner,
43 + common_llm_kwargs,
44 + per_test_common_llm_kwargs,
45 + baseline_llm_kwargs,
46 + test_llm_kwargs,
47 + batch_size,
48 + max_output_len=output_len,
49 + seed=seed,
50 + temperature=0.0)
51
52
53 @pytest.mark.parametrize(
54 "common_llm_kwargs",
55 [{
50 - "model": "JackFram/llama-160m",
56 + "model_name": "JackFram/llama-160m",
57
58 # Skip cuda graph recording for fast test.
59 "enforce_eager": True,
@@ -80,13 +86,19 @@ def test_spec_decode_cuda_graph(baseline_llm_generator, test_llm_generator,
86 @pytest.mark.parametrize("baseline_llm_kwargs", [{}])
87 @pytest.mark.parametrize("batch_size", [2])
88 @pytest.mark.parametrize("seed", [1])
83 -def test_speculative_model_quantization_config(baseline_llm_generator,
84 - test_llm_generator,
85 - batch_size: int):
89 +def test_speculative_model_quantization_config(vllm_runner, common_llm_kwargs,
90 + per_test_common_llm_kwargs,
91 + baseline_llm_kwargs,
92 + test_llm_kwargs,
93 + batch_size: int, seed: int):
94 """Verify spec decode works well with draft model quantization configs.
95 """
88 - run_greedy_equality_correctness_test(baseline_llm_generator,
89 - test_llm_generator,
90 - batch_size,
91 - max_output_len=32,
92 - force_output_len=True)
96 + run_equality_correctness_test(vllm_runner,
97 + common_llm_kwargs,
98 + per_test_common_llm_kwargs,
99 + baseline_llm_kwargs,
100 + test_llm_kwargs,
101 + batch_size,
102 + max_output_len=32,
103 + seed=seed,
104 + temperature=0.0)
tests/spec_decode/e2e/test_integration_dist_tp2.py
+76 -79
@@ -7,42 +7,39 @@ import torch
7
8 from vllm.utils import is_hip
9
10 -from .conftest import run_greedy_equality_correctness_test
10 +from .conftest import run_equality_correctness_test_tp
11
12
13 @pytest.mark.skipif(torch.cuda.device_count() < 2,
14 reason="Need at least 2 GPUs to run the test.")
15 @pytest.mark.parametrize(
16 "common_llm_kwargs",
17 - [{
18 - "model": "JackFram/llama-68m",
19 -
17 + [[
18 # Skip cuda graph recording for fast test.
21 - "enforce_eager": True,
19 + "--enforce-eager",
20
21 # Required for spec decode.
24 - "use_v2_block_manager": True,
25 - "tensor_parallel_size": 2,
26 -
27 - # Use AsyncLLM engine, so that the engine runs in its own process.
28 - # Otherwise, since vLLM does not follow true SPMD, the test runner
29 - # process will have both the engine and the rank0 worker. NCCL is not
30 - # cleaned up properly, and its server host thread leaks, causing the
31 - # second run of the test to fail with internal NCCL error.
32 - "use_async": True,
33 - }])
34 -@pytest.mark.parametrize("per_test_common_llm_kwargs", [{}])
35 -@pytest.mark.parametrize("baseline_llm_kwargs", [{}])
22 + "--use-v2-block-manager",
23 + "--tensor-parallel-size",
24 + "2"
25 + ]])
26 +@pytest.mark.parametrize("per_test_common_llm_kwargs", [[]])
27 +@pytest.mark.parametrize("baseline_llm_kwargs", [[]])
28 @pytest.mark.parametrize("test_llm_kwargs", [
37 - {
38 - "speculative_model": "JackFram/llama-68m",
39 - "num_speculative_tokens": 3,
40 - },
41 - {
42 - "speculative_model": "[ngram]",
43 - "num_speculative_tokens": 5,
44 - "ngram_prompt_lookup_max": 3,
45 - },
29 + [
30 + "--speculative-model",
31 + "JackFram/llama-68m",
32 + "--num-speculative-tokens",
33 + "3",
34 + ],
35 + [
36 + "--speculative-model",
37 + "[ngram]",
38 + "--num-speculative-tokens",
39 + "5",
40 + "--ngram-prompt-lookup-max",
41 + "3",
42 + ],
43 ])
44 @pytest.mark.parametrize("batch_size", [2])
45 @pytest.mark.parametrize(
@@ -52,75 +49,75 @@ from .conftest import run_greedy_equality_correctness_test
49 32,
50 ])
51 @pytest.mark.parametrize("seed", [1])
55 -def test_target_model_tp_gt_1(baseline_llm_generator, test_llm_generator,
56 - batch_size: int, output_len: int):
52 +def test_target_model_tp_gt_1(common_llm_kwargs, per_test_common_llm_kwargs,
53 + baseline_llm_kwargs, test_llm_kwargs,
54 + batch_size: int, output_len: int, seed: int):
55 """Verify greedy equality when tensor parallelism is used.
56 """
57 if is_hip():
58 pytest.skip("hip is not well-supported yet")
61 - run_greedy_equality_correctness_test(baseline_llm_generator,
62 - test_llm_generator,
63 - batch_size,
64 - max_output_len=output_len,
65 - force_output_len=True)
59 + run_equality_correctness_test_tp("JackFram/llama-68m",
60 + common_llm_kwargs,
61 + per_test_common_llm_kwargs,
62 + baseline_llm_kwargs,
63 + test_llm_kwargs,
64 + batch_size,
65 + output_len,
66 + seed,
67 + temperature=0.0)
68
69
70 @pytest.mark.skipif(torch.cuda.device_count() < 2,
71 reason="Need at least 2 GPUs to run the test.")
72 @pytest.mark.parametrize(
73 "common_llm_kwargs",
72 - [{
74 + [[
75 # Skip cuda graph recording for fast test.
74 - "enforce_eager": True,
76 + "--enforce-eager",
77
78 # Required for spec decode.
77 - "use_v2_block_manager": True,
78 - "tensor_parallel_size": 2,
79 -
80 - # Use AsyncLLM engine, so that the engine runs in its own process.
81 - # Otherwise, since vLLM does not follow true SPMD, the test runner
82 - # process will have both the engine and the rank0 worker. NCCL is not
83 - # cleaned up properly, and its server host thread leaks, causing the
84 - # second run of the test to fail with internal NCCL error.
85 - "use_async": True,
79 + "--use_v2_block_manager",
80 + "--tensor_parallel_size",
81 + "2",
82
83 # precision
88 - "dtype": "float32",
89 - }])
90 -@pytest.mark.parametrize("baseline_llm_kwargs", [{}])
91 -@pytest.mark.parametrize(
92 - "per_test_common_llm_kwargs, test_llm_kwargs",
93 - [
94 - (
95 - {
96 - # Use a small model for a fast test.
97 - # Note this is repeated in the test body; to initialize a
98 - # tokenizer.
99 - "model": "JackFram/llama-68m",
100 - },
101 - {
102 - "speculative_model": "JackFram/llama-68m",
103 - "num_speculative_tokens": 5,
104 - "speculative_draft_tensor_parallel_size": 1,
105 - }),
106 - ({
107 - "model": "ibm-granite/granite-3b-code-instruct",
108 - }, {
109 - "speculative_model":
110 - "ibm-granite/granite-3b-code-instruct-accelerator",
111 - "num_speculative_tokens": 5,
112 - "speculative_draft_tensor_parallel_size": 1,
113 - })
114 - ])
84 + "--dtype",
85 + "bfloat16",
86 + ]])
87 +@pytest.mark.parametrize("per_test_common_llm_kwargs", [[]])
88 +@pytest.mark.parametrize("baseline_llm_kwargs", [[]])
89 +@pytest.mark.parametrize("model, test_llm_kwargs",
90 + [("JackFram/llama-68m", [
91 + "--speculative-model",
92 + "JackFram/llama-68m",
93 + "--num_speculative-tokens",
94 + "5",
95 + "--speculative-draft-tensor-parallel-size",
96 + "1",
97 + ]),
98 + ("ibm-granite/granite-3b-code-instruct", [
99 + "--speculative-model",
100 + "ibm-granite/granite-3b-code-instruct",
101 + "--num_speculative-tokens",
102 + "5",
103 + "--speculative-draft-tensor-parallel-size",
104 + "1",
105 + ])])
106 @pytest.mark.parametrize("batch_size", [2])
107 @pytest.mark.parametrize("seed", [1])
117 -def test_draft_model_tp_lt_target_model_tp2(test_llm_generator,
118 - baseline_llm_generator,
119 - batch_size: int):
108 +def test_draft_model_tp_lt_target_model_tp2(model, common_llm_kwargs,
109 + per_test_common_llm_kwargs,
110 + baseline_llm_kwargs,
111 + test_llm_kwargs, batch_size: int,
112 + seed: int):
113 """Verify spec decode works well with smaller tp for draft models.
114 """
122 - run_greedy_equality_correctness_test(baseline_llm_generator,
123 - test_llm_generator,
124 - batch_size,
125 - max_output_len=32,
126 - force_output_len=True)
115 + run_equality_correctness_test_tp(model,
116 + common_llm_kwargs,
117 + per_test_common_llm_kwargs,
118 + baseline_llm_kwargs,
119 + test_llm_kwargs,
120 + batch_size,
121 + max_output_len=32,
122 + seed=seed,
123 + temperature=0.0)
tests/spec_decode/e2e/test_integration_dist_tp4.py
+65 -61
@@ -2,98 +2,97 @@
2 tensor parallelism.
3 """
4
5 +import openai
6 import pytest
7 import torch
8
8 -from .conftest import run_greedy_equality_correctness_test
9 +from .conftest import run_equality_correctness_test_tp
10 +
11 +MAIN_MODEL = "JackFram/llama-68m"
12 +SPEC_MODEL = "JackFram/llama-68m"
13
14
15 @pytest.mark.skipif(torch.cuda.device_count() < 4,
16 reason="Need at least 4 GPUs to run the test.")
17 @pytest.mark.parametrize(
18 "common_llm_kwargs",
15 - [{
16 - # Use a small model for a fast test.
17 - # Note this is repeated in the test body; to initialize a tokenizer.
18 - "model": "JackFram/llama-68m",
19 -
19 + [[
20 # Skip cuda graph recording for fast test.
21 - "enforce_eager": True,
21 + "--enforce_eager",
22
23 # Required for spec decode.
24 - "use_v2_block_manager": True,
25 - "tensor_parallel_size": 4,
26 -
27 - # Use AsyncLLM engine, so that the engine runs in its own process.
28 - # Otherwise, since vLLM does not follow true SPMD, the test runner
29 - # process will have both the engine and the rank0 worker. NCCL is not
30 - # cleaned up properly, and its server host thread leaks, causing the
31 - # second run of the test to fail with internal NCCL error.
32 - "use_async": True,
33 - }])
24 + "--use-v2-block-manager",
25 + "--tensor-parallel-size",
26 + "4",
27 + ]])
28 @pytest.mark.parametrize("per_test_common_llm_kwargs", [
35 - {
36 - "speculative_model": "JackFram/llama-68m",
37 - "num_speculative_tokens": 5,
38 - },
29 + [
30 + "--speculative-model",
31 + f"{SPEC_MODEL}",
32 + "--num-speculative-tokens",
33 + "5",
34 + ],
35 ])
40 -@pytest.mark.parametrize("baseline_llm_kwargs", [{}])
36 +@pytest.mark.parametrize("baseline_llm_kwargs", [[]])
37 @pytest.mark.parametrize(
38 "test_llm_kwargs",
39 [
40 #TODO(wooyeon): add spec_draft_dp=2 case
45 - {
46 - "speculative_draft_tensor_parallel_size": 1,
47 - },
41 + [
42 + "--speculative-draft-tensor-parallel-size",
43 + "1",
44 + ],
45 ])
46 @pytest.mark.parametrize("batch_size", [2])
47 @pytest.mark.parametrize("seed", [1])
51 -def test_draft_model_tp_lt_target_model_tp4(test_llm_generator,
52 - baseline_llm_generator,
53 - batch_size: int):
48 +def test_draft_model_tp_lt_target_model_tp4(common_llm_kwargs,
49 + per_test_common_llm_kwargs,
50 + baseline_llm_kwargs,
51 + test_llm_kwargs, batch_size: int,
52 + seed: int):
53 """Verify spec decode works well with smaller tp for draft models.
54 """
56 - run_greedy_equality_correctness_test(baseline_llm_generator,
57 - test_llm_generator,
58 - batch_size,
59 - max_output_len=32,
60 - force_output_len=True)
55 + run_equality_correctness_test_tp(MAIN_MODEL,
56 + common_llm_kwargs,
57 + per_test_common_llm_kwargs,
58 + baseline_llm_kwargs,
59 + test_llm_kwargs,
60 + batch_size,
61 + max_output_len=32,
62 + seed=seed,
63 + temperature=0.0)
64
65
66 @pytest.mark.skipif(torch.cuda.device_count() < 4,
67 reason="Need at least 4 GPUs to run the test.")
68 @pytest.mark.parametrize(
69 "common_llm_kwargs",
67 - [{
68 - "model": "JackFram/llama-160m",
70 + [[
71
72 # Skip cuda graph recording for fast test.
71 - "enforce_eager": True,
73 + "--enforce-eager",
74
75 # Required for spec decode.
74 - "use_v2_block_manager": True,
75 - "tensor_parallel_size": 4,
76 -
77 - # Use AsyncLLM engine, so that the engine runs in its own process.
78 - # Otherwise, since vLLM does not follow true SPMD, the test runner
79 - # process will have both the engine and the rank0 worker. NCCL is not
80 - # cleaned up properly, and its server host thread leaks, causing the
81 - # second run of the test to fail with internal NCCL error.
82 - "use_async": True,
83 - }])
84 -@pytest.mark.parametrize("per_test_common_llm_kwargs", [{}])
85 -@pytest.mark.parametrize("baseline_llm_kwargs", [{}])
76 + "--use-v2-block-manager",
77 + "--tensor-parallel-size",
78 + "4",
79 + ]])
80 +@pytest.mark.parametrize("per_test_common_llm_kwargs", [[]])
81 +@pytest.mark.parametrize("baseline_llm_kwargs", [[]])
82 @pytest.mark.parametrize(
83 "test_llm_kwargs",
84 [
89 - {
90 - "speculative_model": "JackFram/llama-68m",
91 - "num_speculative_tokens": 5,
85 + [
86 + "--speculative-model",
87 + f"{SPEC_MODEL}",
88 + "--num-speculative-tokens",
89 + "5",
90
91 # Artificially limit the draft model max model len; this forces vLLM
92 # to skip speculation once the sequences grow beyond 32-k tokens.
95 - "speculative_max_model_len": 32,
96 - },
93 + "--speculative-max-model-len",
94 + "32",
95 + ],
96 ])
97 @pytest.mark.parametrize("batch_size", [8])
98 @pytest.mark.parametrize(
@@ -105,8 +104,9 @@ def test_draft_model_tp_lt_target_model_tp4(test_llm_generator,
104 64,
105 ])
106 @pytest.mark.parametrize("seed", [1])
108 -def test_skip_speculation(baseline_llm_generator, test_llm_generator,
109 - batch_size: int, output_len: int):
107 +def test_skip_speculation(common_llm_kwargs, per_test_common_llm_kwargs,
108 + baseline_llm_kwargs, test_llm_kwargs,
109 + batch_size: int, output_len: int, seed: int):
110 """Verify job failure with RuntimeError when all sequences skip speculation.
111 We do this by setting the max model len of the draft model to an
112 artificially low value, such that when the sequences grow beyond it, they
@@ -114,9 +114,13 @@ def test_skip_speculation(baseline_llm_generator, test_llm_generator,
114
115 TODO: fix it to pass without raising Error. (#5814)
116 """
117 - with pytest.raises(RuntimeError):
118 - run_greedy_equality_correctness_test(baseline_llm_generator,
119 - test_llm_generator,
120 - batch_size,
121 - max_output_len=output_len,
122 - force_output_len=True)
117 + with pytest.raises(openai.APIConnectionError):
118 + run_equality_correctness_test_tp(MAIN_MODEL,
119 + common_llm_kwargs,
120 + per_test_common_llm_kwargs,
121 + baseline_llm_kwargs,
122 + test_llm_kwargs,
123 + batch_size,
124 + output_len,
125 + seed,
126 + temperature=0.0)
tests/spec_decode/e2e/test_logprobs.py
+93 -234
@@ -1,24 +1,22 @@
1 -import math
1 from itertools import cycle
2
3 import pytest
4
5 from vllm import SamplingParams
6
8 -from .conftest import get_logprobs_from_llm_generator
7 +from .conftest import run_logprob_correctness_test
8
9
10 @pytest.mark.parametrize(
11 "common_llm_kwargs",
12 [{
14 - "model": "JackFram/llama-68m",
13 + "model_name": "JackFram/llama-68m",
14
15 # Skip cuda graph recording for fast test.
16 "enforce_eager": True,
17
18 # Required for spec decode.
19 "use_v2_block_manager": True,
21 - "max_logprobs": 6,
20 }])
21 @pytest.mark.parametrize("per_test_common_llm_kwargs", [{}])
22 @pytest.mark.parametrize("baseline_llm_kwargs", [{}])
@@ -36,64 +34,29 @@ from .conftest import get_logprobs_from_llm_generator
34 7,
35 ])
36 @pytest.mark.parametrize("seed", [1])
39 -def test_logprobs_equality(baseline_llm_generator, test_llm_generator,
40 - batch_size: int, output_len: int):
37 +@pytest.mark.parametrize("logprobs", [1, 6])
38 +def test_logprobs_equality(vllm_runner, common_llm_kwargs,
39 + per_test_common_llm_kwargs, baseline_llm_kwargs,
40 + test_llm_kwargs, batch_size: int, output_len: int,
41 + seed: int, logprobs: int):
42 """Verify output logprobs are equal with and without speculative decoding.
43 """
43 - run_greedy_logprobs_correctness_test(baseline_llm_generator,
44 - test_llm_generator,
45 - batch_size,
46 - max_output_len=output_len,
47 - force_output_len=True)
44 + run_logprob_correctness_test(vllm_runner,
45 + common_llm_kwargs,
46 + per_test_common_llm_kwargs,
47 + baseline_llm_kwargs,
48 + test_llm_kwargs,
49 + batch_size,
50 + output_len,
51 + seed,
52 + temperature=0.0,
53 + logprobs=logprobs)
54
55
56 @pytest.mark.parametrize(
57 "common_llm_kwargs",
58 [{
53 - "model": "JackFram/llama-68m",
54 -
55 - # Skip cuda graph recording for fast test.
56 - "enforce_eager": True,
57 -
58 - # Required for spec decode.
59 - "use_v2_block_manager": True,
60 - "max_logprobs": 6,
61 - }])
62 -@pytest.mark.parametrize("per_test_common_llm_kwargs", [{}])
63 -@pytest.mark.parametrize("baseline_llm_kwargs", [{}])
64 -@pytest.mark.parametrize("test_llm_kwargs",
65 - [{
66 - "speculative_model": "JackFram/llama-160m",
67 - "num_speculative_tokens": 3,
68 - "disable_logprobs_during_spec_decoding": False,
69 - }])
70 -@pytest.mark.parametrize("batch_size", [1])
71 -@pytest.mark.parametrize("num_logprobs", [6])
72 -@pytest.mark.parametrize(
73 - "output_len",
74 - [
75 - # Use smaller output len for fast test.
76 - 7,
77 - ])
78 -@pytest.mark.parametrize("seed", [1])
79 -def test_diff_num_logprobs(baseline_llm_generator, test_llm_generator,
80 - batch_size: int, output_len: int,
81 - num_logprobs: int):
82 - """Verify output logprobs are equal with and without spec decode.
83 - This specifies a number of logprobs >1.
84 - """
85 - run_greedy_logprobs_correctness_test(baseline_llm_generator,
86 - test_llm_generator,
87 - batch_size,
88 - max_output_len=output_len,
89 - force_output_len=True,
90 - logprob_rank=num_logprobs)
91 -
92 -
93 -@pytest.mark.parametrize(
94 - "common_llm_kwargs",
95 - [{
96 - "model": "JackFram/llama-68m",
59 + "model_name": "JackFram/llama-68m",
60
61 # Skip cuda graph recording for fast test.
62 "enforce_eager": True,
@@ -121,21 +84,29 @@ def test_diff_num_logprobs(baseline_llm_generator, test_llm_generator,
84 32,
85 ])
86 @pytest.mark.parametrize("seed", [1])
124 -def test_logprobs_different_k(baseline_llm_generator, test_llm_generator,
125 - batch_size: int, output_len: int):
87 +@pytest.mark.parametrize("logprobs", [1, 6])
88 +def test_logprobs_different_k(vllm_runner, common_llm_kwargs,
89 + per_test_common_llm_kwargs, baseline_llm_kwargs,
90 + test_llm_kwargs, batch_size: int,
91 + output_len: int, seed: int, logprobs: int):
92 """Veriy logprob greedy equality with different speculation lens.
93 """
128 - run_greedy_logprobs_correctness_test(baseline_llm_generator,
129 - test_llm_generator,
130 - batch_size,
131 - max_output_len=output_len,
132 - force_output_len=True)
94 + run_logprob_correctness_test(vllm_runner,
95 + common_llm_kwargs,
96 + per_test_common_llm_kwargs,
97 + baseline_llm_kwargs,
98 + test_llm_kwargs,
99 + batch_size,
100 + output_len,
101 + seed,
102 + temperature=0.0,
103 + logprobs=logprobs)
104
105
106 @pytest.mark.parametrize(
107 "common_llm_kwargs",
108 [{
138 - "model": "JackFram/llama-68m",
109 + "model_name": "JackFram/llama-68m",
110
111 # Skip cuda graph recording for fast test.
112 "enforce_eager": True,
@@ -164,22 +135,30 @@ def test_logprobs_different_k(baseline_llm_generator, test_llm_generator,
135 32,
136 ])
137 @pytest.mark.parametrize("seed", [1])
167 -def test_logprobs_when_skip_speculation(baseline_llm_generator,
168 - test_llm_generator, batch_size: int,
169 - output_len: int):
138 +@pytest.mark.parametrize("logprobs", [1])
139 +def test_logprobs_when_skip_speculation(vllm_runner, common_llm_kwargs,
140 + per_test_common_llm_kwargs,
141 + baseline_llm_kwargs, test_llm_kwargs,
142 + batch_size: int, output_len: int,
143 + seed: int, logprobs: int):
144 """Verify logprobs greedy equality when some sequences skip speculation.
145 """
172 - run_greedy_logprobs_correctness_test(baseline_llm_generator,
173 - test_llm_generator,
174 - batch_size,
175 - max_output_len=output_len,
176 - force_output_len=True)
146 + run_logprob_correctness_test(vllm_runner,
147 + common_llm_kwargs,
148 + per_test_common_llm_kwargs,
149 + baseline_llm_kwargs,
150 + test_llm_kwargs,
151 + batch_size,
152 + output_len,
153 + seed,
154 + temperature=0.0,
155 + logprobs=logprobs)
156
157
158 @pytest.mark.parametrize(
159 "common_llm_kwargs",
160 [{
182 - "model": "JackFram/llama-68m",
161 + "model_name": "JackFram/llama-68m",
162
163 # Skip cuda graph recording for fast test.
164 "enforce_eager": True,
@@ -203,19 +182,17 @@ def test_logprobs_when_skip_speculation(baseline_llm_generator,
182 32,
183 ])
184 @pytest.mark.parametrize("seed", [1])
206 -def test_logprobs_temp_1(baseline_llm_generator, test_llm_generator,
207 - batch_size: int, output_len: int):
185 +@pytest.mark.parametrize("logprobs", [6])
186 +def test_logprobs_temp_1(vllm_runner, common_llm_kwargs,
187 + per_test_common_llm_kwargs, baseline_llm_kwargs,
188 + test_llm_kwargs, batch_size: int, output_len: int,
189 + seed: int, logprobs: int):
190 """Verify at least one logprob result has num_logprobs+1, which tests the
191 case where the sampled token is not in top-k logprobs.
192
193 Ideally, this test should validate equality with non-spec by getting
194 logprobs. This is left as future improvement.
195 """
214 - batch_size = 8
215 - max_output_len = output_len
216 - force_output_len = True
217 - logprob_rank = 5
218 -
196 temperature = 1.0
197
198 prompts = [
@@ -231,129 +208,40 @@ def test_logprobs_temp_1(baseline_llm_generator, test_llm_generator,
208
209 prompts = [prompt for prompt, _ in zip(cycle(prompts), range(batch_size))]
210
234 - # If the test requires that we generated max_output_len tokens, then set the
235 - # sampling params to ignore eos token.
236 - ignore_eos = force_output_len
237 -
211 sampling_params = SamplingParams(
239 - max_tokens=max_output_len,
240 - ignore_eos=ignore_eos,
212 + max_tokens=output_len,
213 + ignore_eos=True,
214 temperature=temperature,
242 - logprobs=logprob_rank,
215 + logprobs=logprobs,
216 )
217
245 - spec_batch_logprobs = get_logprobs_from_llm_generator(
246 - test_llm_generator, prompts, sampling_params)
218 + sd_args = {
219 + **common_llm_kwargs,
220 + **per_test_common_llm_kwargs,
221 + **test_llm_kwargs,
222 + }
223 +
224 + with vllm_runner(**sd_args) as vllm_model:
225 + sd_outputs = vllm_model.generate_w_logprobs(prompts, sampling_params)
226
227 num_returned_logprobs = [
249 - len(logprob_dict) for seq_logprobs in spec_batch_logprobs
250 - for logprob_dict in seq_logprobs
228 + len(seq_logprobs) for seq_logprobs in sd_outputs[-1]
229 ]
230
231 # Assert one of the returned logprobs has > num_logprobs (indicating the
232 # sampled token is not in top-k).
255 - assert any([
256 - num_returned > logprob_rank for num_returned in num_returned_logprobs
257 - ])
258 -
259 -
260 -def run_greedy_logprobs_correctness_test(baseline_llm_generator,
261 - test_llm_generator,
262 - batch_size,
263 - max_output_len,
264 - force_output_len: bool,
265 - logprob_rank: int = 1):
266 - """Helper method that compares the logprobs outputs of both the baseline LLM
267 - and the test LLM. It asserts greedy equality of the logprobs when the
268 - temperature is zero.
269 - """
270 - temperature = 0.0
271 -
272 - prompts = [
273 - "Hello, my name is",
274 - "The president of the United States is",
275 - "The capital of France is",
276 - "The future of AI is",
277 - "San Francisco is know for its",
278 - "Facebook was created in 2004 by",
279 - "Curious George is a",
280 - "Python 3.11 brings improvements to its",
281 - ]
282 -
283 - prompts = [prompt for prompt, _ in zip(cycle(prompts), range(batch_size))]
284 -
285 - # If the test requires that we generated max_output_len tokens, then set the
286 - # sampling params to ignore eos token.
287 - ignore_eos = force_output_len
288 -
289 - sampling_params = SamplingParams(
290 - max_tokens=max_output_len,
291 - ignore_eos=ignore_eos,
292 - temperature=temperature,
293 - logprobs=logprob_rank,
294 - )
295 -
296 - spec_batch_logprobs = get_logprobs_from_llm_generator(
297 - test_llm_generator, prompts, sampling_params)
298 - baseline_batch_logprobs = get_logprobs_from_llm_generator(
299 - baseline_llm_generator, prompts, sampling_params)
300 -
301 - assert len(baseline_batch_logprobs) == len(prompts)
302 - assert len(spec_batch_logprobs) == len(prompts)
303 -
304 - # For each sequence in the batch.
305 - for i, (baseline_logprobs, spec_logprobs) in enumerate(
306 - zip(baseline_batch_logprobs, spec_batch_logprobs)):
307 - assert len(spec_logprobs) == len(baseline_logprobs)
308 -
309 - # For each generated position of the sequence.
310 - for pos, (spec_pos_logprobs, baseline_pos_logprobs) in enumerate(
311 - zip(spec_logprobs, baseline_logprobs)):
312 -
313 - # Map rank to token/logprob in spec output.
314 - spec_rank_to_token_id = {
315 - value.rank: key
316 - for key, value in spec_pos_logprobs.items()
317 - }
318 - spec_rank_to_logprob = {
319 - value.rank: value.logprob
320 - for key, value in spec_pos_logprobs.items()
321 - }
322 -
323 - # Map rank to token/logprob in baseline output.
324 - baseline_rank_to_token_id = {
325 - value.rank: key
326 - for key, value in baseline_pos_logprobs.items()
327 - }
328 - baseline_rank_to_logprob = {
329 - value.rank: value.logprob
330 - for key, value in baseline_pos_logprobs.items()
331 - }
332 -
333 - # Assert set of ranks returned is equal.
334 - assert set(spec_rank_to_token_id.keys()) == set(
335 - baseline_rank_to_token_id.keys())
336 -
337 - # Assert each logprob/token id is correct, keyed by rank.
338 - for rank in sorted(set(spec_rank_to_token_id.keys())):
339 - assert spec_rank_to_token_id[
340 - rank] == baseline_rank_to_token_id[rank], f"{rank}"
341 - assert math.isclose(
342 - a=spec_rank_to_logprob[rank],
343 - b=baseline_rank_to_logprob[rank],
344 - abs_tol=1e-1,
345 - )
233 + assert any(
234 + [num_returned > logprobs for num_returned in num_returned_logprobs])
235
236
237 @pytest.mark.parametrize(
238 "common_llm_kwargs",
239 [{
351 - "model": "JackFram/llama-160m",
240 + "model_name": "JackFram/llama-160m",
241 # Skip cuda graph recording for fast test.
242 "enforce_eager": True,
243 # Required for spec decode.
244 "use_v2_block_manager": True,
356 - "max_logprobs": 6,
245 }])
246 @pytest.mark.parametrize("per_test_common_llm_kwargs", [{}])
247 @pytest.mark.parametrize("baseline_llm_kwargs", [{}])
@@ -364,57 +252,28 @@ def run_greedy_logprobs_correctness_test(baseline_llm_generator,
252 "disable_logprobs_during_spec_decoding": True,
253 }])
254 @pytest.mark.parametrize("seed", [1])
367 -def test_logprobs_disabled(baseline_llm_generator, test_llm_generator):
255 +@pytest.mark.parametrize("batch_size", [4])
256 +@pytest.mark.parametrize(
257 + "output_len",
258 + [
259 + # Use smaller output len for fast test.
260 + 32,
261 + ])
262 +@pytest.mark.parametrize("logprobs", [0])
263 +def test_logprobs_disabled(vllm_runner, common_llm_kwargs,
264 + per_test_common_llm_kwargs, baseline_llm_kwargs,
265 + test_llm_kwargs, batch_size: int, output_len: int,
266 + seed: int, logprobs: int):
267 """Check the behavior when logprobs are disabled.
268 Token choices should match with the base model.
269 """
371 - prompts = [
372 - "Hello, my name is",
373 - "The president of the United States is",
374 - "The capital of France is",
375 - "The future of AI is",
376 - "San Francisco is know for its",
377 - "Facebook was created in 2004 by",
378 - "Curious George is a",
379 - "Python 3.11 brings improvements to its",
380 - ]
381 -
382 - prompts = [prompt for prompt, _ in zip(cycle(prompts), range(4))]
383 -
384 - sampling_params = SamplingParams(
385 - # Use smaller output len for fast test
386 - max_tokens=7,
387 - ignore_eos=True,
388 - temperature=0.0,
389 - logprobs=2,
390 - )
391 -
392 - spec_batch_logprobs = get_logprobs_from_llm_generator(
393 - test_llm_generator, prompts, sampling_params)
394 - baseline_batch_logprobs = get_logprobs_from_llm_generator(
395 - baseline_llm_generator, prompts, sampling_params)
396 -
397 - assert len(baseline_batch_logprobs) == len(prompts)
398 - assert len(spec_batch_logprobs) == len(prompts)
399 -
400 - # For each sequence in the batch.
401 - for _, (baseline_logprobs, spec_logprobs) in enumerate(
402 - zip(baseline_batch_logprobs, spec_batch_logprobs)):
403 - assert len(spec_logprobs) == len(baseline_logprobs)
404 -
405 - # For each generated position of the sequence.
406 - for _, (spec_pos_logprobs, baseline_pos_logprobs) in enumerate(
407 - zip(spec_logprobs, baseline_logprobs)):
408 -
409 - assert len(spec_pos_logprobs) == 1
410 - spec_top_token_id = list(spec_pos_logprobs)[0]
411 -
412 - spec_top_logprob = spec_pos_logprobs[spec_top_token_id]
413 - assert spec_top_logprob.logprob == 0.0
414 - assert spec_top_logprob.rank == -1
415 -
416 - # check that the chosen token matches the base model
417 - baseline_logprob = baseline_pos_logprobs[spec_top_token_id]
418 - assert baseline_logprob.rank == 1
419 - assert spec_top_logprob.decoded_token \
420 - == baseline_logprob.decoded_token
270 + run_logprob_correctness_test(vllm_runner,
271 + common_llm_kwargs,
272 + per_test_common_llm_kwargs,
273 + baseline_llm_kwargs,
274 + test_llm_kwargs,
275 + batch_size,
276 + output_len,
277 + seed,
278 + temperature=0.0,
279 + logprobs=logprobs)
tests/spec_decode/e2e/test_medusa_correctness.py
+72 -46
@@ -21,7 +21,7 @@ correctess for the target model outputs.
21
22 import pytest
23
24 -from .conftest import run_greedy_equality_correctness_test
24 +from .conftest import run_equality_correctness_test
25
26 # main model
27 # lmsys/vicuna-7b-v1.3 was to be used but it's causing
@@ -55,7 +55,7 @@ PRECISION = "float32"
55 "dtype": PRECISION,
56
57 # Main model
58 - "model": MAIN_MODEL,
58 + "model_name": MAIN_MODEL,
59 }])
60 @pytest.mark.parametrize("per_test_common_llm_kwargs", [{}])
61 @pytest.mark.parametrize("baseline_llm_kwargs", [{}])
@@ -70,15 +70,21 @@ PRECISION = "float32"
70 ])
71 @pytest.mark.parametrize("batch_size", [1, 32])
72 @pytest.mark.parametrize("seed", [1])
73 -def test_medusa_e2e_greedy_correctness(baseline_llm_generator,
74 - test_llm_generator, batch_size: int,
75 - output_len: int):
73 +def test_medusa_e2e_greedy_correctness(vllm_runner, common_llm_kwargs,
74 + per_test_common_llm_kwargs,
75 + baseline_llm_kwargs, test_llm_kwargs,
76 + batch_size: int, output_len: int,
77 + seed: int):
78 """Verify greedy equality with different batch size."""
77 - run_greedy_equality_correctness_test(baseline_llm_generator,
78 - test_llm_generator,
79 - batch_size,
80 - max_output_len=output_len,
81 - force_output_len=True)
79 + run_equality_correctness_test(vllm_runner,
80 + common_llm_kwargs,
81 + per_test_common_llm_kwargs,
82 + baseline_llm_kwargs,
83 + test_llm_kwargs,
84 + batch_size,
85 + max_output_len=output_len,
86 + seed=seed,
87 + temperature=0.0)
88
89
90 @pytest.mark.parametrize(
@@ -96,7 +102,7 @@ def test_medusa_e2e_greedy_correctness(baseline_llm_generator,
102 "dtype": PRECISION,
103
104 # Main model
99 - "model": MAIN_MODEL,
105 + "model_name": MAIN_MODEL,
106 }])
107 @pytest.mark.parametrize("per_test_common_llm_kwargs", [{}])
108 @pytest.mark.parametrize("baseline_llm_kwargs", [{}])
@@ -111,17 +117,21 @@ def test_medusa_e2e_greedy_correctness(baseline_llm_generator,
117 ])
118 @pytest.mark.parametrize("batch_size", [1, 32])
119 @pytest.mark.parametrize("seed", [1])
114 -def test_medusa_e2e_greedy_correctness_cuda_graph(baseline_llm_generator,
115 - test_llm_generator,
116 - batch_size: int,
117 - output_len: int):
120 +def test_medusa_e2e_greedy_correctness_cuda_graph(
121 + vllm_runner, common_llm_kwargs, per_test_common_llm_kwargs,
122 + baseline_llm_kwargs, test_llm_kwargs, batch_size: int, output_len: int,
123 + seed: int):
124 """Verify greedy equality with cuda graph enabled and different
125 batch sizes."""
120 - run_greedy_equality_correctness_test(baseline_llm_generator,
121 - test_llm_generator,
122 - batch_size,
123 - max_output_len=output_len,
124 - force_output_len=True)
126 + run_equality_correctness_test(vllm_runner,
127 + common_llm_kwargs,
128 + per_test_common_llm_kwargs,
129 + baseline_llm_kwargs,
130 + test_llm_kwargs,
131 + batch_size,
132 + max_output_len=output_len,
133 + seed=seed,
134 + temperature=0.0)
135
136
137 @pytest.mark.parametrize(
@@ -142,7 +152,7 @@ def test_medusa_e2e_greedy_correctness_cuda_graph(baseline_llm_generator,
152 "dtype": PRECISION,
153
154 # Main model
145 - "model": MAIN_MODEL,
155 + "model_name": MAIN_MODEL,
156 }])
157 @pytest.mark.parametrize("per_test_common_llm_kwargs", [{}])
158 @pytest.mark.parametrize("baseline_llm_kwargs", [{}])
@@ -160,18 +170,22 @@ def test_medusa_e2e_greedy_correctness_cuda_graph(baseline_llm_generator,
170 ])
171 @pytest.mark.parametrize("batch_size", [4])
172 @pytest.mark.parametrize("seed", [1])
163 -def test_medusa_e2e_greedy_correctness_with_preemption(baseline_llm_generator,
164 - test_llm_generator,
165 - batch_size: int,
166 - output_len: int):
173 +def test_medusa_e2e_greedy_correctness_with_preemption(
174 + vllm_runner, common_llm_kwargs, per_test_common_llm_kwargs,
175 + baseline_llm_kwargs, test_llm_kwargs, batch_size: int, output_len: int,
176 + seed: int):
177 """Verify greedy equality, even when some sequences are preempted mid-
178 generation.
179 """
170 - run_greedy_equality_correctness_test(baseline_llm_generator,
171 - test_llm_generator,
172 - batch_size,
173 - max_output_len=output_len,
174 - force_output_len=True)
180 + run_equality_correctness_test(vllm_runner,
181 + common_llm_kwargs,
182 + per_test_common_llm_kwargs,
183 + baseline_llm_kwargs,
184 + test_llm_kwargs,
185 + batch_size,
186 + max_output_len=output_len,
187 + seed=seed,
188 + temperature=0.0)
189
190
191 @pytest.mark.parametrize(
@@ -187,7 +201,7 @@ def test_medusa_e2e_greedy_correctness_with_preemption(baseline_llm_generator,
201 "dtype": PRECISION,
202
203 # Main model
190 - "model": MAIN_MODEL,
204 + "model_name": MAIN_MODEL,
205 }])
206 @pytest.mark.parametrize("per_test_common_llm_kwargs", [{}])
207 @pytest.mark.parametrize("baseline_llm_kwargs", [{}])
@@ -209,16 +223,22 @@ def test_medusa_e2e_greedy_correctness_with_preemption(baseline_llm_generator,
223 32,
224 ])
225 @pytest.mark.parametrize("seed", [1])
212 -def test_medusa_different_k(baseline_llm_generator, test_llm_generator,
213 - batch_size: int, output_len: int):
226 +def test_medusa_different_k(vllm_runner, common_llm_kwargs,
227 + per_test_common_llm_kwargs, baseline_llm_kwargs,
228 + test_llm_kwargs, batch_size: int, output_len: int,
229 + seed: int):
230 """Verify that medusa speculative decoding produces exact equality
231 to without spec decode with different values of num_speculative_tokens.
232 """
217 - run_greedy_equality_correctness_test(baseline_llm_generator,
218 - test_llm_generator,
219 - batch_size,
220 - max_output_len=output_len,
221 - force_output_len=True)
233 + run_equality_correctness_test(vllm_runner,
234 + common_llm_kwargs,
235 + per_test_common_llm_kwargs,
236 + baseline_llm_kwargs,
237 + test_llm_kwargs,
238 + batch_size,
239 + max_output_len=output_len,
240 + seed=seed,
241 + temperature=0.0)
242
243
244 @pytest.mark.parametrize(
@@ -234,7 +254,7 @@ def test_medusa_different_k(baseline_llm_generator, test_llm_generator,
254 "dtype": PRECISION,
255
256 # Main model
237 - "model": MAIN_MODEL,
257 + "model_name": MAIN_MODEL,
258 }])
259 @pytest.mark.parametrize("per_test_common_llm_kwargs", [{}])
260 @pytest.mark.parametrize("baseline_llm_kwargs", [{}])
@@ -252,17 +272,23 @@ def test_medusa_different_k(baseline_llm_generator, test_llm_generator,
272 32,
273 ])
274 @pytest.mark.parametrize("seed", [1])
255 -def test_medusa_disable_queue(baseline_llm_generator, test_llm_generator,
256 - batch_size: int, output_len: int):
275 +def test_medusa_disable_queue(vllm_runner, common_llm_kwargs,
276 + per_test_common_llm_kwargs, baseline_llm_kwargs,
277 + test_llm_kwargs, batch_size: int,
278 + output_len: int, seed: int):
279 """Verify that medusa speculative decoding produces exact equality
280 to without spec decode when speculation is disabled for large
281 batch sizes.
282 """
261 - run_greedy_equality_correctness_test(baseline_llm_generator,
262 - test_llm_generator,
263 - batch_size,
264 - max_output_len=output_len,
265 - force_output_len=True)
283 + run_equality_correctness_test(vllm_runner,
284 + common_llm_kwargs,
285 + per_test_common_llm_kwargs,
286 + baseline_llm_kwargs,
287 + test_llm_kwargs,
288 + batch_size,
289 + max_output_len=output_len,
290 + seed=seed,
291 + temperature=0.0)
292
293
294 if __name__ == "__main__":
tests/spec_decode/e2e/test_mlp_correctness.py
+102 -65
@@ -25,8 +25,7 @@ import pytest
25
26 from vllm.model_executor.layers.vocab_parallel_embedding import pad_vocab_size
27
28 -from .conftest import (run_equality_correctness_test,
29 - run_greedy_equality_correctness_test)
28 +from .conftest import run_equality_correctness_test
29
30 # main model
31 MAIN_MODEL = "JackFram/llama-160m"
@@ -58,7 +57,7 @@ PRECISION = "float32"
57 "dtype": PRECISION,
58
59 # Main model
61 - "model": MAIN_MODEL,
60 + "model_name": MAIN_MODEL,
61 }])
62 @pytest.mark.parametrize("per_test_common_llm_kwargs", [{}])
63 @pytest.mark.parametrize("baseline_llm_kwargs", [{}])
@@ -72,14 +71,21 @@ PRECISION = "float32"
71 ])
72 @pytest.mark.parametrize("batch_size", [1, 32])
73 @pytest.mark.parametrize("seed", [1])
75 -def test_mlp_e2e_greedy_correctness(baseline_llm_generator, test_llm_generator,
76 - batch_size: int, output_len: int):
74 +def test_mlp_e2e_greedy_correctness(vllm_runner, common_llm_kwargs,
75 + per_test_common_llm_kwargs,
76 + baseline_llm_kwargs, test_llm_kwargs,
77 + batch_size: int, output_len: int,
78 + seed: int):
79 """Verify greedy equality with different batch size."""
78 - run_greedy_equality_correctness_test(baseline_llm_generator,
79 - test_llm_generator,
80 - batch_size,
81 - max_output_len=output_len,
82 - force_output_len=True)
80 + run_equality_correctness_test(vllm_runner,
81 + common_llm_kwargs,
82 + per_test_common_llm_kwargs,
83 + baseline_llm_kwargs,
84 + test_llm_kwargs,
85 + batch_size,
86 + max_output_len=output_len,
87 + seed=seed,
88 + temperature=0.0)
89
90
91 @pytest.mark.parametrize(
@@ -98,7 +104,7 @@ def test_mlp_e2e_greedy_correctness(baseline_llm_generator, test_llm_generator,
104 "dtype": PRECISION,
105
106 # Main model
101 - "model": MAIN_MODEL,
107 + "model_name": MAIN_MODEL,
108 }])
109 @pytest.mark.parametrize("per_test_common_llm_kwargs", [{}])
110 @pytest.mark.parametrize("baseline_llm_kwargs", [{}])
@@ -110,17 +116,21 @@ def test_mlp_e2e_greedy_correctness(baseline_llm_generator, test_llm_generator,
116 @pytest.mark.parametrize("output_len", [2048])
117 @pytest.mark.parametrize("batch_size", [1, 32])
118 @pytest.mark.parametrize("seed", [1])
113 -def test_mlp_e2e_acceptance_rate(baseline_llm_generator, test_llm_generator,
114 - batch_size: int, output_len: int):
119 +def test_mlp_e2e_acceptance_rate(vllm_runner, common_llm_kwargs,
120 + per_test_common_llm_kwargs,
121 + baseline_llm_kwargs, test_llm_kwargs,
122 + batch_size: int, output_len: int, seed: int):
123 """Verify acceptance rate with different batch size and large output
124 length."""
117 - run_equality_correctness_test(baseline_llm_generator,
118 - test_llm_generator,
125 + run_equality_correctness_test(vllm_runner,
126 + common_llm_kwargs,
127 + per_test_common_llm_kwargs,
128 + baseline_llm_kwargs,
129 + test_llm_kwargs,
130 batch_size,
131 max_output_len=output_len,
132 temperature=0.0,
122 - seeded=True,
123 - force_output_len=True,
133 + seed=seed,
134 expected_acceptance_rate=0.48)
135
136
@@ -140,7 +150,7 @@ def test_mlp_e2e_acceptance_rate(baseline_llm_generator, test_llm_generator,
150 "dtype": PRECISION,
151
152 # Main model
143 - "model": MAIN_MODEL,
153 + "model_name": MAIN_MODEL,
154
155 # Speculative model
156 "speculative_model": SPEC_MODEL,
@@ -151,28 +161,35 @@ def test_mlp_e2e_acceptance_rate(baseline_llm_generator, test_llm_generator,
161 @pytest.mark.parametrize("output_len", [64])
162 @pytest.mark.parametrize("batch_size", [1, 32])
163 @pytest.mark.parametrize("temperature", [0.1, 1.0])
154 -@pytest.mark.parametrize("seed", [None])
155 -def test_mlp_e2e_seeded_correctness(baseline_llm_generator, test_llm_generator,
164 +@pytest.mark.parametrize("seed", [1])
165 +def test_mlp_e2e_seeded_correctness(vllm_runner, common_llm_kwargs,
166 + per_test_common_llm_kwargs,
167 + baseline_llm_kwargs, test_llm_kwargs,
168 batch_size: int, output_len: int,
157 - temperature: float):
169 + temperature: float, seed: int):
170 """Verify seeded runs produce the same output."""
159 - run_equality_correctness_test(baseline_llm_generator,
160 - test_llm_generator,
171 + run_equality_correctness_test(vllm_runner,
172 + common_llm_kwargs,
173 + per_test_common_llm_kwargs,
174 + baseline_llm_kwargs,
175 + test_llm_kwargs,
176 batch_size,
177 max_output_len=output_len,
178 temperature=temperature,
164 - seeded=True,
165 - force_output_len=True)
179 + seed=seed)
180
181 # Ensure this same test does fail if we _don't_ include per-request seeds
182 with pytest.raises(AssertionError):
169 - run_equality_correctness_test(baseline_llm_generator,
170 - test_llm_generator,
183 + run_equality_correctness_test(vllm_runner,
184 + common_llm_kwargs,
185 + per_test_common_llm_kwargs,
186 + baseline_llm_kwargs,
187 + test_llm_kwargs,
188 batch_size,
189 max_output_len=output_len,
190 temperature=temperature,
174 - seeded=False,
175 - force_output_len=True)
191 + seed=seed,
192 + disable_seed=True)
193
194
195 @pytest.mark.parametrize(
@@ -193,7 +210,7 @@ def test_mlp_e2e_seeded_correctness(baseline_llm_generator, test_llm_generator,
210 "dtype": PRECISION,
211
212 # Main model
196 - "model": MAIN_MODEL,
213 + "model_name": MAIN_MODEL,
214 }])
215 @pytest.mark.parametrize("per_test_common_llm_kwargs", [{}])
216 @pytest.mark.parametrize("baseline_llm_kwargs", [{}])
@@ -210,18 +227,22 @@ def test_mlp_e2e_seeded_correctness(baseline_llm_generator, test_llm_generator,
227 ])
228 @pytest.mark.parametrize("batch_size", [4])
229 @pytest.mark.parametrize("seed", [1])
213 -def test_mlp_e2e_greedy_correctness_with_preemption(baseline_llm_generator,
214 - test_llm_generator,
215 - batch_size: int,
216 - output_len: int):
230 +def test_mlp_e2e_greedy_correctness_with_preemption(
231 + vllm_runner, common_llm_kwargs, per_test_common_llm_kwargs,
232 + baseline_llm_kwargs, test_llm_kwargs, batch_size: int, output_len: int,
233 + seed: int):
234 """Verify greedy equality, even when some sequences are preempted mid-
235 generation.
236 """
220 - run_greedy_equality_correctness_test(baseline_llm_generator,
221 - test_llm_generator,
222 - batch_size,
223 - max_output_len=output_len,
224 - force_output_len=True)
237 + run_equality_correctness_test(vllm_runner,
238 + common_llm_kwargs,
239 + per_test_common_llm_kwargs,
240 + baseline_llm_kwargs,
241 + test_llm_kwargs,
242 + batch_size,
243 + max_output_len=output_len,
244 + seed=seed,
245 + temperature=0.0)
246
247
248 @pytest.mark.parametrize(
@@ -242,7 +263,7 @@ def test_mlp_e2e_greedy_correctness_with_preemption(baseline_llm_generator,
263 "dtype": PRECISION,
264
265 # Main model
245 - "model": MAIN_MODEL,
266 + "model_name": MAIN_MODEL,
267 }])
268 @pytest.mark.parametrize("per_test_common_llm_kwargs", [{}])
269 @pytest.mark.parametrize("baseline_llm_kwargs", [{}])
@@ -259,10 +280,10 @@ def test_mlp_e2e_greedy_correctness_with_preemption(baseline_llm_generator,
280 ])
281 @pytest.mark.parametrize("batch_size", [4])
282 @pytest.mark.parametrize("seed", [1])
262 -def test_mlp_e2e_greedy_correctness_with_padding(baseline_llm_generator,
263 - test_llm_generator,
264 - batch_size: int,
265 - output_len: int):
283 +def test_mlp_e2e_greedy_correctness_with_padding(
284 + vllm_runner, common_llm_kwargs, per_test_common_llm_kwargs,
285 + baseline_llm_kwargs, test_llm_kwargs, batch_size: int, output_len: int,
286 + seed: int):
287 """Verify greedy equality when the vocab dimension is padded
288 """
289
@@ -273,11 +294,15 @@ def test_mlp_e2e_greedy_correctness_with_padding(baseline_llm_generator,
294 with patch(
295 "vllm.model_executor.layers.vocab_parallel_embedding.pad_vocab_size",
296 patched_pad_vocab_size):
276 - run_greedy_equality_correctness_test(baseline_llm_generator,
277 - test_llm_generator,
278 - batch_size,
279 - max_output_len=output_len,
280 - force_output_len=True)
297 + run_equality_correctness_test(vllm_runner,
298 + common_llm_kwargs,
299 + per_test_common_llm_kwargs,
300 + baseline_llm_kwargs,
301 + test_llm_kwargs,
302 + batch_size,
303 + max_output_len=output_len,
304 + seed=seed,
305 + temperature=0.0)
306
307
308 @pytest.mark.parametrize(
@@ -293,7 +318,7 @@ def test_mlp_e2e_greedy_correctness_with_padding(baseline_llm_generator,
318 "dtype": PRECISION,
319
320 # Main model
296 - "model": MAIN_MODEL,
321 + "model_name": MAIN_MODEL,
322 }])
323 @pytest.mark.parametrize("per_test_common_llm_kwargs", [{}])
324 @pytest.mark.parametrize("baseline_llm_kwargs", [{}])
@@ -315,16 +340,22 @@ def test_mlp_e2e_greedy_correctness_with_padding(baseline_llm_generator,
340 32,
341 ])
342 @pytest.mark.parametrize("seed", [1])
318 -def test_mlp_different_k(baseline_llm_generator, test_llm_generator,
319 - batch_size: int, output_len: int):
343 +def test_mlp_different_k(vllm_runner, common_llm_kwargs,
344 + per_test_common_llm_kwargs, baseline_llm_kwargs,
345 + test_llm_kwargs, batch_size: int, seed: int,
346 + output_len: int):
347 """Verify that mlp speculative decoding produces exact equality
348 to without spec decode with different values of num_speculative_tokens.
349 """
323 - run_greedy_equality_correctness_test(baseline_llm_generator,
324 - test_llm_generator,
325 - batch_size,
326 - max_output_len=output_len,
327 - force_output_len=True)
350 + run_equality_correctness_test(vllm_runner,
351 + common_llm_kwargs,
352 + per_test_common_llm_kwargs,
353 + baseline_llm_kwargs,
354 + test_llm_kwargs,
355 + batch_size,
356 + max_output_len=output_len,
357 + seed=seed,
358 + temperature=0.0)
359
360
361 @pytest.mark.parametrize(
@@ -340,7 +371,7 @@ def test_mlp_different_k(baseline_llm_generator, test_llm_generator,
371 "dtype": PRECISION,
372
373 # Main model
343 - "model": MAIN_MODEL,
374 + "model_name": MAIN_MODEL,
375 }])
376 @pytest.mark.parametrize("per_test_common_llm_kwargs", [{}])
377 @pytest.mark.parametrize("baseline_llm_kwargs", [{}])
@@ -357,14 +388,20 @@ def test_mlp_different_k(baseline_llm_generator, test_llm_generator,
388 32,
389 ])
390 @pytest.mark.parametrize("seed", [1])
360 -def test_mlp_disable_queue(baseline_llm_generator, test_llm_generator,
361 - batch_size: int, output_len: int):
391 +def test_mlp_disable_queue(vllm_runner, common_llm_kwargs,
392 + per_test_common_llm_kwargs, baseline_llm_kwargs,
393 + test_llm_kwargs, batch_size: int, seed: int,
394 + output_len: int):
395 """Verify that mlp speculative decoding produces exact equality
396 to without spec decode when speculation is disabled for large
397 batch sizes.
398 """
366 - run_greedy_equality_correctness_test(baseline_llm_generator,
367 - test_llm_generator,
368 - batch_size,
369 - max_output_len=output_len,
370 - force_output_len=True)
399 + run_equality_correctness_test(vllm_runner,
400 + common_llm_kwargs,
401 + per_test_common_llm_kwargs,
402 + baseline_llm_kwargs,
403 + test_llm_kwargs,
404 + batch_size,
405 + max_output_len=output_len,
406 + seed=seed,
407 + temperature=0.0)
tests/spec_decode/e2e/test_multistep_correctness.py
+171 -136
@@ -41,8 +41,9 @@ from transformers import AutoTokenizer
41
42 from vllm import SamplingParams
43
44 +from ...utils import fork_new_process_for_each_test
45 from .conftest import (get_output_from_llm_generator,
45 - run_greedy_equality_correctness_test)
46 + run_equality_correctness_test)
47
48
49 @pytest.mark.parametrize(
@@ -73,6 +74,7 @@ from .conftest import (get_output_from_llm_generator,
74 @pytest.mark.parametrize("test_llm_kwargs", [{}])
75 @pytest.mark.parametrize("batch_size", [1, 32])
76 @pytest.mark.parametrize("seed", [1])
77 +@fork_new_process_for_each_test
78 def test_spec_decode_e2e_with_detokenization(test_llm_generator,
79 batch_size: int):
80 """Run generation with speculative decoding on a batch. Verify the engine
@@ -116,44 +118,6 @@ def test_spec_decode_e2e_with_detokenization(test_llm_generator,
118 assert actual_tokens.strip() == expected_tokens.strip()
119
120
119 -@pytest.mark.parametrize(
120 - "common_llm_kwargs",
121 - [{
122 - # Use a small model for a fast test.
123 - # Note this is repeated in the test body; to initialize a tokenizer.
124 - "model": "JackFram/llama-68m",
125 -
126 - # Skip cuda graph recording for fast test.
127 - "enforce_eager": True,
128 -
129 - # Required for spec decode.
130 - "use_v2_block_manager": True,
131 -
132 - # Use AsyncLLM engine
133 - "use_async": True,
134 - }])
135 -@pytest.mark.parametrize("baseline_llm_kwargs", [{}])
136 -@pytest.mark.parametrize("per_test_common_llm_kwargs", [
137 - {
138 - "speculative_model": "JackFram/llama-68m",
139 - "num_speculative_tokens": 5,
140 - },
141 -])
142 -@pytest.mark.parametrize("test_llm_kwargs", [{}])
143 -@pytest.mark.parametrize("batch_size", [2])
144 -@pytest.mark.parametrize("seed", [1])
145 -def test_spec_decode_e2e_with_async_engine(test_llm_generator,
146 - baseline_llm_generator,
147 - batch_size: int):
148 - """Verify spec decode works well with async LLM engine.
149 - """
150 - run_greedy_equality_correctness_test(baseline_llm_generator,
151 - test_llm_generator,
152 - batch_size,
153 - max_output_len=32,
154 - force_output_len=True)
155 -
156 -
121 @pytest.mark.parametrize(
122 "common_llm_kwargs",
123 [{
@@ -172,10 +136,10 @@ def test_spec_decode_e2e_with_async_engine(test_llm_generator,
136 # Try two different tiny base models.
137 # Note that one is equal to the draft model, another isn't.
138 {
175 - "model": "JackFram/llama-68m",
139 + "model_name": "JackFram/llama-68m",
140 },
141 {
178 - "model": "JackFram/llama-160m",
142 + "model_name": "JackFram/llama-160m",
143 },
144 ])
145 @pytest.mark.parametrize("baseline_llm_kwargs", [{}])
@@ -189,13 +153,15 @@ def test_spec_decode_e2e_with_async_engine(test_llm_generator,
153 "output_len",
154 [
155 # Use long output len for the small model test.
192 - 1536,
156 + 10,
157 ])
158 @pytest.mark.parametrize("batch_size", [1])
159 @pytest.mark.parametrize("seed", [1])
160 +@fork_new_process_for_each_test
161 def test_spec_decode_e2e_greedy_correctness_tiny_model_bs1(
197 - baseline_llm_generator, test_llm_generator, batch_size: int,
198 - output_len: int):
162 + vllm_runner, common_llm_kwargs, per_test_common_llm_kwargs,
163 + baseline_llm_kwargs, test_llm_kwargs, batch_size: int, output_len: int,
164 + seed: int):
165 """Verify greedy equality on a tiny model with batch size of one.
166
167 Since this test is cheaper than other e2e correctness tests, we generate
@@ -204,14 +170,18 @@ def test_spec_decode_e2e_greedy_correctness_tiny_model_bs1(
170 When the draft model is the same as the target model, we further check
171 whether all speculative tokens are accepted.
172 """
207 - ensure_all_accepted = test_llm_generator.same_draft_target_model
208 - run_greedy_equality_correctness_test(
209 - baseline_llm_generator,
210 - test_llm_generator,
211 - batch_size,
212 - max_output_len=output_len,
213 - force_output_len=True,
214 - ensure_all_accepted=ensure_all_accepted)
173 + ensure_all_accepted = per_test_common_llm_kwargs.get(
174 + "model_name") == test_llm_kwargs.get("speculative_model")
175 + run_equality_correctness_test(vllm_runner,
176 + common_llm_kwargs,
177 + per_test_common_llm_kwargs,
178 + baseline_llm_kwargs,
179 + test_llm_kwargs,
180 + batch_size,
181 + max_output_len=output_len,
182 + seed=seed,
183 + temperature=0.0,
184 + ensure_all_accepted=ensure_all_accepted)
185
186
187 @pytest.mark.parametrize(
@@ -232,10 +202,10 @@ def test_spec_decode_e2e_greedy_correctness_tiny_model_bs1(
202 # Try two different tiny base models.
203 # Note that one is equal to the draft model, another isn't.
204 {
235 - "model": "JackFram/llama-68m",
205 + "model_name": "JackFram/llama-68m",
206 },
207 {
238 - "model": "JackFram/llama-160m",
208 + "model_name": "JackFram/llama-160m",
209 },
210 ])
211 @pytest.mark.parametrize("baseline_llm_kwargs", [{}])
@@ -253,16 +223,22 @@ def test_spec_decode_e2e_greedy_correctness_tiny_model_bs1(
223 ])
224 @pytest.mark.parametrize("batch_size", [64])
225 @pytest.mark.parametrize("seed", [1])
226 +@fork_new_process_for_each_test
227 def test_spec_decode_e2e_greedy_correctness_tiny_model_large_bs(
257 - baseline_llm_generator, test_llm_generator, batch_size: int,
258 - output_len: int):
228 + vllm_runner, common_llm_kwargs, per_test_common_llm_kwargs,
229 + baseline_llm_kwargs, test_llm_kwargs, batch_size: int, output_len: int,
230 + seed: int):
231 """Verify greedy equality on a tiny model and large batch size.
232 """
261 - run_greedy_equality_correctness_test(baseline_llm_generator,
262 - test_llm_generator,
263 - batch_size,
264 - max_output_len=output_len,
265 - force_output_len=True)
233 + run_equality_correctness_test(vllm_runner,
234 + common_llm_kwargs,
235 + per_test_common_llm_kwargs,
236 + baseline_llm_kwargs,
237 + test_llm_kwargs,
238 + batch_size,
239 + max_output_len=output_len,
240 + seed=seed,
241 + temperature=0.0)
242
243
244 @pytest.mark.parametrize(
@@ -280,10 +256,10 @@ def test_spec_decode_e2e_greedy_correctness_tiny_model_large_bs(
256 # Try two different tiny base models.
257 # Note that one is equal to the draft model, another isn't.
258 {
283 - "model": "JackFram/llama-68m",
259 + "model_name": "JackFram/llama-68m",
260 },
261 {
286 - "model": "JackFram/llama-160m",
262 + "model_name": "JackFram/llama-160m",
263 },
264 ])
265 @pytest.mark.parametrize("baseline_llm_kwargs", [{}])
@@ -298,24 +274,31 @@ def test_spec_decode_e2e_greedy_correctness_tiny_model_large_bs(
274 ])
275 @pytest.mark.parametrize("batch_size", [32])
276 @pytest.mark.parametrize("seed", [1])
277 +@fork_new_process_for_each_test
278 def test_spec_decode_e2e_greedy_correctness_tiny_model_large_bs_diff_output_len(
302 - baseline_llm_generator, test_llm_generator, batch_size: int,
303 - max_output_len: int):
279 + vllm_runner, common_llm_kwargs, per_test_common_llm_kwargs,
280 + baseline_llm_kwargs, test_llm_kwargs, batch_size: int,
281 + max_output_len: int, seed: int):
282 """Verify greedy equality on a tiny model, with a large batch size, and when
283 sampling respects the EOS token.
284 """
307 - run_greedy_equality_correctness_test(baseline_llm_generator,
308 - test_llm_generator,
309 - batch_size,
310 - max_output_len,
311 - force_output_len=False)
285 + run_equality_correctness_test(vllm_runner,
286 + common_llm_kwargs,
287 + per_test_common_llm_kwargs,
288 + baseline_llm_kwargs,
289 + test_llm_kwargs,
290 + batch_size,
291 + max_output_len,
292 + seed=seed,
293 + temperature=0.0,
294 + ignore_eos=False)
295
296
297 @pytest.mark.parametrize(
298 "common_llm_kwargs",
299 [{
300 # A "real" model (not tiny).
318 - "model": "meta-llama/Llama-2-7b-chat-hf",
301 + "model_name": "meta-llama/Llama-2-7b-chat-hf",
302
303 # Skip cuda graph recording for fast test.
304 "enforce_eager": True,
@@ -342,24 +325,30 @@ def test_spec_decode_e2e_greedy_correctness_tiny_model_large_bs_diff_output_len(
325 256,
326 ])
327 @pytest.mark.parametrize("seed", [1])
328 +@fork_new_process_for_each_test
329 def test_spec_decode_e2e_greedy_correctness_real_model_bs1(
346 - baseline_llm_generator, test_llm_generator, batch_size: int,
347 - output_len: int):
330 + vllm_runner, common_llm_kwargs, per_test_common_llm_kwargs,
331 + baseline_llm_kwargs, test_llm_kwargs, batch_size: int, output_len: int,
332 + seed: int):
333 """Verify greedy equality on a "real" model and batch size of 1. This is
334 separate from large BS tests to make identifying the source of bugs easier.
335 """
351 - run_greedy_equality_correctness_test(baseline_llm_generator,
352 - test_llm_generator,
353 - batch_size,
354 - max_output_len=output_len,
355 - force_output_len=True)
336 + run_equality_correctness_test(vllm_runner,
337 + common_llm_kwargs,
338 + per_test_common_llm_kwargs,
339 + baseline_llm_kwargs,
340 + test_llm_kwargs,
341 + batch_size,
342 + max_output_len=output_len,
343 + seed=seed,
344 + temperature=0.0)
345
346
347 @pytest.mark.parametrize(
348 "common_llm_kwargs",
349 [{
350 # A "real" model (not tiny).
362 - "model": "meta-llama/Llama-2-7b-chat-hf",
351 + "model_name": "meta-llama/Llama-2-7b-chat-hf",
352
353 # Skip cuda graph recording for fast test.
354 "enforce_eager": True,
@@ -386,17 +375,23 @@ def test_spec_decode_e2e_greedy_correctness_real_model_bs1(
375 64,
376 ])
377 @pytest.mark.parametrize("seed", [1])
378 +@fork_new_process_for_each_test
379 def test_spec_decode_e2e_greedy_correctness_real_model_large_bs(
390 - baseline_llm_generator, test_llm_generator, batch_size: int,
391 - output_len: int):
380 + vllm_runner, common_llm_kwargs, per_test_common_llm_kwargs,
381 + baseline_llm_kwargs, test_llm_kwargs, batch_size: int, output_len: int,
382 + seed: int):
383 """Verify greedy equality with a "real" model on a nontrivial batch size.
384 This is the closest test to a real production workload.
385 """
395 - run_greedy_equality_correctness_test(baseline_llm_generator,
396 - test_llm_generator,
397 - batch_size,
398 - max_output_len=output_len,
399 - force_output_len=True)
386 + run_equality_correctness_test(vllm_runner,
387 + common_llm_kwargs,
388 + per_test_common_llm_kwargs,
389 + baseline_llm_kwargs,
390 + test_llm_kwargs,
391 + batch_size,
392 + max_output_len=output_len,
393 + seed=seed,
394 + temperature=0.0)
395
396
397 @pytest.mark.parametrize(
@@ -415,7 +410,7 @@ def test_spec_decode_e2e_greedy_correctness_real_model_large_bs(
410 }])
411 @pytest.mark.parametrize("per_test_common_llm_kwargs", [
412 {
418 - "model": "JackFram/llama-160m",
413 + "model_name": "JackFram/llama-160m",
414 },
415 ])
416 @pytest.mark.parametrize("baseline_llm_kwargs", [{}])
@@ -433,23 +428,29 @@ def test_spec_decode_e2e_greedy_correctness_real_model_large_bs(
428 ])
429 @pytest.mark.parametrize("batch_size", [4])
430 @pytest.mark.parametrize("seed", [1])
431 +@fork_new_process_for_each_test
432 def test_spec_decode_e2e_greedy_correctness_with_preemption(
437 - baseline_llm_generator, test_llm_generator, batch_size: int,
438 - output_len: int):
433 + vllm_runner, common_llm_kwargs, per_test_common_llm_kwargs,
434 + baseline_llm_kwargs, test_llm_kwargs, batch_size: int, output_len: int,
435 + seed: int):
436 """Verify greedy equality, even when some sequences are preempted mid-
437 generation.
438 """
442 - run_greedy_equality_correctness_test(baseline_llm_generator,
443 - test_llm_generator,
444 - batch_size,
445 - max_output_len=output_len,
446 - force_output_len=True)
439 + run_equality_correctness_test(vllm_runner,
440 + common_llm_kwargs,
441 + per_test_common_llm_kwargs,
442 + baseline_llm_kwargs,
443 + test_llm_kwargs,
444 + batch_size,
445 + max_output_len=output_len,
446 + seed=seed,
447 + temperature=0.0)
448
449
450 @pytest.mark.parametrize(
451 "common_llm_kwargs",
452 [{
452 - "model": "JackFram/llama-160m",
453 + "model_name": "JackFram/llama-160m",
454
455 # Skip cuda graph recording for fast test.
456 "enforce_eager": True,
@@ -487,22 +488,29 @@ def test_spec_decode_e2e_greedy_correctness_with_preemption(
488 32,
489 ])
490 @pytest.mark.parametrize("seed", [1])
490 -def test_spec_decode_different_block_size(baseline_llm_generator,
491 - test_llm_generator, batch_size: int,
492 - output_len: int):
491 +@fork_new_process_for_each_test
492 +def test_spec_decode_different_block_size(vllm_runner, common_llm_kwargs,
493 + per_test_common_llm_kwargs,
494 + baseline_llm_kwargs, test_llm_kwargs,
495 + batch_size: int, output_len: int,
496 + seed: int):
497 """Verify greedy equality over different block sizes.
498 """
495 - run_greedy_equality_correctness_test(baseline_llm_generator,
496 - test_llm_generator,
497 - batch_size,
498 - max_output_len=output_len,
499 - force_output_len=True)
499 + run_equality_correctness_test(vllm_runner,
500 + common_llm_kwargs,
501 + per_test_common_llm_kwargs,
502 + baseline_llm_kwargs,
503 + test_llm_kwargs,
504 + batch_size,
505 + max_output_len=output_len,
506 + seed=seed,
507 + temperature=0.0)
508
509
510 @pytest.mark.parametrize(
511 "common_llm_kwargs",
512 [{
505 - "model": "JackFram/llama-160m",
513 + "model_name": "JackFram/llama-160m",
514
515 # Skip cuda graph recording for fast test.
516 "enforce_eager": True,
@@ -534,24 +542,31 @@ def test_spec_decode_different_block_size(baseline_llm_generator,
542 64,
543 ])
544 @pytest.mark.parametrize("seed", [1])
537 -def test_skip_speculation(baseline_llm_generator, test_llm_generator,
538 - batch_size: int, output_len: int):
545 +@fork_new_process_for_each_test
546 +def test_skip_speculation(vllm_runner, common_llm_kwargs,
547 + per_test_common_llm_kwargs, baseline_llm_kwargs,
548 + test_llm_kwargs, batch_size: int, output_len: int,
549 + seed: int):
550 """Verify greedy equality when some (or all) sequences skip speculation.
551 We do this by setting the max model len of the draft model to an
552 artificially low value, such that when the sequences grow beyond it, they
553 are skipped in speculative decoding.
554 """
544 - run_greedy_equality_correctness_test(baseline_llm_generator,
545 - test_llm_generator,
546 - batch_size,
547 - max_output_len=output_len,
548 - force_output_len=True)
555 + run_equality_correctness_test(vllm_runner,
556 + common_llm_kwargs,
557 + per_test_common_llm_kwargs,
558 + baseline_llm_kwargs,
559 + test_llm_kwargs,
560 + batch_size,
561 + max_output_len=output_len,
562 + seed=seed,
563 + temperature=0.0)
564
565
566 @pytest.mark.parametrize(
567 "common_llm_kwargs",
568 [{
554 - "model": "JackFram/llama-160m",
569 + "model_name": "JackFram/llama-160m",
570
571 # Skip cuda graph recording for fast test.
572 "enforce_eager": True,
@@ -571,21 +586,28 @@ def test_skip_speculation(baseline_llm_generator, test_llm_generator,
586 @pytest.mark.parametrize("batch_size", [8])
587 @pytest.mark.parametrize("output_len", [10])
588 @pytest.mark.parametrize("seed", [1])
574 -def test_disable_speculation(baseline_llm_generator, test_llm_generator,
575 - batch_size: int, output_len: int):
589 +@fork_new_process_for_each_test
590 +def test_disable_speculation(vllm_runner, common_llm_kwargs,
591 + per_test_common_llm_kwargs, baseline_llm_kwargs,
592 + test_llm_kwargs, batch_size: int, output_len: int,
593 + seed: int):
594 """Verify greedy equality when all sequences disable speculation.
595 """
578 - run_greedy_equality_correctness_test(baseline_llm_generator,
579 - test_llm_generator,
580 - batch_size,
581 - max_output_len=output_len,
582 - force_output_len=True)
596 + run_equality_correctness_test(vllm_runner,
597 + common_llm_kwargs,
598 + per_test_common_llm_kwargs,
599 + baseline_llm_kwargs,
600 + test_llm_kwargs,
601 + batch_size,
602 + max_output_len=output_len,
603 + seed=seed,
604 + temperature=0.0)
605
606
607 @pytest.mark.parametrize(
608 "common_llm_kwargs",
609 [{
588 - "model": "JackFram/llama-68m",
610 + "model_name": "JackFram/llama-68m",
611
612 # Skip cuda graph recording for fast test.
613 "enforce_eager": True,
@@ -613,22 +635,28 @@ def test_disable_speculation(baseline_llm_generator, test_llm_generator,
635 32,
636 ])
637 @pytest.mark.parametrize("seed", [1])
616 -def test_many_k(baseline_llm_generator, test_llm_generator, batch_size: int,
617 - output_len: int):
638 +@fork_new_process_for_each_test
639 +def test_many_k(vllm_runner, common_llm_kwargs, per_test_common_llm_kwargs,
640 + baseline_llm_kwargs, test_llm_kwargs, batch_size: int,
641 + output_len: int, seed: int):
642 """Verify that speculative decoding produces exact equality to without spec
643 decode with many different values of k.
644 """
621 - run_greedy_equality_correctness_test(baseline_llm_generator,
622 - test_llm_generator,
623 - batch_size,
624 - max_output_len=output_len,
625 - force_output_len=True)
645 + run_equality_correctness_test(vllm_runner,
646 + common_llm_kwargs,
647 + per_test_common_llm_kwargs,
648 + baseline_llm_kwargs,
649 + test_llm_kwargs,
650 + batch_size,
651 + max_output_len=output_len,
652 + seed=seed,
653 + temperature=0.0)
654
655
656 @pytest.mark.parametrize(
657 "common_llm_kwargs",
658 [{
631 - "model": "JackFram/llama-160m",
659 + "model_name": "JackFram/llama-160m",
660
661 # Skip cuda graph recording for fast test.
662 "enforce_eager": True,
@@ -657,15 +685,22 @@ def test_many_k(baseline_llm_generator, test_llm_generator, batch_size: int,
685 32,
686 ])
687 @pytest.mark.parametrize("seed", [1])
660 -def test_typical_acceptance_sampling(baseline_llm_generator,
661 - test_llm_generator, batch_size: int,
662 - output_len: int):
688 +@fork_new_process_for_each_test
689 +def test_typical_acceptance_sampling(vllm_runner, common_llm_kwargs,
690 + per_test_common_llm_kwargs,
691 + baseline_llm_kwargs, test_llm_kwargs,
692 + batch_size: int, output_len: int,
693 + seed: int):
694 """Verify that speculative decoding produces exact equality to without spec
695 decode with TypicalAcceptanceSampler as the draft token acceptance
696 sampling method.
697 """
667 - run_greedy_equality_correctness_test(baseline_llm_generator,
668 - test_llm_generator,
669 - batch_size,
670 - max_output_len=output_len,
671 - force_output_len=True)
698 + run_equality_correctness_test(vllm_runner,
699 + common_llm_kwargs,
700 + per_test_common_llm_kwargs,
701 + baseline_llm_kwargs,
702 + test_llm_kwargs,
703 + batch_size,
704 + max_output_len=output_len,
705 + seed=seed,
706 + temperature=0.0)
tests/spec_decode/e2e/test_ngram_correctness.py
+58 -36
@@ -26,7 +26,7 @@ for the target model outputs.
26
27 import pytest
28
29 -from .conftest import run_greedy_equality_correctness_test
29 +from .conftest import run_equality_correctness_test
30
31
32 @pytest.mark.parametrize(
@@ -43,7 +43,7 @@ from .conftest import run_greedy_equality_correctness_test
43 }])
44 @pytest.mark.parametrize("per_test_common_llm_kwargs", [
45 {
46 - "model": "JackFram/llama-68m",
46 + "model_name": "JackFram/llama-68m",
47 },
48 ])
49 @pytest.mark.parametrize("baseline_llm_kwargs", [{}])
@@ -59,15 +59,21 @@ from .conftest import run_greedy_equality_correctness_test
59 ])
60 @pytest.mark.parametrize("batch_size", [1, 32])
61 @pytest.mark.parametrize("seed", [1])
62 -def test_ngram_e2e_greedy_correctness(baseline_llm_generator,
63 - test_llm_generator, batch_size: int,
64 - output_len: int):
62 +def test_ngram_e2e_greedy_correctness(vllm_runner, common_llm_kwargs,
63 + per_test_common_llm_kwargs,
64 + baseline_llm_kwargs, test_llm_kwargs,
65 + batch_size: int, output_len: int,
66 + seed: int):
67 """Verify greedy equality on a tiny model with different batch size."""
66 - run_greedy_equality_correctness_test(baseline_llm_generator,
67 - test_llm_generator,
68 - batch_size,
69 - max_output_len=output_len,
70 - force_output_len=True)
68 + run_equality_correctness_test(vllm_runner,
69 + common_llm_kwargs,
70 + per_test_common_llm_kwargs,
71 + baseline_llm_kwargs,
72 + test_llm_kwargs,
73 + batch_size,
74 + max_output_len=output_len,
75 + seed=seed,
76 + temperature=0.0)
77
78
79 @pytest.mark.parametrize(
@@ -86,7 +92,7 @@ def test_ngram_e2e_greedy_correctness(baseline_llm_generator,
92 }])
93 @pytest.mark.parametrize("per_test_common_llm_kwargs", [
94 {
89 - "model": "JackFram/llama-160m",
95 + "model_name": "JackFram/llama-160m",
96 },
97 ])
98 @pytest.mark.parametrize("baseline_llm_kwargs", [{}])
@@ -105,24 +111,28 @@ def test_ngram_e2e_greedy_correctness(baseline_llm_generator,
111 ])
112 @pytest.mark.parametrize("batch_size", [4])
113 @pytest.mark.parametrize("seed", [1])
108 -def test_ngram_e2e_greedy_correctness_with_preemption(baseline_llm_generator,
109 - test_llm_generator,
110 - batch_size: int,
111 - output_len: int):
114 +def test_ngram_e2e_greedy_correctness_with_preemption(
115 + vllm_runner, common_llm_kwargs, per_test_common_llm_kwargs,
116 + baseline_llm_kwargs, test_llm_kwargs, batch_size: int, output_len: int,
117 + seed: int):
118 """Verify greedy equality, even when some sequences are preempted mid-
119 generation.
120 """
115 - run_greedy_equality_correctness_test(baseline_llm_generator,
116 - test_llm_generator,
117 - batch_size,
118 - max_output_len=output_len,
119 - force_output_len=True)
121 + run_equality_correctness_test(vllm_runner,
122 + common_llm_kwargs,
123 + per_test_common_llm_kwargs,
124 + baseline_llm_kwargs,
125 + test_llm_kwargs,
126 + batch_size,
127 + max_output_len=output_len,
128 + temperature=0,
129 + seed=seed)
130
131
132 @pytest.mark.parametrize(
133 "common_llm_kwargs",
134 [{
125 - "model": "JackFram/llama-68m",
135 + "model_name": "JackFram/llama-68m",
136
137 # Skip cuda graph recording for fast test.
138 "enforce_eager": True,
@@ -159,23 +169,29 @@ def test_ngram_e2e_greedy_correctness_with_preemption(baseline_llm_generator,
169 32,
170 ])
171 @pytest.mark.parametrize("seed", [1])
162 -def test_ngram_different_k(baseline_llm_generator, test_llm_generator,
163 - batch_size: int, output_len: int):
172 +def test_ngram_different_k(vllm_runner, common_llm_kwargs,
173 + per_test_common_llm_kwargs, baseline_llm_kwargs,
174 + test_llm_kwargs, batch_size: int, output_len: int,
175 + seed: int):
176 """Verify that ngram speculative decoding produces exact equality
177 to without spec decode with many different values of k and
178 different ngram_prompt_lookup_max.
179 """
168 - run_greedy_equality_correctness_test(baseline_llm_generator,
169 - test_llm_generator,
170 - batch_size,
171 - max_output_len=output_len,
172 - force_output_len=True)
180 + run_equality_correctness_test(vllm_runner,
181 + common_llm_kwargs,
182 + per_test_common_llm_kwargs,
183 + baseline_llm_kwargs,
184 + test_llm_kwargs,
185 + batch_size,
186 + max_output_len=output_len,
187 + seed=seed,
188 + temperature=0.0)
189
190
191 @pytest.mark.parametrize(
192 "common_llm_kwargs",
193 [{
178 - "model": "JackFram/llama-68m",
194 + "model_name": "JackFram/llama-68m",
195
196 # Skip cuda graph recording for fast test.
197 "enforce_eager": True,
@@ -200,14 +216,20 @@ def test_ngram_different_k(baseline_llm_generator, test_llm_generator,
216 32,
217 ])
218 @pytest.mark.parametrize("seed", [1])
203 -def test_ngram_disable_queue(baseline_llm_generator, test_llm_generator,
204 - batch_size: int, output_len: int):
219 +def test_ngram_disable_queue(vllm_runner, common_llm_kwargs,
220 + per_test_common_llm_kwargs, baseline_llm_kwargs,
221 + test_llm_kwargs, batch_size: int, output_len: int,
222 + seed: int):
223 """Verify that ngram speculative decoding produces exact equality
224 to without spec decode with many different values of k and
225 different ngram_prompt_lookup_max.
226 """
209 - run_greedy_equality_correctness_test(baseline_llm_generator,
210 - test_llm_generator,
211 - batch_size,
212 - max_output_len=output_len,
213 - force_output_len=True)
227 + run_equality_correctness_test(vllm_runner,
228 + common_llm_kwargs,
229 + per_test_common_llm_kwargs,
230 + baseline_llm_kwargs,
231 + test_llm_kwargs,
232 + batch_size,
233 + max_output_len=output_len,
234 + seed=seed,
235 + temperature=0.0)
tests/spec_decode/e2e/test_seed.py
+33 -19
@@ -2,11 +2,17 @@ import pytest
2
3 from .conftest import run_equality_correctness_test
4
5 +# main model
6 +MAIN_MODEL = "JackFram/llama-68m"
7 +
8 +# speculative model
9 +SPEC_MODEL = "JackFram/llama-160m"
10 +
11
12 @pytest.mark.parametrize(
13 "common_llm_kwargs",
14 [{
9 - "model": "JackFram/llama-68m",
15 + "model_name": "JackFram/llama-68m",
16
17 # Skip cuda graph recording for fast test.
18 "enforce_eager": True,
@@ -31,26 +37,34 @@ from .conftest import run_equality_correctness_test
37 # Use smaller output len for fast test.
38 20,
39 ])
34 -@pytest.mark.parametrize("seed", [None])
35 -def test_seeded_consistency(baseline_llm_generator, test_llm_generator,
36 - batch_size: int, temperature: float,
37 - output_len: int):
40 +def test_seeded_consistency(vllm_runner, common_llm_kwargs,
41 + per_test_common_llm_kwargs, baseline_llm_kwargs,
42 + test_llm_kwargs, batch_size: int,
43 + temperature: float, output_len: int):
44 """Verify outputs are consistent across multiple runs with same seed
45 """
40 - run_equality_correctness_test(baseline_llm_generator,
41 - test_llm_generator,
42 - batch_size,
43 - max_output_len=output_len,
44 - temperature=temperature,
45 - seeded=True,
46 - force_output_len=True)
46 + run_equality_correctness_test(
47 + vllm_runner,
48 + common_llm_kwargs,
49 + per_test_common_llm_kwargs,
50 + baseline_llm_kwargs,
51 + test_llm_kwargs,
52 + batch_size,
53 + max_output_len=output_len,
54 + temperature=temperature,
55 + disable_seed=False,
56 + )
57
58 # Ensure this same test does fail if we _don't_ include per-request seeds
59 with pytest.raises(AssertionError):
50 - run_equality_correctness_test(baseline_llm_generator,
51 - test_llm_generator,
52 - batch_size,
53 - max_output_len=output_len,
54 - temperature=temperature,
55 - seeded=False,
56 - force_output_len=True)
60 + run_equality_correctness_test(
61 + vllm_runner,
62 + common_llm_kwargs,
63 + per_test_common_llm_kwargs,
64 + baseline_llm_kwargs,
65 + test_llm_kwargs,
66 + batch_size,
67 + max_output_len=output_len,
68 + temperature=temperature,
69 + disable_seed=True,
70 + )