@tduncan / Nvidia-Optimization-Stack-VLLM Mirror · read-only / commits / 01b6f9e1f0
Fork 0

A high-throughput and memory-efficient inference and serving engine for LLMs

Mirrored from https://github.com/tduncan-lum/Nvidia-Optimization-Stack---VLLM.git

Synced Nov 16, 15:45 UTC

[Core][Bugfix] Support prompt_logprobs returned with speculative decoding (#8047)

Signed-off-by: Travis Johnson <tsjohnso@us.ibm.com>

Travis Johnson committed Sep 24, 2024 at 18:29 UTC 01b6f9e1f0530a7cb81486ff34d3d935e4f75d28
14 files changed +492 -134
tests/conftest.py
+1 -3
@@ -675,8 +675,6 @@ class VllmRunner:
675 videos: Optional[PromptVideoInput] = None,
676 ) -> Union[List[TokensTextLogprobs],
677 List[TokensTextLogprobsPromptLogprobs]]:
678 - assert sampling_params.logprobs is not None
679 -
678 if images is not None:
679 assert len(prompts) == len(images)
680
@@ -754,7 +752,7 @@ class VllmRunner:
752 temperature=0.0,
753 max_tokens=max_tokens,
754 logprobs=num_logprobs,
757 - prompt_logprobs=(num_prompt_logprobs),
755 + prompt_logprobs=num_prompt_logprobs,
756 stop_token_ids=stop_token_ids)
757
758 return self.generate_w_logprobs(prompts,
tests/spec_decode/e2e/conftest.py
+92 -47
@@ -1,13 +1,16 @@
1 from itertools import cycle
2 -from typing import List, Optional, Tuple
2 +from typing import List, Optional, Sequence, Tuple, Union
3
4 import pytest
5
6 from vllm import LLM, SamplingParams
7 from vllm.model_executor.utils import set_random_seed
8 +from vllm.sequence import PromptLogprobs, SampleLogprobs
9
10 from ...conftest import cleanup
10 -from ...models.utils import check_logprobs_close, check_outputs_equal
11 +from ...models.utils import (TokensTextLogprobs,
12 + TokensTextLogprobsPromptLogprobs,
13 + check_logprobs_close, check_outputs_equal)
14 from ...utils import RemoteOpenAIServer
15
16 PROMPTS = [
@@ -81,45 +84,77 @@ def get_output_from_llm_generator(
84 return tokens, token_ids, acceptance_rate
85
86
84 -def run_logprob_correctness_test(vllm_runner,
85 - common_llm_kwargs,
86 - per_test_common_llm_kwargs,
87 - baseline_llm_kwargs,
88 - test_llm_kwargs,
89 - batch_size: int,
90 - max_output_len: int,
91 - seed: Optional[int] = 0,
92 - temperature: float = 0.0,
93 - logprobs: int = 1):
94 - org_args = {
95 - **common_llm_kwargs,
96 - **per_test_common_llm_kwargs,
97 - **baseline_llm_kwargs,
98 - }
99 -
100 - sd_args = {
101 - **common_llm_kwargs,
102 - **per_test_common_llm_kwargs,
103 - **test_llm_kwargs,
104 - }
105 -
106 - prompts = [prompt for prompt, _ in zip(cycle(PROMPTS), range(batch_size))]
107 -
108 - sampling_params = SamplingParams(temperature=temperature,
109 - max_tokens=max_output_len,
110 - seed=seed,
111 - logprobs=logprobs)
112 -
113 - with vllm_runner(**org_args) as vllm_model:
114 - org_outputs = vllm_model.generate_w_logprobs(prompts, sampling_params)
115 -
116 - with vllm_runner(**sd_args) as vllm_model:
117 - sd_outputs = vllm_model.generate_w_logprobs(prompts, sampling_params)
118 -
119 - check_logprobs_close(outputs_0_lst=org_outputs,
120 - outputs_1_lst=sd_outputs,
121 - name_0="org",
122 - name_1="sd")
87 +def check_logprobs_correctness(
88 + spec_outputs: Sequence[Union[TokensTextLogprobs,
89 + TokensTextLogprobsPromptLogprobs]],
90 + baseline_outputs: Sequence[Union[TokensTextLogprobs,
91 + TokensTextLogprobsPromptLogprobs]],
92 + disable_logprobs: bool = False,
93 +):
94 + """Compare sampled and prompt logprobs between baseline and spec decoding
95 + """
96 + if not disable_logprobs:
97 + return check_logprobs_close(
98 + outputs_0_lst=baseline_outputs,
99 + outputs_1_lst=spec_outputs,
100 + name_0="org",
101 + name_1="sd",
102 + )
103 +
104 + # Check correctness when disable_logprobs == True
105 + for spec_output, baseline_output in zip(spec_outputs, baseline_outputs):
106 + # Check generated token logprobs.
107 + spec_logprobs = spec_output[2]
108 + baseline_logprobs = baseline_output[2]
109 + _check_logprobs_when_output_disabled(spec_logprobs,
110 + baseline_logprobs,
111 + is_prompt_logprobs=False)
112 +
113 + # Check prompt logprobs too, if they exist
114 + if len(baseline_output) == 4:
115 + assert len(spec_output) == 4
116 + spec_prompt_logprobs = spec_output[3]
117 + baseline_prompt_logprobs = baseline_output[3]
118 + _check_logprobs_when_output_disabled(spec_prompt_logprobs,
119 + baseline_prompt_logprobs,
120 + is_prompt_logprobs=True)
121 +
122 +
123 +def _check_logprobs_when_output_disabled(
124 + spec_logprobs: Union[Optional[PromptLogprobs], SampleLogprobs],
125 + baseline_logprobs: Union[Optional[PromptLogprobs], SampleLogprobs],
126 + is_prompt_logprobs: bool = False,
127 +):
128 + # Prompt logprobs are optional
129 + if is_prompt_logprobs and baseline_logprobs is None:
130 + assert spec_logprobs is None
131 + return
132 +
133 + assert spec_logprobs is not None
134 + assert baseline_logprobs is not None
135 + assert len(spec_logprobs) == len(baseline_logprobs)
136 +
137 + # For each generated position of the sequence.
138 + for pos, (spec_pos_logprobs, baseline_pos_logprobs) in enumerate(
139 + zip(spec_logprobs, baseline_logprobs)):
140 +
141 + # First prompt logprob is expected to be None
142 + if is_prompt_logprobs and baseline_pos_logprobs is None:
143 + assert spec_pos_logprobs is None
144 + assert pos == 0
145 + continue
146 +
147 + assert spec_pos_logprobs is not None
148 + assert baseline_pos_logprobs is not None
149 +
150 + # When disabled, the 1 logprob is returned with dummy values for the
151 + # score and rank, but the token id should match the baseline model
152 + assert len(spec_pos_logprobs) == 1
153 + (spec_pos_logprob_token_id,
154 + spec_pos_logprob) = next(iter(spec_pos_logprobs.items()))
155 + assert spec_pos_logprob.rank == -1
156 + assert spec_pos_logprob.logprob == 0.0
157 + assert spec_pos_logprob_token_id in baseline_pos_logprobs
158
159
160 def run_equality_correctness_test(
@@ -135,7 +170,10 @@ def run_equality_correctness_test(
170 disable_seed: bool = False,
171 ignore_eos: bool = True,
172 ensure_all_accepted: bool = False,
138 - expected_acceptance_rate: Optional[float] = None):
173 + expected_acceptance_rate: Optional[float] = None,
174 + logprobs: Optional[int] = None,
175 + prompt_logprobs: Optional[int] = None,
176 + disable_logprobs: bool = False):
177
178 org_args = {
179 **common_llm_kwargs,
@@ -157,10 +195,12 @@ def run_equality_correctness_test(
195 sampling_params = SamplingParams(temperature=temperature,
196 max_tokens=max_output_len,
197 seed=seed,
160 - ignore_eos=ignore_eos)
198 + ignore_eos=ignore_eos,
199 + logprobs=logprobs,
200 + prompt_logprobs=prompt_logprobs)
201
202 with vllm_runner(**org_args) as vllm_model:
163 - org_outputs = vllm_model.generate(prompts, sampling_params)
203 + org_outputs = vllm_model.generate_w_logprobs(prompts, sampling_params)
204
205 with vllm_runner(**sd_args) as vllm_model:
206 if ensure_all_accepted or expected_acceptance_rate is not None:
@@ -169,7 +209,7 @@ def run_equality_correctness_test(
209 'prometheus']
210 stat_logger.local_interval = -100
211
172 - sd_outputs = vllm_model.generate(prompts, sampling_params)
212 + sd_outputs = vllm_model.generate_w_logprobs(prompts, sampling_params)
213
214 if ensure_all_accepted or expected_acceptance_rate is not None:
215 acceptance_rate = (stat_logger.metrics.
@@ -185,11 +225,16 @@ def run_equality_correctness_test(
225 if expected_acceptance_rate is not None:
226 assert acceptance_rate >= expected_acceptance_rate - 1e-2
227
188 - check_outputs_equal(outputs_0_lst=org_outputs,
189 - outputs_1_lst=sd_outputs,
228 + # Only pass token entries, not the logprobs
229 + check_outputs_equal(outputs_0_lst=[out[0:2] for out in org_outputs],
230 + outputs_1_lst=[out[0:2] for out in sd_outputs],
231 name_0="org",
232 name_1="sd")
233
234 + # Check logprobs if requested
235 + if logprobs is not None or prompt_logprobs is not None:
236 + check_logprobs_correctness(sd_outputs, org_outputs, disable_logprobs)
237 +
238
239 def run_equality_correctness_test_tp(model,
240 common_llm_kwargs,
tests/spec_decode/e2e/test_eagle_correctness.py
+58
@@ -80,6 +80,64 @@ def test_eagle_e2e_greedy_correctness(vllm_runner, common_llm_kwargs,
80 batch_size, output_len, seed)
81
82
83 +@pytest.mark.parametrize(
84 + "common_llm_kwargs",
85 + [{
86 + # Skip cuda graph recording for fast test.
87 + "enforce_eager": True,
88 +
89 + # Required for spec decode.
90 + "use_v2_block_manager": True,
91 +
92 + # Print spec metrics.
93 + "disable_log_stats": False,
94 +
95 + # Precision
96 + "dtype": PRECISION,
97 +
98 + # Main model
99 + "model_name": MAIN_MODEL,
100 + }])
101 +@pytest.mark.parametrize("per_test_common_llm_kwargs", [{}])
102 +@pytest.mark.parametrize("baseline_llm_kwargs", [{}])
103 +@pytest.mark.parametrize("test_llm_kwargs", [
104 + {
105 + "speculative_model": SPEC_MODEL,
106 + "num_speculative_tokens": MAX_SPEC_TOKENS,
107 + "disable_logprobs_during_spec_decoding": False,
108 + },
109 + {
110 + "speculative_model": SPEC_MODEL,
111 + "num_speculative_tokens": MAX_SPEC_TOKENS,
112 + "disable_logprobs_during_spec_decoding": True,
113 + },
114 +])
115 +@pytest.mark.parametrize("output_len", [
116 + 128,
117 +])
118 +@pytest.mark.parametrize("batch_size", [8])
119 +@pytest.mark.parametrize("seed", [1])
120 +@pytest.mark.parametrize("logprobs", [1, 6])
121 +def test_eagle_e2e_greedy_logprobs(vllm_runner, common_llm_kwargs,
122 + per_test_common_llm_kwargs,
123 + baseline_llm_kwargs, test_llm_kwargs,
124 + batch_size: int, output_len: int, seed: int,
125 + logprobs: int):
126 +
127 + run_equality_correctness_test(vllm_runner,
128 + common_llm_kwargs,
129 + per_test_common_llm_kwargs,
130 + baseline_llm_kwargs,
131 + test_llm_kwargs,
132 + batch_size,
133 + output_len,
134 + seed,
135 + logprobs=logprobs,
136 + prompt_logprobs=logprobs,
137 + disable_logprobs=test_llm_kwargs[
138 + 'disable_logprobs_during_spec_decoding'])
139 +
140 +
141 @pytest.mark.parametrize(
142 "common_llm_kwargs",
143 [{
tests/spec_decode/e2e/test_logprobs.py
+54 -41
@@ -4,7 +4,7 @@ import pytest
4
5 from vllm import SamplingParams
6
7 -from .conftest import run_logprob_correctness_test
7 +from .conftest import run_equality_correctness_test
8
9
10 @pytest.mark.parametrize(
@@ -25,6 +25,10 @@ from .conftest import run_logprob_correctness_test
25 "speculative_model": "JackFram/llama-160m",
26 "num_speculative_tokens": 3,
27 "disable_logprobs_during_spec_decoding": False,
28 + }, {
29 + "speculative_model": "JackFram/llama-160m",
30 + "num_speculative_tokens": 3,
31 + "disable_logprobs_during_spec_decoding": True,
32 }])
33 @pytest.mark.parametrize("batch_size", [8])
34 @pytest.mark.parametrize(
@@ -41,16 +45,19 @@ def test_logprobs_equality(vllm_runner, common_llm_kwargs,
45 seed: int, logprobs: int):
46 """Verify output logprobs are equal with and without speculative decoding.
47 """
44 - run_logprob_correctness_test(vllm_runner,
45 - common_llm_kwargs,
46 - per_test_common_llm_kwargs,
47 - baseline_llm_kwargs,
48 - test_llm_kwargs,
49 - batch_size,
50 - output_len,
51 - seed,
52 - temperature=0.0,
53 - logprobs=logprobs)
48 + run_equality_correctness_test(vllm_runner,
49 + common_llm_kwargs,
50 + per_test_common_llm_kwargs,
51 + baseline_llm_kwargs,
52 + test_llm_kwargs,
53 + batch_size,
54 + output_len,
55 + seed,
56 + temperature=0.0,
57 + logprobs=logprobs,
58 + prompt_logprobs=logprobs,
59 + disable_logprobs=test_llm_kwargs[
60 + 'disable_logprobs_during_spec_decoding'])
61
62
63 @pytest.mark.parametrize(
@@ -91,16 +98,18 @@ def test_logprobs_different_k(vllm_runner, common_llm_kwargs,
98 output_len: int, seed: int, logprobs: int):
99 """Veriy logprob greedy equality with different speculation lens.
100 """
94 - run_logprob_correctness_test(vllm_runner,
95 - common_llm_kwargs,
96 - per_test_common_llm_kwargs,
97 - baseline_llm_kwargs,
98 - test_llm_kwargs,
99 - batch_size,
100 - output_len,
101 - seed,
102 - temperature=0.0,
103 - logprobs=logprobs)
101 + run_equality_correctness_test(vllm_runner,
102 + common_llm_kwargs,
103 + per_test_common_llm_kwargs,
104 + baseline_llm_kwargs,
105 + test_llm_kwargs,
106 + batch_size,
107 + output_len,
108 + seed,
109 + temperature=0.0,
110 + logprobs=logprobs,
111 + disable_logprobs=test_llm_kwargs[
112 + 'disable_logprobs_during_spec_decoding'])
113
114
115 @pytest.mark.parametrize(
@@ -143,16 +152,18 @@ def test_logprobs_when_skip_speculation(vllm_runner, common_llm_kwargs,
152 seed: int, logprobs: int):
153 """Verify logprobs greedy equality when some sequences skip speculation.
154 """
146 - run_logprob_correctness_test(vllm_runner,
147 - common_llm_kwargs,
148 - per_test_common_llm_kwargs,
149 - baseline_llm_kwargs,
150 - test_llm_kwargs,
151 - batch_size,
152 - output_len,
153 - seed,
154 - temperature=0.0,
155 - logprobs=logprobs)
155 + run_equality_correctness_test(vllm_runner,
156 + common_llm_kwargs,
157 + per_test_common_llm_kwargs,
158 + baseline_llm_kwargs,
159 + test_llm_kwargs,
160 + batch_size,
161 + output_len,
162 + seed,
163 + temperature=0.0,
164 + logprobs=logprobs,
165 + disable_logprobs=test_llm_kwargs[
166 + 'disable_logprobs_during_spec_decoding'])
167
168
169 @pytest.mark.parametrize(
@@ -267,13 +278,15 @@ def test_logprobs_disabled(vllm_runner, common_llm_kwargs,
278 """Check the behavior when logprobs are disabled.
279 Token choices should match with the base model.
280 """
270 - run_logprob_correctness_test(vllm_runner,
271 - common_llm_kwargs,
272 - per_test_common_llm_kwargs,
273 - baseline_llm_kwargs,
274 - test_llm_kwargs,
275 - batch_size,
276 - output_len,
277 - seed,
278 - temperature=0.0,
279 - logprobs=logprobs)
281 + run_equality_correctness_test(vllm_runner,
282 + common_llm_kwargs,
283 + per_test_common_llm_kwargs,
284 + baseline_llm_kwargs,
285 + test_llm_kwargs,
286 + batch_size,
287 + output_len,
288 + seed,
289 + temperature=0.0,
290 + logprobs=logprobs,
291 + disable_logprobs=test_llm_kwargs[
292 + 'disable_logprobs_during_spec_decoding'])
tests/spec_decode/e2e/test_medusa_correctness.py
+59
@@ -87,6 +87,65 @@ def test_medusa_e2e_greedy_correctness(vllm_runner, common_llm_kwargs,
87 temperature=0.0)
88
89
90 +@pytest.mark.parametrize(
91 + "common_llm_kwargs",
92 + [{
93 + # Skip cuda graph recording for fast test.
94 + "enforce_eager": True,
95 +
96 + # Required for spec decode.
97 + "use_v2_block_manager": True,
98 +
99 + # Print spec metrics.
100 + "disable_log_stats": False,
101 +
102 + # Precision
103 + "dtype": PRECISION,
104 +
105 + # Main model
106 + "model_name": MAIN_MODEL,
107 + }])
108 +@pytest.mark.parametrize("per_test_common_llm_kwargs", [{}])
109 +@pytest.mark.parametrize("baseline_llm_kwargs", [{}])
110 +@pytest.mark.parametrize("test_llm_kwargs", [
111 + {
112 + "speculative_model": SPEC_MODEL,
113 + "num_speculative_tokens": MAX_SPEC_TOKENS,
114 + "disable_logprobs_during_spec_decoding": False,
115 + },
116 + {
117 + "speculative_model": SPEC_MODEL,
118 + "num_speculative_tokens": MAX_SPEC_TOKENS,
119 + "disable_logprobs_during_spec_decoding": True,
120 + },
121 +])
122 +@pytest.mark.parametrize("output_len", [
123 + 8,
124 +])
125 +@pytest.mark.parametrize("batch_size", [8])
126 +@pytest.mark.parametrize("seed", [1])
127 +@pytest.mark.parametrize("logprobs", [1, 6])
128 +def test_medusa_e2e_greedy_logprobs(vllm_runner, common_llm_kwargs,
129 + per_test_common_llm_kwargs,
130 + baseline_llm_kwargs, test_llm_kwargs,
131 + batch_size: int, output_len: int,
132 + seed: int, logprobs: int):
133 + """Verify greedy equality with different batch size."""
134 + run_equality_correctness_test(vllm_runner,
135 + common_llm_kwargs,
136 + per_test_common_llm_kwargs,
137 + baseline_llm_kwargs,
138 + test_llm_kwargs,
139 + batch_size,
140 + max_output_len=output_len,
141 + seed=seed,
142 + temperature=0.0,
143 + logprobs=logprobs,
144 + prompt_logprobs=logprobs,
145 + disable_logprobs=test_llm_kwargs[
146 + 'disable_logprobs_during_spec_decoding'])
147 +
148 +
149 @pytest.mark.parametrize(
150 "common_llm_kwargs",
151 [{
tests/spec_decode/e2e/test_mlp_correctness.py
+56 -1
@@ -16,7 +16,7 @@ However, we still need to verify below scenario could be passed:
16 * Test greedy equality under various number of speculative tokens.
17
18 With those tests, we can say at least, MLPSpeculator would not break the
19 -correctess for the target model outputs.
19 +correctness for the target model outputs.
20 """
21
22 from unittest.mock import patch
@@ -88,6 +88,61 @@ def test_mlp_e2e_greedy_correctness(vllm_runner, common_llm_kwargs,
88 temperature=0.0)
89
90
91 +@pytest.mark.parametrize(
92 + "common_llm_kwargs",
93 + [{
94 + # Skip cuda graph recording for fast test.
95 + "enforce_eager": True,
96 +
97 + # Required for spec decode.
98 + "use_v2_block_manager": True,
99 +
100 + # Print spec metrics.
101 + "disable_log_stats": False,
102 +
103 + # Precision
104 + "dtype": PRECISION,
105 +
106 + # Main model
107 + "model_name": MAIN_MODEL,
108 + }])
109 +@pytest.mark.parametrize("per_test_common_llm_kwargs", [{}])
110 +@pytest.mark.parametrize("baseline_llm_kwargs", [{}])
111 +@pytest.mark.parametrize("test_llm_kwargs", [
112 + {
113 + "speculative_model": SPEC_MODEL,
114 + "disable_logprobs_during_spec_decoding": False,
115 + },
116 + {
117 + "speculative_model": SPEC_MODEL,
118 + "disable_logprobs_during_spec_decoding": True,
119 + },
120 +])
121 +@pytest.mark.parametrize("output_len", [8])
122 +@pytest.mark.parametrize("batch_size", [8])
123 +@pytest.mark.parametrize("seed", [1])
124 +@pytest.mark.parametrize("logprobs", [1, 6])
125 +def test_mlp_e2e_greedy_logprobs(vllm_runner, common_llm_kwargs,
126 + per_test_common_llm_kwargs,
127 + baseline_llm_kwargs, test_llm_kwargs,
128 + batch_size: int, output_len: int, seed: int,
129 + logprobs: int):
130 + """Verify greedy equality with different batch size."""
131 + run_equality_correctness_test(vllm_runner,
132 + common_llm_kwargs,
133 + per_test_common_llm_kwargs,
134 + baseline_llm_kwargs,
135 + test_llm_kwargs,
136 + batch_size,
137 + max_output_len=output_len,
138 + seed=seed,
139 + temperature=0.0,
140 + logprobs=logprobs,
141 + prompt_logprobs=logprobs,
142 + disable_logprobs=test_llm_kwargs[
143 + 'disable_logprobs_during_spec_decoding'])
144 +
145 +
146 @pytest.mark.parametrize(
147 "common_llm_kwargs",
148 [{
tests/spec_decode/e2e/test_ngram_correctness.py
+59
@@ -76,6 +76,65 @@ def test_ngram_e2e_greedy_correctness(vllm_runner, common_llm_kwargs,
76 temperature=0.0)
77
78
79 +@pytest.mark.parametrize(
80 + "common_llm_kwargs",
81 + [{
82 + # Skip cuda graph recording for fast test.
83 + "enforce_eager": True,
84 +
85 + # Required for spec decode.
86 + "use_v2_block_manager": True,
87 +
88 + # Print spec metrics.
89 + "disable_log_stats": False,
90 + }])
91 +@pytest.mark.parametrize("per_test_common_llm_kwargs", [
92 + {
93 + "model_name": "JackFram/llama-68m",
94 + },
95 +])
96 +@pytest.mark.parametrize("baseline_llm_kwargs", [{}])
97 +@pytest.mark.parametrize("test_llm_kwargs", [
98 + {
99 + "speculative_model": "[ngram]",
100 + "num_speculative_tokens": 5,
101 + "ngram_prompt_lookup_max": 3,
102 + "disable_logprobs_during_spec_decoding": False,
103 + },
104 + {
105 + "speculative_model": "[ngram]",
106 + "num_speculative_tokens": 5,
107 + "ngram_prompt_lookup_max": 3,
108 + "disable_logprobs_during_spec_decoding": True,
109 + },
110 +])
111 +@pytest.mark.parametrize("output_len", [
112 + 8,
113 +])
114 +@pytest.mark.parametrize("batch_size", [8])
115 +@pytest.mark.parametrize("seed", [1])
116 +@pytest.mark.parametrize("logprobs", [1, 6])
117 +def test_ngram_e2e_greedy_logprobs(vllm_runner, common_llm_kwargs,
118 + per_test_common_llm_kwargs,
119 + baseline_llm_kwargs, test_llm_kwargs,
120 + batch_size: int, output_len: int, seed: int,
121 + logprobs: int):
122 + """Verify greedy equality on a tiny model with different batch size."""
123 + run_equality_correctness_test(vllm_runner,
124 + common_llm_kwargs,
125 + per_test_common_llm_kwargs,
126 + baseline_llm_kwargs,
127 + test_llm_kwargs,
128 + batch_size,
129 + max_output_len=output_len,
130 + seed=seed,
131 + temperature=0.0,
132 + logprobs=logprobs,
133 + prompt_logprobs=logprobs,
134 + disable_logprobs=test_llm_kwargs[
135 + 'disable_logprobs_during_spec_decoding'])
136 +
137 +
138 @pytest.mark.parametrize(
139 "common_llm_kwargs",
140 [{
vllm/engine/output_processor/multi_step.py
+5 -4
@@ -9,8 +9,8 @@ from vllm.engine.output_processor.single_step import (
9 from vllm.engine.output_processor.stop_checker import StopChecker
10 from vllm.logger import init_logger
11 from vllm.sampling_params import SamplingParams
12 -from vllm.sequence import (Sequence, SequenceGroup, SequenceGroupOutput,
13 - SequenceOutput, SequenceStatus)
12 +from vllm.sequence import (VLLM_INVALID_TOKEN_ID, Sequence, SequenceGroup,
13 + SequenceGroupOutput, SequenceOutput, SequenceStatus)
14 from vllm.transformers_utils.detokenizer import Detokenizer
15 from vllm.transformers_utils.tokenizer import AnyTokenizer
16 from vllm.utils import Counter
@@ -110,10 +110,11 @@ class MultiStepOutputProcessor(SequenceGroupOutputProcessor):
110 # we can take the first sample.
111 samples = [output.samples[0] for output in outputs]
112
113 - # -1 means the output token is not valid (eg. due to spec decode
113 + # entries in sample tokens may be invalid (eg. due to spec decode
114 # rejecting tokens).
115 valid_samples = [
116 - sample for sample in samples if sample.output_token != -1
116 + sample for sample in samples
117 + if sample.output_token != VLLM_INVALID_TOKEN_ID
118 ]
119 assert valid_samples
120
vllm/model_executor/layers/sampler.py
+6 -5
@@ -15,7 +15,8 @@ from vllm.model_executor.sampling_metadata import (SamplingMetadata,
15 SamplingTensors,
16 SequenceGroupToSample)
17 from vllm.sampling_params import SamplingType
18 -from vllm.sequence import (CompletionSequenceGroupOutput, Logprob,
18 +from vllm.sequence import (VLLM_INVALID_TOKEN_ID,
19 + CompletionSequenceGroupOutput, Logprob,
20 PromptLogprobs, SampleLogprobs, SequenceOutput)
21 from vllm.spec_decode.metrics import SpecDecodeWorkerMetrics
22
@@ -759,10 +760,10 @@ def _sample_with_torch(
760
761 # Create output tensor for sampled token ids.
762 if include_gpu_probs_tensor:
762 - sampled_token_ids_tensor = torch.empty(logprobs.shape[0],
763 - 1,
764 - dtype=torch.long,
765 - device=logprobs.device)
763 + sampled_token_ids_tensor = torch.full((logprobs.shape[0], 1),
764 + VLLM_INVALID_TOKEN_ID,
765 + dtype=torch.long,
766 + device=logprobs.device)
767 else:
768 sampled_token_ids_tensor = None
769
vllm/sequence.py
+2
@@ -26,6 +26,8 @@ if TYPE_CHECKING:
26
27 VLLM_TOKEN_ID_ARRAY_TYPE = "l"
28
29 +VLLM_INVALID_TOKEN_ID = -1
30 +
31
32 # We use dataclass for now because it is used for
33 # openai server output, and msgspec is not serializable.
vllm/spec_decode/batch_expansion.py
+5 -5
@@ -6,9 +6,9 @@ import torch
6
7 from vllm import SamplingParams
8 from vllm.model_executor.layers.sampler import SamplerOutput
9 -from vllm.sequence import (VLLM_TOKEN_ID_ARRAY_TYPE, ExecuteModelRequest,
10 - SequenceData, SequenceGroupMetadata,
11 - get_all_seq_ids)
9 +from vllm.sequence import (VLLM_INVALID_TOKEN_ID, VLLM_TOKEN_ID_ARRAY_TYPE,
10 + ExecuteModelRequest, SequenceData,
11 + SequenceGroupMetadata, get_all_seq_ids)
12 from vllm.spec_decode.interfaces import (SpeculativeProposals,
13 SpeculativeScorer, SpeculativeScores)
14 from vllm.spec_decode.util import nvtx_range, split_batch_by_proposal_len
@@ -69,10 +69,10 @@ class BatchExpansionTop1Scorer(SpeculativeScorer):
69 proposal_lens_list = proposals.proposal_lens.tolist()
70 proposal_token_ids_list = proposals.proposal_token_ids.tolist()
71
72 - # Filter the list to ignore -1 proposals.
72 + # Filter the list to ignore invalid proposals.
73 proposal_token_ids_list_without_skips = [
74 proposals for proposals in proposal_token_ids_list
75 - if -1 not in proposals
75 + if VLLM_INVALID_TOKEN_ID not in proposals
76 ]
77
78 (spec_indices, non_spec_indices, target_seq_group_metadata_list,
vllm/spec_decode/spec_decode_worker.py
+51 -11
@@ -13,9 +13,10 @@ from vllm.model_executor.layers.spec_decode_base_sampler import (
13 SpecDecodeBaseSampler, SpecDecodeStochasticBaseSampler)
14 from vllm.model_executor.layers.typical_acceptance_sampler import (
15 TypicalAcceptanceSampler)
16 -from vllm.sequence import (CompletionSequenceGroupOutput, ExecuteModelRequest,
16 +from vllm.sequence import (VLLM_INVALID_TOKEN_ID,
17 + CompletionSequenceGroupOutput, ExecuteModelRequest,
18 HiddenStates, SequenceGroupMetadata,
18 - get_all_seq_ids, get_all_seq_ids_and_request_ids)
19 + get_all_seq_ids_and_request_ids)
20 from vllm.spec_decode.batch_expansion import BatchExpansionTop1Scorer
21 from vllm.spec_decode.draft_model_runner import TP1DraftModelRunner
22 from vllm.spec_decode.interfaces import (SpeculativeProposals,
@@ -28,7 +29,8 @@ from vllm.spec_decode.ngram_worker import NGramWorker
29 from vllm.spec_decode.proposer_worker_base import ProposerWorkerBase
30 from vllm.spec_decode.smaller_tp_proposer_worker import SmallerTpProposerWorker
31 from vllm.spec_decode.target_model_runner import TargetModelRunner
31 -from vllm.spec_decode.util import (Timer, create_sequence_group_output,
32 +from vllm.spec_decode.util import (Timer, create_logprobs_output,
33 + create_sequence_group_output,
34 get_all_num_logprobs,
35 get_sampled_token_logprobs, nvtx_range,
36 split_batch_by_proposal_len)
@@ -436,8 +438,8 @@ class SpecDecodeWorker(LoraNotSupportedWorkerBase):
438 self, execute_model_req: ExecuteModelRequest,
439 sampler_output: SamplerOutput) -> SamplerOutput:
440 """
439 - Creates and returns a `SamplerOutput` with only the sampled token IDs
440 - being serialized to CPU & populated in `CompletionSequenceGroupOutput`.
441 + Creates and returns a `SamplerOutput` with only the token IDs being
442 + serialized to CPU and populated in `CompletionSequenceGroupOutput`.
443 All other parameters in `CompletionSequenceGroupOutput` related to log
444 probabilities are skipped.
445
@@ -449,14 +451,46 @@ class SpecDecodeWorker(LoraNotSupportedWorkerBase):
451
452 Returns:
453 SamplerOutput: A new `SamplerOutput` instance containing a list of
452 - `CompletionSequenceGroupOutput` objects with only sampled token
453 - IDs populated.
454 + `CompletionSequenceGroupOutput` objects with only token IDs
455 + populated.
456 """
455 - seq_ids = get_all_seq_ids(execute_model_req.seq_group_metadata_list)
456 - sampled_token_ids_list = sampler_output.sampled_token_ids.tolist()
457 + seq_output_prompt_logprobs = [
458 + seq.is_prompt and seq.sampling_params.prompt_logprobs is not None
459 + and seq.sampling_params.prompt_logprobs > 0
460 + for seq in execute_model_req.seq_group_metadata_list
461 + ]
462 + # ignore slots for prompt tokens that are filled with INVALID_TOKEN_ID
463 + sampled_token_ids_list = (sampler_output.sampled_token_ids[torch.where(
464 + # subtracting is faster than testing for equality
465 + sampler_output.sampled_token_ids - VLLM_INVALID_TOKEN_ID)[0]] \
466 + if any(seq_output_prompt_logprobs) else \
467 + sampler_output.sampled_token_ids).tolist()
468 +
469 + seq_data_entries = (
470 + (seq_id, seq_data) for sg in \
471 + execute_model_req.seq_group_metadata_list \
472 + for seq_id, seq_data in sg.seq_data.items()
473 + )
474 completion_seq_group_output_list: List[
475 CompletionSequenceGroupOutput] = []
459 - for index, seq_id in enumerate(seq_ids):
476 + for index, ((seq_id, seq_data), needs_prompt_logprobs) in \
477 + enumerate(zip(seq_data_entries, seq_output_prompt_logprobs)):
478 + if needs_prompt_logprobs:
479 + prompt_token_ids = seq_data.get_prompt_token_ids()
480 + prompt_logprobs = [
481 + create_logprobs_output(
482 + token_id=p_token_id,
483 + token_id_logprob_rank=-1,
484 + token_id_logprob=0.0,
485 + topk_token_ids=[],
486 + topk_logprobs=[],
487 + )
488 + # no prompt logprobs for the first token
489 + for p_token_id in prompt_token_ids[1:]
490 + ]
491 + else:
492 + prompt_logprobs = None
493 +
494 completion_seq_group_output_list.append(
495 create_sequence_group_output(
496 token_id=sampled_token_ids_list[index][0],
@@ -465,7 +499,7 @@ class SpecDecodeWorker(LoraNotSupportedWorkerBase):
499 seq_id=seq_id,
500 topk_token_ids=[],
501 topk_logprobs=[],
468 - ))
502 + prompt_logprobs=prompt_logprobs))
503 return SamplerOutput(outputs=completion_seq_group_output_list)
504
505 @nvtx_range("spec_decode_worker._run_no_spec")
@@ -485,6 +519,12 @@ class SpecDecodeWorker(LoraNotSupportedWorkerBase):
519 # Store hidden states from target model execution.
520 hidden_states = sampler_output.hidden_states
521 if hidden_states is not None:
522 + # remove hidden_states for prompt tokens
523 + if any(seq.is_prompt
524 + for seq in execute_model_req.seq_group_metadata_list):
525 + hidden_states = hidden_states[
526 + torch.where(sampler_output.sampled_token_ids -
527 + VLLM_INVALID_TOKEN_ID)[0]]
528 if self.previous_hidden_states is None:
529 self.previous_hidden_states = HiddenStates(
530 hidden_states, execute_model_req.seq_group_metadata_list)
vllm/spec_decode/util.py
+37 -8
@@ -6,7 +6,8 @@ import torch
6
7 from vllm.model_executor.layers.sampler import SamplerOutput
8 from vllm.sequence import (CompletionSequenceGroupOutput, Logprob,
9 - SequenceGroupMetadata, SequenceOutput)
9 + PromptLogprobs, SequenceGroupMetadata,
10 + SequenceOutput)
11
12 SeqId = int
13
@@ -49,21 +50,19 @@ def get_sampled_token_logprobs(
50 return sampled_token_ids_ranks, selected_logprobs
51
52
52 -def create_sequence_group_output(
53 +def create_logprobs_output(
54 token_id: int,
55 token_id_logprob_rank: int,
56 token_id_logprob: float,
56 - seq_id: SeqId,
57 topk_token_ids: List[Optional[int]],
58 topk_logprobs: List[Optional[float]],
59 -) -> CompletionSequenceGroupOutput:
60 - """Create a SequenceGroupOutput given the sampling results.
59 +) -> Dict[int, Logprob]:
60 + """Create a Logprob Dict for a token given the sampling results.
61
62 Args:
63 token_id (int): The sampled token for the sequence.
64 token_id_logprob_rank (int): The logprob rank of the sampled token.
65 token_id_logprob (float): The logprob value of the sampled token.
66 - seq_id (int): The sequence id.
66 topk_token_ids (List[Optional[int]]): The list of top-k token ids.
67 topk_logprobs (List[Optional[float]]): The list of top-k logprobs.
68 """
@@ -85,14 +84,44 @@ def create_sequence_group_output(
84 if topk_token_id is not None
85 })
86
87 + return logprobs
88 +
89 +
90 +def create_sequence_group_output(
91 + token_id: int,
92 + token_id_logprob_rank: int,
93 + token_id_logprob: float,
94 + seq_id: SeqId,
95 + topk_token_ids: List[Optional[int]],
96 + topk_logprobs: List[Optional[float]],
97 + prompt_logprobs: Optional[PromptLogprobs] = None,
98 +) -> CompletionSequenceGroupOutput:
99 + """Create a SequenceGroupOutput given the sampling results.
100 +
101 + Args:
102 + token_id (int): The sampled token for the sequence.
103 + token_id_logprob_rank (int): The logprob rank of the sampled token.
104 + token_id_logprob (float): The logprob value of the sampled token.
105 + seq_id (int): The sequence id.
106 + topk_token_ids (List[Optional[int]]): The list of top-k token ids.
107 + topk_logprobs (List[Optional[float]]): The list of top-k logprobs.
108 + """
109 +
110 + logprobs = create_logprobs_output(
111 + token_id,
112 + token_id_logprob_rank,
113 + token_id_logprob,
114 + topk_token_ids,
115 + topk_logprobs,
116 + )
117 +
118 return CompletionSequenceGroupOutput(
119 samples=[
120 SequenceOutput(parent_seq_id=seq_id,
121 output_token=token_id,
122 logprobs=logprobs)
123 ],
94 - # TODO add prompt logprobs support.
95 - prompt_logprobs=None,
124 + prompt_logprobs=prompt_logprobs,
125 )
126
127
vllm/transformers_utils/detokenizer.py
+7 -9
@@ -1,13 +1,11 @@
1 from typing import Dict, List, Optional, Tuple
2
3 -from vllm.sequence import Logprob, SamplingParams, Sequence, SequenceGroup
3 +from vllm.sequence import (VLLM_INVALID_TOKEN_ID, Logprob, SamplingParams,
4 + Sequence, SequenceGroup)
5
6 from .tokenizer import AnyTokenizer
7 from .tokenizer_group import BaseTokenizerGroup
8
8 -# Used eg. for marking rejected tokens in spec decoding.
9 -INVALID_TOKEN_ID = -1
10 -
9
10 class Detokenizer:
11 """Provides methods to decode the output of a model into text."""
@@ -61,7 +59,7 @@ class Detokenizer:
59 continue
60 for token_id, sample_logprob in prompt_logprobs_for_token.items():
61 if (sample_logprob.decoded_token is None
64 - and token_id != INVALID_TOKEN_ID):
62 + and token_id != VLLM_INVALID_TOKEN_ID):
63 prompt_token_ids_with_token = (
64 prompt_token_ids[:token_position] + [token_id])
65 (new_tokens, new_text, new_prefix_offset,
@@ -143,7 +141,7 @@ class Detokenizer:
141 continue
142
143 if (sample_logprob.decoded_token is None
146 - and token_id != INVALID_TOKEN_ID):
144 + and token_id != VLLM_INVALID_TOKEN_ID):
145 all_input_ids_with_logprob = previous_tokens + [token_id]
146 (_, new_text, _, _) = detokenize_incrementally(
147 tokenizer=tokenizer,
@@ -282,14 +280,14 @@ def detokenize_incrementally(
280 assert prev_tokens is not None
281
282 # If the new token id is out of bounds, return an empty string.
285 - if new_token_id >= len(tokenizer):
286 - new_tokens = [""]
287 - else:
283 + if 0 <= new_token_id < len(tokenizer):
284 # Put new_token_id in a list so skip_special_tokens is respected
285 new_tokens = tokenizer.convert_ids_to_tokens(
286 [new_token_id], skip_special_tokens=skip_special_tokens)
287 if isinstance(new_tokens, str):
288 new_tokens = [new_tokens]
289 + else:
290 + new_tokens = [""]
291 output_tokens = prev_tokens + new_tokens
292
293 # If this is the first iteration, return all tokens.