A high-throughput and memory-efficient inference and serving engine for LLMs
Mirrored from https://github.com/tduncan-lum/Nvidia-Optimization-Stack---VLLM.git
Synced Nov 16, 15:45 UTC
[Core][Bugfix] Support prompt_logprobs returned with speculative decoding (#8047)
Signed-off-by: Travis Johnson <tsjohnso@us.ibm.com>
Travis Johnson committed
Sep 24, 2024 at 18:29 UTC
01b6f9e1f0530a7cb81486ff34d3d935e4f75d28
14 files changed
+492
-134
tests/conftest.py
+1
-3
@@ -675,8 +675,6 @@ class VllmRunner:
675
videos: Optional[PromptVideoInput] = None,
676
) -> Union[List[TokensTextLogprobs],
677
List[TokensTextLogprobsPromptLogprobs]]:
678
- assert sampling_params.logprobs is not None
679
-
678
if images is not None:
679
assert len(prompts) == len(images)
680
@@ -754,7 +752,7 @@ class VllmRunner:
752
temperature=0.0,
753
max_tokens=max_tokens,
754
logprobs=num_logprobs,
757
- prompt_logprobs=(num_prompt_logprobs),
755
+ prompt_logprobs=num_prompt_logprobs,
756
stop_token_ids=stop_token_ids)
757
758
return self.generate_w_logprobs(prompts,
tests/spec_decode/e2e/conftest.py
+92
-47
@@ -1,13 +1,16 @@
1
from itertools import cycle
2
-from typing import List, Optional, Tuple
2
+from typing import List, Optional, Sequence, Tuple, Union
3
4
import pytest
5
6
from vllm import LLM, SamplingParams
7
from vllm.model_executor.utils import set_random_seed
8
+from vllm.sequence import PromptLogprobs, SampleLogprobs
9
10
from ...conftest import cleanup
10
-from ...models.utils import check_logprobs_close, check_outputs_equal
11
+from ...models.utils import (TokensTextLogprobs,
12
+ TokensTextLogprobsPromptLogprobs,
13
+ check_logprobs_close, check_outputs_equal)
14
from ...utils import RemoteOpenAIServer
15
16
PROMPTS = [
@@ -81,45 +84,77 @@ def get_output_from_llm_generator(
84
return tokens, token_ids, acceptance_rate
85
86
84
-def run_logprob_correctness_test(vllm_runner,
85
- common_llm_kwargs,
86
- per_test_common_llm_kwargs,
87
- baseline_llm_kwargs,
88
- test_llm_kwargs,
89
- batch_size: int,
90
- max_output_len: int,
91
- seed: Optional[int] = 0,
92
- temperature: float = 0.0,
93
- logprobs: int = 1):
94
- org_args = {
95
- **common_llm_kwargs,
96
- **per_test_common_llm_kwargs,
97
- **baseline_llm_kwargs,
98
- }
99
-
100
- sd_args = {
101
- **common_llm_kwargs,
102
- **per_test_common_llm_kwargs,
103
- **test_llm_kwargs,
104
- }
105
-
106
- prompts = [prompt for prompt, _ in zip(cycle(PROMPTS), range(batch_size))]
107
-
108
- sampling_params = SamplingParams(temperature=temperature,
109
- max_tokens=max_output_len,
110
- seed=seed,
111
- logprobs=logprobs)
112
-
113
- with vllm_runner(**org_args) as vllm_model:
114
- org_outputs = vllm_model.generate_w_logprobs(prompts, sampling_params)
115
-
116
- with vllm_runner(**sd_args) as vllm_model:
117
- sd_outputs = vllm_model.generate_w_logprobs(prompts, sampling_params)
118
-
119
- check_logprobs_close(outputs_0_lst=org_outputs,
120
- outputs_1_lst=sd_outputs,
121
- name_0="org",
122
- name_1="sd")
87
+def check_logprobs_correctness(
88
+ spec_outputs: Sequence[Union[TokensTextLogprobs,
89
+ TokensTextLogprobsPromptLogprobs]],
90
+ baseline_outputs: Sequence[Union[TokensTextLogprobs,
91
+ TokensTextLogprobsPromptLogprobs]],
92
+ disable_logprobs: bool = False,
93
+):
94
+ """Compare sampled and prompt logprobs between baseline and spec decoding
95
+ """
96
+ if not disable_logprobs:
97
+ return check_logprobs_close(
98
+ outputs_0_lst=baseline_outputs,
99
+ outputs_1_lst=spec_outputs,
100
+ name_0="org",
101
+ name_1="sd",
102
+ )
103
+
104
+ # Check correctness when disable_logprobs == True
105
+ for spec_output, baseline_output in zip(spec_outputs, baseline_outputs):
106
+ # Check generated token logprobs.
107
+ spec_logprobs = spec_output[2]
108
+ baseline_logprobs = baseline_output[2]
109
+ _check_logprobs_when_output_disabled(spec_logprobs,
110
+ baseline_logprobs,
111
+ is_prompt_logprobs=False)
112
+
113
+ # Check prompt logprobs too, if they exist
114
+ if len(baseline_output) == 4:
115
+ assert len(spec_output) == 4
116
+ spec_prompt_logprobs = spec_output[3]
117
+ baseline_prompt_logprobs = baseline_output[3]
118
+ _check_logprobs_when_output_disabled(spec_prompt_logprobs,
119
+ baseline_prompt_logprobs,
120
+ is_prompt_logprobs=True)
121
+
122
+
123
+def _check_logprobs_when_output_disabled(
124
+ spec_logprobs: Union[Optional[PromptLogprobs], SampleLogprobs],
125
+ baseline_logprobs: Union[Optional[PromptLogprobs], SampleLogprobs],
126
+ is_prompt_logprobs: bool = False,
127
+):
128
+ # Prompt logprobs are optional
129
+ if is_prompt_logprobs and baseline_logprobs is None:
130
+ assert spec_logprobs is None
131
+ return
132
+
133
+ assert spec_logprobs is not None
134
+ assert baseline_logprobs is not None
135
+ assert len(spec_logprobs) == len(baseline_logprobs)
136
+
137
+ # For each generated position of the sequence.
138
+ for pos, (spec_pos_logprobs, baseline_pos_logprobs) in enumerate(
139
+ zip(spec_logprobs, baseline_logprobs)):
140
+
141
+ # First prompt logprob is expected to be None
142
+ if is_prompt_logprobs and baseline_pos_logprobs is None:
143
+ assert spec_pos_logprobs is None
144
+ assert pos == 0
145
+ continue
146
+
147
+ assert spec_pos_logprobs is not None
148
+ assert baseline_pos_logprobs is not None
149
+
150
+ # When disabled, the 1 logprob is returned with dummy values for the
151
+ # score and rank, but the token id should match the baseline model
152
+ assert len(spec_pos_logprobs) == 1
153
+ (spec_pos_logprob_token_id,
154
+ spec_pos_logprob) = next(iter(spec_pos_logprobs.items()))
155
+ assert spec_pos_logprob.rank == -1
156
+ assert spec_pos_logprob.logprob == 0.0
157
+ assert spec_pos_logprob_token_id in baseline_pos_logprobs
158
159
160
def run_equality_correctness_test(
@@ -135,7 +170,10 @@ def run_equality_correctness_test(
170
disable_seed: bool = False,
171
ignore_eos: bool = True,
172
ensure_all_accepted: bool = False,
138
- expected_acceptance_rate: Optional[float] = None):
173
+ expected_acceptance_rate: Optional[float] = None,
174
+ logprobs: Optional[int] = None,
175
+ prompt_logprobs: Optional[int] = None,
176
+ disable_logprobs: bool = False):
177
178
org_args = {
179
**common_llm_kwargs,
@@ -157,10 +195,12 @@ def run_equality_correctness_test(
195
sampling_params = SamplingParams(temperature=temperature,
196
max_tokens=max_output_len,
197
seed=seed,
160
- ignore_eos=ignore_eos)
198
+ ignore_eos=ignore_eos,
199
+ logprobs=logprobs,
200
+ prompt_logprobs=prompt_logprobs)
201
202
with vllm_runner(**org_args) as vllm_model:
163
- org_outputs = vllm_model.generate(prompts, sampling_params)
203
+ org_outputs = vllm_model.generate_w_logprobs(prompts, sampling_params)
204
205
with vllm_runner(**sd_args) as vllm_model:
206
if ensure_all_accepted or expected_acceptance_rate is not None:
@@ -169,7 +209,7 @@ def run_equality_correctness_test(
209
'prometheus']
210
stat_logger.local_interval = -100
211
172
- sd_outputs = vllm_model.generate(prompts, sampling_params)
212
+ sd_outputs = vllm_model.generate_w_logprobs(prompts, sampling_params)
213
214
if ensure_all_accepted or expected_acceptance_rate is not None:
215
acceptance_rate = (stat_logger.metrics.
@@ -185,11 +225,16 @@ def run_equality_correctness_test(
225
if expected_acceptance_rate is not None:
226
assert acceptance_rate >= expected_acceptance_rate - 1e-2
227
188
- check_outputs_equal(outputs_0_lst=org_outputs,
189
- outputs_1_lst=sd_outputs,
228
+ # Only pass token entries, not the logprobs
229
+ check_outputs_equal(outputs_0_lst=[out[0:2] for out in org_outputs],
230
+ outputs_1_lst=[out[0:2] for out in sd_outputs],
231
name_0="org",
232
name_1="sd")
233
234
+ # Check logprobs if requested
235
+ if logprobs is not None or prompt_logprobs is not None:
236
+ check_logprobs_correctness(sd_outputs, org_outputs, disable_logprobs)
237
+
238
239
def run_equality_correctness_test_tp(model,
240
common_llm_kwargs,
tests/spec_decode/e2e/test_eagle_correctness.py
+58
@@ -80,6 +80,64 @@ def test_eagle_e2e_greedy_correctness(vllm_runner, common_llm_kwargs,
80
batch_size, output_len, seed)
81
82
83
+@pytest.mark.parametrize(
84
+ "common_llm_kwargs",
85
+ [{
86
+ # Skip cuda graph recording for fast test.
87
+ "enforce_eager": True,
88
+
89
+ # Required for spec decode.
90
+ "use_v2_block_manager": True,
91
+
92
+ # Print spec metrics.
93
+ "disable_log_stats": False,
94
+
95
+ # Precision
96
+ "dtype": PRECISION,
97
+
98
+ # Main model
99
+ "model_name": MAIN_MODEL,
100
+ }])
101
+@pytest.mark.parametrize("per_test_common_llm_kwargs", [{}])
102
+@pytest.mark.parametrize("baseline_llm_kwargs", [{}])
103
+@pytest.mark.parametrize("test_llm_kwargs", [
104
+ {
105
+ "speculative_model": SPEC_MODEL,
106
+ "num_speculative_tokens": MAX_SPEC_TOKENS,
107
+ "disable_logprobs_during_spec_decoding": False,
108
+ },
109
+ {
110
+ "speculative_model": SPEC_MODEL,
111
+ "num_speculative_tokens": MAX_SPEC_TOKENS,
112
+ "disable_logprobs_during_spec_decoding": True,
113
+ },
114
+])
115
+@pytest.mark.parametrize("output_len", [
116
+ 128,
117
+])
118
+@pytest.mark.parametrize("batch_size", [8])
119
+@pytest.mark.parametrize("seed", [1])
120
+@pytest.mark.parametrize("logprobs", [1, 6])
121
+def test_eagle_e2e_greedy_logprobs(vllm_runner, common_llm_kwargs,
122
+ per_test_common_llm_kwargs,
123
+ baseline_llm_kwargs, test_llm_kwargs,
124
+ batch_size: int, output_len: int, seed: int,
125
+ logprobs: int):
126
+
127
+ run_equality_correctness_test(vllm_runner,
128
+ common_llm_kwargs,
129
+ per_test_common_llm_kwargs,
130
+ baseline_llm_kwargs,
131
+ test_llm_kwargs,
132
+ batch_size,
133
+ output_len,
134
+ seed,
135
+ logprobs=logprobs,
136
+ prompt_logprobs=logprobs,
137
+ disable_logprobs=test_llm_kwargs[
138
+ 'disable_logprobs_during_spec_decoding'])
139
+
140
+
141
@pytest.mark.parametrize(
142
"common_llm_kwargs",
143
[{
tests/spec_decode/e2e/test_logprobs.py
+54
-41
@@ -4,7 +4,7 @@ import pytest
4
5
from vllm import SamplingParams
6
7
-from .conftest import run_logprob_correctness_test
7
+from .conftest import run_equality_correctness_test
8
9
10
@pytest.mark.parametrize(
@@ -25,6 +25,10 @@ from .conftest import run_logprob_correctness_test
25
"speculative_model": "JackFram/llama-160m",
26
"num_speculative_tokens": 3,
27
"disable_logprobs_during_spec_decoding": False,
28
+ }, {
29
+ "speculative_model": "JackFram/llama-160m",
30
+ "num_speculative_tokens": 3,
31
+ "disable_logprobs_during_spec_decoding": True,
32
}])
33
@pytest.mark.parametrize("batch_size", [8])
34
@pytest.mark.parametrize(
@@ -41,16 +45,19 @@ def test_logprobs_equality(vllm_runner, common_llm_kwargs,
45
seed: int, logprobs: int):
46
"""Verify output logprobs are equal with and without speculative decoding.
47
"""
44
- run_logprob_correctness_test(vllm_runner,
45
- common_llm_kwargs,
46
- per_test_common_llm_kwargs,
47
- baseline_llm_kwargs,
48
- test_llm_kwargs,
49
- batch_size,
50
- output_len,
51
- seed,
52
- temperature=0.0,
53
- logprobs=logprobs)
48
+ run_equality_correctness_test(vllm_runner,
49
+ common_llm_kwargs,
50
+ per_test_common_llm_kwargs,
51
+ baseline_llm_kwargs,
52
+ test_llm_kwargs,
53
+ batch_size,
54
+ output_len,
55
+ seed,
56
+ temperature=0.0,
57
+ logprobs=logprobs,
58
+ prompt_logprobs=logprobs,
59
+ disable_logprobs=test_llm_kwargs[
60
+ 'disable_logprobs_during_spec_decoding'])
61
62
63
@pytest.mark.parametrize(
@@ -91,16 +98,18 @@ def test_logprobs_different_k(vllm_runner, common_llm_kwargs,
98
output_len: int, seed: int, logprobs: int):
99
"""Veriy logprob greedy equality with different speculation lens.
100
"""
94
- run_logprob_correctness_test(vllm_runner,
95
- common_llm_kwargs,
96
- per_test_common_llm_kwargs,
97
- baseline_llm_kwargs,
98
- test_llm_kwargs,
99
- batch_size,
100
- output_len,
101
- seed,
102
- temperature=0.0,
103
- logprobs=logprobs)
101
+ run_equality_correctness_test(vllm_runner,
102
+ common_llm_kwargs,
103
+ per_test_common_llm_kwargs,
104
+ baseline_llm_kwargs,
105
+ test_llm_kwargs,
106
+ batch_size,
107
+ output_len,
108
+ seed,
109
+ temperature=0.0,
110
+ logprobs=logprobs,
111
+ disable_logprobs=test_llm_kwargs[
112
+ 'disable_logprobs_during_spec_decoding'])
113
114
115
@pytest.mark.parametrize(
@@ -143,16 +152,18 @@ def test_logprobs_when_skip_speculation(vllm_runner, common_llm_kwargs,
152
seed: int, logprobs: int):
153
"""Verify logprobs greedy equality when some sequences skip speculation.
154
"""
146
- run_logprob_correctness_test(vllm_runner,
147
- common_llm_kwargs,
148
- per_test_common_llm_kwargs,
149
- baseline_llm_kwargs,
150
- test_llm_kwargs,
151
- batch_size,
152
- output_len,
153
- seed,
154
- temperature=0.0,
155
- logprobs=logprobs)
155
+ run_equality_correctness_test(vllm_runner,
156
+ common_llm_kwargs,
157
+ per_test_common_llm_kwargs,
158
+ baseline_llm_kwargs,
159
+ test_llm_kwargs,
160
+ batch_size,
161
+ output_len,
162
+ seed,
163
+ temperature=0.0,
164
+ logprobs=logprobs,
165
+ disable_logprobs=test_llm_kwargs[
166
+ 'disable_logprobs_during_spec_decoding'])
167
168
169
@pytest.mark.parametrize(
@@ -267,13 +278,15 @@ def test_logprobs_disabled(vllm_runner, common_llm_kwargs,
278
"""Check the behavior when logprobs are disabled.
279
Token choices should match with the base model.
280
"""
270
- run_logprob_correctness_test(vllm_runner,
271
- common_llm_kwargs,
272
- per_test_common_llm_kwargs,
273
- baseline_llm_kwargs,
274
- test_llm_kwargs,
275
- batch_size,
276
- output_len,
277
- seed,
278
- temperature=0.0,
279
- logprobs=logprobs)
281
+ run_equality_correctness_test(vllm_runner,
282
+ common_llm_kwargs,
283
+ per_test_common_llm_kwargs,
284
+ baseline_llm_kwargs,
285
+ test_llm_kwargs,
286
+ batch_size,
287
+ output_len,
288
+ seed,
289
+ temperature=0.0,
290
+ logprobs=logprobs,
291
+ disable_logprobs=test_llm_kwargs[
292
+ 'disable_logprobs_during_spec_decoding'])
tests/spec_decode/e2e/test_medusa_correctness.py
+59
@@ -87,6 +87,65 @@ def test_medusa_e2e_greedy_correctness(vllm_runner, common_llm_kwargs,
87
temperature=0.0)
88
89
90
+@pytest.mark.parametrize(
91
+ "common_llm_kwargs",
92
+ [{
93
+ # Skip cuda graph recording for fast test.
94
+ "enforce_eager": True,
95
+
96
+ # Required for spec decode.
97
+ "use_v2_block_manager": True,
98
+
99
+ # Print spec metrics.
100
+ "disable_log_stats": False,
101
+
102
+ # Precision
103
+ "dtype": PRECISION,
104
+
105
+ # Main model
106
+ "model_name": MAIN_MODEL,
107
+ }])
108
+@pytest.mark.parametrize("per_test_common_llm_kwargs", [{}])
109
+@pytest.mark.parametrize("baseline_llm_kwargs", [{}])
110
+@pytest.mark.parametrize("test_llm_kwargs", [
111
+ {
112
+ "speculative_model": SPEC_MODEL,
113
+ "num_speculative_tokens": MAX_SPEC_TOKENS,
114
+ "disable_logprobs_during_spec_decoding": False,
115
+ },
116
+ {
117
+ "speculative_model": SPEC_MODEL,
118
+ "num_speculative_tokens": MAX_SPEC_TOKENS,
119
+ "disable_logprobs_during_spec_decoding": True,
120
+ },
121
+])
122
+@pytest.mark.parametrize("output_len", [
123
+ 8,
124
+])
125
+@pytest.mark.parametrize("batch_size", [8])
126
+@pytest.mark.parametrize("seed", [1])
127
+@pytest.mark.parametrize("logprobs", [1, 6])
128
+def test_medusa_e2e_greedy_logprobs(vllm_runner, common_llm_kwargs,
129
+ per_test_common_llm_kwargs,
130
+ baseline_llm_kwargs, test_llm_kwargs,
131
+ batch_size: int, output_len: int,
132
+ seed: int, logprobs: int):
133
+ """Verify greedy equality with different batch size."""
134
+ run_equality_correctness_test(vllm_runner,
135
+ common_llm_kwargs,
136
+ per_test_common_llm_kwargs,
137
+ baseline_llm_kwargs,
138
+ test_llm_kwargs,
139
+ batch_size,
140
+ max_output_len=output_len,
141
+ seed=seed,
142
+ temperature=0.0,
143
+ logprobs=logprobs,
144
+ prompt_logprobs=logprobs,
145
+ disable_logprobs=test_llm_kwargs[
146
+ 'disable_logprobs_during_spec_decoding'])
147
+
148
+
149
@pytest.mark.parametrize(
150
"common_llm_kwargs",
151
[{
tests/spec_decode/e2e/test_mlp_correctness.py
+56
-1
@@ -16,7 +16,7 @@ However, we still need to verify below scenario could be passed:
16
* Test greedy equality under various number of speculative tokens.
17
18
With those tests, we can say at least, MLPSpeculator would not break the
19
-correctess for the target model outputs.
19
+correctness for the target model outputs.
20
"""
21
22
from unittest.mock import patch
@@ -88,6 +88,61 @@ def test_mlp_e2e_greedy_correctness(vllm_runner, common_llm_kwargs,
88
temperature=0.0)
89
90
91
+@pytest.mark.parametrize(
92
+ "common_llm_kwargs",
93
+ [{
94
+ # Skip cuda graph recording for fast test.
95
+ "enforce_eager": True,
96
+
97
+ # Required for spec decode.
98
+ "use_v2_block_manager": True,
99
+
100
+ # Print spec metrics.
101
+ "disable_log_stats": False,
102
+
103
+ # Precision
104
+ "dtype": PRECISION,
105
+
106
+ # Main model
107
+ "model_name": MAIN_MODEL,
108
+ }])
109
+@pytest.mark.parametrize("per_test_common_llm_kwargs", [{}])
110
+@pytest.mark.parametrize("baseline_llm_kwargs", [{}])
111
+@pytest.mark.parametrize("test_llm_kwargs", [
112
+ {
113
+ "speculative_model": SPEC_MODEL,
114
+ "disable_logprobs_during_spec_decoding": False,
115
+ },
116
+ {
117
+ "speculative_model": SPEC_MODEL,
118
+ "disable_logprobs_during_spec_decoding": True,
119
+ },
120
+])
121
+@pytest.mark.parametrize("output_len", [8])
122
+@pytest.mark.parametrize("batch_size", [8])
123
+@pytest.mark.parametrize("seed", [1])
124
+@pytest.mark.parametrize("logprobs", [1, 6])
125
+def test_mlp_e2e_greedy_logprobs(vllm_runner, common_llm_kwargs,
126
+ per_test_common_llm_kwargs,
127
+ baseline_llm_kwargs, test_llm_kwargs,
128
+ batch_size: int, output_len: int, seed: int,
129
+ logprobs: int):
130
+ """Verify greedy equality with different batch size."""
131
+ run_equality_correctness_test(vllm_runner,
132
+ common_llm_kwargs,
133
+ per_test_common_llm_kwargs,
134
+ baseline_llm_kwargs,
135
+ test_llm_kwargs,
136
+ batch_size,
137
+ max_output_len=output_len,
138
+ seed=seed,
139
+ temperature=0.0,
140
+ logprobs=logprobs,
141
+ prompt_logprobs=logprobs,
142
+ disable_logprobs=test_llm_kwargs[
143
+ 'disable_logprobs_during_spec_decoding'])
144
+
145
+
146
@pytest.mark.parametrize(
147
"common_llm_kwargs",
148
[{
tests/spec_decode/e2e/test_ngram_correctness.py
+59
@@ -76,6 +76,65 @@ def test_ngram_e2e_greedy_correctness(vllm_runner, common_llm_kwargs,
76
temperature=0.0)
77
78
79
+@pytest.mark.parametrize(
80
+ "common_llm_kwargs",
81
+ [{
82
+ # Skip cuda graph recording for fast test.
83
+ "enforce_eager": True,
84
+
85
+ # Required for spec decode.
86
+ "use_v2_block_manager": True,
87
+
88
+ # Print spec metrics.
89
+ "disable_log_stats": False,
90
+ }])
91
+@pytest.mark.parametrize("per_test_common_llm_kwargs", [
92
+ {
93
+ "model_name": "JackFram/llama-68m",
94
+ },
95
+])
96
+@pytest.mark.parametrize("baseline_llm_kwargs", [{}])
97
+@pytest.mark.parametrize("test_llm_kwargs", [
98
+ {
99
+ "speculative_model": "[ngram]",
100
+ "num_speculative_tokens": 5,
101
+ "ngram_prompt_lookup_max": 3,
102
+ "disable_logprobs_during_spec_decoding": False,
103
+ },
104
+ {
105
+ "speculative_model": "[ngram]",
106
+ "num_speculative_tokens": 5,
107
+ "ngram_prompt_lookup_max": 3,
108
+ "disable_logprobs_during_spec_decoding": True,
109
+ },
110
+])
111
+@pytest.mark.parametrize("output_len", [
112
+ 8,
113
+])
114
+@pytest.mark.parametrize("batch_size", [8])
115
+@pytest.mark.parametrize("seed", [1])
116
+@pytest.mark.parametrize("logprobs", [1, 6])
117
+def test_ngram_e2e_greedy_logprobs(vllm_runner, common_llm_kwargs,
118
+ per_test_common_llm_kwargs,
119
+ baseline_llm_kwargs, test_llm_kwargs,
120
+ batch_size: int, output_len: int, seed: int,
121
+ logprobs: int):
122
+ """Verify greedy equality on a tiny model with different batch size."""
123
+ run_equality_correctness_test(vllm_runner,
124
+ common_llm_kwargs,
125
+ per_test_common_llm_kwargs,
126
+ baseline_llm_kwargs,
127
+ test_llm_kwargs,
128
+ batch_size,
129
+ max_output_len=output_len,
130
+ seed=seed,
131
+ temperature=0.0,
132
+ logprobs=logprobs,
133
+ prompt_logprobs=logprobs,
134
+ disable_logprobs=test_llm_kwargs[
135
+ 'disable_logprobs_during_spec_decoding'])
136
+
137
+
138
@pytest.mark.parametrize(
139
"common_llm_kwargs",
140
[{
vllm/engine/output_processor/multi_step.py
+5
-4
@@ -9,8 +9,8 @@ from vllm.engine.output_processor.single_step import (
9
from vllm.engine.output_processor.stop_checker import StopChecker
10
from vllm.logger import init_logger
11
from vllm.sampling_params import SamplingParams
12
-from vllm.sequence import (Sequence, SequenceGroup, SequenceGroupOutput,
13
- SequenceOutput, SequenceStatus)
12
+from vllm.sequence import (VLLM_INVALID_TOKEN_ID, Sequence, SequenceGroup,
13
+ SequenceGroupOutput, SequenceOutput, SequenceStatus)
14
from vllm.transformers_utils.detokenizer import Detokenizer
15
from vllm.transformers_utils.tokenizer import AnyTokenizer
16
from vllm.utils import Counter
@@ -110,10 +110,11 @@ class MultiStepOutputProcessor(SequenceGroupOutputProcessor):
110
# we can take the first sample.
111
samples = [output.samples[0] for output in outputs]
112
113
- # -1 means the output token is not valid (eg. due to spec decode
113
+ # entries in sample tokens may be invalid (eg. due to spec decode
114
# rejecting tokens).
115
valid_samples = [
116
- sample for sample in samples if sample.output_token != -1
116
+ sample for sample in samples
117
+ if sample.output_token != VLLM_INVALID_TOKEN_ID
118
]
119
assert valid_samples
120
vllm/model_executor/layers/sampler.py
+6
-5
@@ -15,7 +15,8 @@ from vllm.model_executor.sampling_metadata import (SamplingMetadata,
15
SamplingTensors,
16
SequenceGroupToSample)
17
from vllm.sampling_params import SamplingType
18
-from vllm.sequence import (CompletionSequenceGroupOutput, Logprob,
18
+from vllm.sequence import (VLLM_INVALID_TOKEN_ID,
19
+ CompletionSequenceGroupOutput, Logprob,
20
PromptLogprobs, SampleLogprobs, SequenceOutput)
21
from vllm.spec_decode.metrics import SpecDecodeWorkerMetrics
22
@@ -759,10 +760,10 @@ def _sample_with_torch(
760
761
# Create output tensor for sampled token ids.
762
if include_gpu_probs_tensor:
762
- sampled_token_ids_tensor = torch.empty(logprobs.shape[0],
763
- 1,
764
- dtype=torch.long,
765
- device=logprobs.device)
763
+ sampled_token_ids_tensor = torch.full((logprobs.shape[0], 1),
764
+ VLLM_INVALID_TOKEN_ID,
765
+ dtype=torch.long,
766
+ device=logprobs.device)
767
else:
768
sampled_token_ids_tensor = None
769
vllm/sequence.py
+2
@@ -26,6 +26,8 @@ if TYPE_CHECKING:
26
27
VLLM_TOKEN_ID_ARRAY_TYPE = "l"
28
29
+VLLM_INVALID_TOKEN_ID = -1
30
+
31
32
# We use dataclass for now because it is used for
33
# openai server output, and msgspec is not serializable.
vllm/spec_decode/batch_expansion.py
+5
-5
@@ -6,9 +6,9 @@ import torch
6
7
from vllm import SamplingParams
8
from vllm.model_executor.layers.sampler import SamplerOutput
9
-from vllm.sequence import (VLLM_TOKEN_ID_ARRAY_TYPE, ExecuteModelRequest,
10
- SequenceData, SequenceGroupMetadata,
11
- get_all_seq_ids)
9
+from vllm.sequence import (VLLM_INVALID_TOKEN_ID, VLLM_TOKEN_ID_ARRAY_TYPE,
10
+ ExecuteModelRequest, SequenceData,
11
+ SequenceGroupMetadata, get_all_seq_ids)
12
from vllm.spec_decode.interfaces import (SpeculativeProposals,
13
SpeculativeScorer, SpeculativeScores)
14
from vllm.spec_decode.util import nvtx_range, split_batch_by_proposal_len
@@ -69,10 +69,10 @@ class BatchExpansionTop1Scorer(SpeculativeScorer):
69
proposal_lens_list = proposals.proposal_lens.tolist()
70
proposal_token_ids_list = proposals.proposal_token_ids.tolist()
71
72
- # Filter the list to ignore -1 proposals.
72
+ # Filter the list to ignore invalid proposals.
73
proposal_token_ids_list_without_skips = [
74
proposals for proposals in proposal_token_ids_list
75
- if -1 not in proposals
75
+ if VLLM_INVALID_TOKEN_ID not in proposals
76
]
77
78
(spec_indices, non_spec_indices, target_seq_group_metadata_list,
vllm/spec_decode/spec_decode_worker.py
+51
-11
@@ -13,9 +13,10 @@ from vllm.model_executor.layers.spec_decode_base_sampler import (
13
SpecDecodeBaseSampler, SpecDecodeStochasticBaseSampler)
14
from vllm.model_executor.layers.typical_acceptance_sampler import (
15
TypicalAcceptanceSampler)
16
-from vllm.sequence import (CompletionSequenceGroupOutput, ExecuteModelRequest,
16
+from vllm.sequence import (VLLM_INVALID_TOKEN_ID,
17
+ CompletionSequenceGroupOutput, ExecuteModelRequest,
18
HiddenStates, SequenceGroupMetadata,
18
- get_all_seq_ids, get_all_seq_ids_and_request_ids)
19
+ get_all_seq_ids_and_request_ids)
20
from vllm.spec_decode.batch_expansion import BatchExpansionTop1Scorer
21
from vllm.spec_decode.draft_model_runner import TP1DraftModelRunner
22
from vllm.spec_decode.interfaces import (SpeculativeProposals,
@@ -28,7 +29,8 @@ from vllm.spec_decode.ngram_worker import NGramWorker
29
from vllm.spec_decode.proposer_worker_base import ProposerWorkerBase
30
from vllm.spec_decode.smaller_tp_proposer_worker import SmallerTpProposerWorker
31
from vllm.spec_decode.target_model_runner import TargetModelRunner
31
-from vllm.spec_decode.util import (Timer, create_sequence_group_output,
32
+from vllm.spec_decode.util import (Timer, create_logprobs_output,
33
+ create_sequence_group_output,
34
get_all_num_logprobs,
35
get_sampled_token_logprobs, nvtx_range,
36
split_batch_by_proposal_len)
@@ -436,8 +438,8 @@ class SpecDecodeWorker(LoraNotSupportedWorkerBase):
438
self, execute_model_req: ExecuteModelRequest,
439
sampler_output: SamplerOutput) -> SamplerOutput:
440
"""
439
- Creates and returns a `SamplerOutput` with only the sampled token IDs
440
- being serialized to CPU & populated in `CompletionSequenceGroupOutput`.
441
+ Creates and returns a `SamplerOutput` with only the token IDs being
442
+ serialized to CPU and populated in `CompletionSequenceGroupOutput`.
443
All other parameters in `CompletionSequenceGroupOutput` related to log
444
probabilities are skipped.
445
@@ -449,14 +451,46 @@ class SpecDecodeWorker(LoraNotSupportedWorkerBase):
451
452
Returns:
453
SamplerOutput: A new `SamplerOutput` instance containing a list of
452
- `CompletionSequenceGroupOutput` objects with only sampled token
453
- IDs populated.
454
+ `CompletionSequenceGroupOutput` objects with only token IDs
455
+ populated.
456
"""
455
- seq_ids = get_all_seq_ids(execute_model_req.seq_group_metadata_list)
456
- sampled_token_ids_list = sampler_output.sampled_token_ids.tolist()
457
+ seq_output_prompt_logprobs = [
458
+ seq.is_prompt and seq.sampling_params.prompt_logprobs is not None
459
+ and seq.sampling_params.prompt_logprobs > 0
460
+ for seq in execute_model_req.seq_group_metadata_list
461
+ ]
462
+ # ignore slots for prompt tokens that are filled with INVALID_TOKEN_ID
463
+ sampled_token_ids_list = (sampler_output.sampled_token_ids[torch.where(
464
+ # subtracting is faster than testing for equality
465
+ sampler_output.sampled_token_ids - VLLM_INVALID_TOKEN_ID)[0]] \
466
+ if any(seq_output_prompt_logprobs) else \
467
+ sampler_output.sampled_token_ids).tolist()
468
+
469
+ seq_data_entries = (
470
+ (seq_id, seq_data) for sg in \
471
+ execute_model_req.seq_group_metadata_list \
472
+ for seq_id, seq_data in sg.seq_data.items()
473
+ )
474
completion_seq_group_output_list: List[
475
CompletionSequenceGroupOutput] = []
459
- for index, seq_id in enumerate(seq_ids):
476
+ for index, ((seq_id, seq_data), needs_prompt_logprobs) in \
477
+ enumerate(zip(seq_data_entries, seq_output_prompt_logprobs)):
478
+ if needs_prompt_logprobs:
479
+ prompt_token_ids = seq_data.get_prompt_token_ids()
480
+ prompt_logprobs = [
481
+ create_logprobs_output(
482
+ token_id=p_token_id,
483
+ token_id_logprob_rank=-1,
484
+ token_id_logprob=0.0,
485
+ topk_token_ids=[],
486
+ topk_logprobs=[],
487
+ )
488
+ # no prompt logprobs for the first token
489
+ for p_token_id in prompt_token_ids[1:]
490
+ ]
491
+ else:
492
+ prompt_logprobs = None
493
+
494
completion_seq_group_output_list.append(
495
create_sequence_group_output(
496
token_id=sampled_token_ids_list[index][0],
@@ -465,7 +499,7 @@ class SpecDecodeWorker(LoraNotSupportedWorkerBase):
499
seq_id=seq_id,
500
topk_token_ids=[],
501
topk_logprobs=[],
468
- ))
502
+ prompt_logprobs=prompt_logprobs))
503
return SamplerOutput(outputs=completion_seq_group_output_list)
504
505
@nvtx_range("spec_decode_worker._run_no_spec")
@@ -485,6 +519,12 @@ class SpecDecodeWorker(LoraNotSupportedWorkerBase):
519
# Store hidden states from target model execution.
520
hidden_states = sampler_output.hidden_states
521
if hidden_states is not None:
522
+ # remove hidden_states for prompt tokens
523
+ if any(seq.is_prompt
524
+ for seq in execute_model_req.seq_group_metadata_list):
525
+ hidden_states = hidden_states[
526
+ torch.where(sampler_output.sampled_token_ids -
527
+ VLLM_INVALID_TOKEN_ID)[0]]
528
if self.previous_hidden_states is None:
529
self.previous_hidden_states = HiddenStates(
530
hidden_states, execute_model_req.seq_group_metadata_list)
vllm/spec_decode/util.py
+37
-8
@@ -6,7 +6,8 @@ import torch
6
7
from vllm.model_executor.layers.sampler import SamplerOutput
8
from vllm.sequence import (CompletionSequenceGroupOutput, Logprob,
9
- SequenceGroupMetadata, SequenceOutput)
9
+ PromptLogprobs, SequenceGroupMetadata,
10
+ SequenceOutput)
11
12
SeqId = int
13
@@ -49,21 +50,19 @@ def get_sampled_token_logprobs(
50
return sampled_token_ids_ranks, selected_logprobs
51
52
52
-def create_sequence_group_output(
53
+def create_logprobs_output(
54
token_id: int,
55
token_id_logprob_rank: int,
56
token_id_logprob: float,
56
- seq_id: SeqId,
57
topk_token_ids: List[Optional[int]],
58
topk_logprobs: List[Optional[float]],
59
-) -> CompletionSequenceGroupOutput:
60
- """Create a SequenceGroupOutput given the sampling results.
59
+) -> Dict[int, Logprob]:
60
+ """Create a Logprob Dict for a token given the sampling results.
61
62
Args:
63
token_id (int): The sampled token for the sequence.
64
token_id_logprob_rank (int): The logprob rank of the sampled token.
65
token_id_logprob (float): The logprob value of the sampled token.
66
- seq_id (int): The sequence id.
66
topk_token_ids (List[Optional[int]]): The list of top-k token ids.
67
topk_logprobs (List[Optional[float]]): The list of top-k logprobs.
68
"""
@@ -85,14 +84,44 @@ def create_sequence_group_output(
84
if topk_token_id is not None
85
})
86
87
+ return logprobs
88
+
89
+
90
+def create_sequence_group_output(
91
+ token_id: int,
92
+ token_id_logprob_rank: int,
93
+ token_id_logprob: float,
94
+ seq_id: SeqId,
95
+ topk_token_ids: List[Optional[int]],
96
+ topk_logprobs: List[Optional[float]],
97
+ prompt_logprobs: Optional[PromptLogprobs] = None,
98
+) -> CompletionSequenceGroupOutput:
99
+ """Create a SequenceGroupOutput given the sampling results.
100
+
101
+ Args:
102
+ token_id (int): The sampled token for the sequence.
103
+ token_id_logprob_rank (int): The logprob rank of the sampled token.
104
+ token_id_logprob (float): The logprob value of the sampled token.
105
+ seq_id (int): The sequence id.
106
+ topk_token_ids (List[Optional[int]]): The list of top-k token ids.
107
+ topk_logprobs (List[Optional[float]]): The list of top-k logprobs.
108
+ """
109
+
110
+ logprobs = create_logprobs_output(
111
+ token_id,
112
+ token_id_logprob_rank,
113
+ token_id_logprob,
114
+ topk_token_ids,
115
+ topk_logprobs,
116
+ )
117
+
118
return CompletionSequenceGroupOutput(
119
samples=[
120
SequenceOutput(parent_seq_id=seq_id,
121
output_token=token_id,
122
logprobs=logprobs)
123
],
94
- # TODO add prompt logprobs support.
95
- prompt_logprobs=None,
124
+ prompt_logprobs=prompt_logprobs,
125
)
126
127
vllm/transformers_utils/detokenizer.py
+7
-9
@@ -1,13 +1,11 @@
1
from typing import Dict, List, Optional, Tuple
2
3
-from vllm.sequence import Logprob, SamplingParams, Sequence, SequenceGroup
3
+from vllm.sequence import (VLLM_INVALID_TOKEN_ID, Logprob, SamplingParams,
4
+ Sequence, SequenceGroup)
5
6
from .tokenizer import AnyTokenizer
7
from .tokenizer_group import BaseTokenizerGroup
8
8
-# Used eg. for marking rejected tokens in spec decoding.
9
-INVALID_TOKEN_ID = -1
10
-
9
10
class Detokenizer:
11
"""Provides methods to decode the output of a model into text."""
@@ -61,7 +59,7 @@ class Detokenizer:
59
continue
60
for token_id, sample_logprob in prompt_logprobs_for_token.items():
61
if (sample_logprob.decoded_token is None
64
- and token_id != INVALID_TOKEN_ID):
62
+ and token_id != VLLM_INVALID_TOKEN_ID):
63
prompt_token_ids_with_token = (
64
prompt_token_ids[:token_position] + [token_id])
65
(new_tokens, new_text, new_prefix_offset,
@@ -143,7 +141,7 @@ class Detokenizer:
141
continue
142
143
if (sample_logprob.decoded_token is None
146
- and token_id != INVALID_TOKEN_ID):
144
+ and token_id != VLLM_INVALID_TOKEN_ID):
145
all_input_ids_with_logprob = previous_tokens + [token_id]
146
(_, new_text, _, _) = detokenize_incrementally(
147
tokenizer=tokenizer,
@@ -282,14 +280,14 @@ def detokenize_incrementally(
280
assert prev_tokens is not None
281
282
# If the new token id is out of bounds, return an empty string.
285
- if new_token_id >= len(tokenizer):
286
- new_tokens = [""]
287
- else:
283
+ if 0 <= new_token_id < len(tokenizer):
284
# Put new_token_id in a list so skip_special_tokens is respected
285
new_tokens = tokenizer.convert_ids_to_tokens(
286
[new_token_id], skip_special_tokens=skip_special_tokens)
287
if isinstance(new_tokens, str):
288
new_tokens = [new_tokens]
289
+ else:
290
+ new_tokens = [""]
291
output_tokens = prev_tokens + new_tokens
292
293
# If this is the first iteration, return all tokens.