A high-throughput and memory-efficient inference and serving engine for LLMs
Mirrored from https://github.com/tduncan-lum/Nvidia-Optimization-Stack---VLLM.git
Synced Nov 17, 01:45 UTC
[Speculative decoding] Support target-model logprobs (#4378)
Cade Daniel committed
May 3, 2024 at 15:52 UTC
ab502751117d3785384b9c33ee88e0aff93bbf05
15 files changed
+728
-87
tests/spec_decode/e2e/conftest.py
+63
-3
@@ -1,9 +1,13 @@
1
import asyncio
2
+import time
3
from itertools import cycle
3
-from typing import List, Optional, Tuple, Union
4
+from typing import Dict, List, Optional, Tuple, Union
5
6
import pytest
7
import ray
8
+import torch
9
+from pynvml import (nvmlDeviceGetHandleByIndex, nvmlDeviceGetMemoryInfo,
10
+ nvmlInit)
11
12
from tests.conftest import cleanup
13
from vllm import LLM
@@ -13,7 +17,7 @@ from vllm.lora.request import LoRARequest
17
from vllm.model_executor.utils import set_random_seed
18
from vllm.outputs import RequestOutput
19
from vllm.sampling_params import SamplingParams
16
-from vllm.sequence import MultiModalData
20
+from vllm.sequence import Logprob, MultiModalData
21
from vllm.usage.usage_lib import UsageContext
22
from vllm.utils import Counter, random_uuid
23
@@ -153,12 +157,19 @@ def create_llm_generator(baseline_or_test, request, common_llm_kwargs,
157
test_name = request.node.name
158
159
def generator_inner():
156
- print(f'Creating {baseline_or_test=} LLM for {test_name=}. {kwargs=}')
160
+
161
+ wait_for_gpu_memory_to_clear(
162
+ devices=list(range(torch.cuda.device_count())),
163
+ threshold_bytes=2 * 2**30,
164
+ timeout_s=60,
165
+ )
166
167
use_async = False
168
if "use_async" in kwargs:
169
use_async = kwargs.pop("use_async")
170
+ print(f'{use_async=}')
171
172
+ print(f'Creating {baseline_or_test=} LLM for {test_name=}. {kwargs=}')
173
llm = AsyncLLM(**kwargs) if use_async else LLM(**kwargs)
174
set_random_seed(seed)
175
@@ -188,6 +199,20 @@ def get_output_from_llm_generator(
199
return tokens, token_ids
200
201
202
+def get_logprobs_from_llm_generator(
203
+ llm_generator, prompts,
204
+ sampling_params) -> List[List[Dict[int, Logprob]]]:
205
+ """Returns a dict of (token_id: Logprob) for each generated position, for
206
+ each sequence in the batch.
207
+ """
208
+ for llm in llm_generator():
209
+ outputs = llm.generate(prompts, sampling_params, use_tqdm=True)
210
+ logprobs = [output.outputs[0].logprobs[:] for output in outputs]
211
+ del llm
212
+
213
+ return logprobs
214
+
215
+
216
def run_greedy_equality_correctness_test(baseline_llm_generator,
217
test_llm_generator,
218
batch_size,
@@ -243,3 +268,38 @@ def run_greedy_equality_correctness_test(baseline_llm_generator,
268
print(f'{i=} {baseline_token_ids=}')
269
print(f'{i=} {spec_token_ids=}')
270
assert baseline_token_ids == spec_token_ids
271
+
272
+
273
+def wait_for_gpu_memory_to_clear(devices: List[int],
274
+ threshold_bytes: int,
275
+ timeout_s: float = 120) -> None:
276
+ # Use nvml instead of pytorch to reduce measurement error from torch cuda
277
+ # context.
278
+ nvmlInit()
279
+ start_time = time.time()
280
+ while True:
281
+ output = {}
282
+ output_raw = {}
283
+ for device in devices:
284
+ dev_handle = nvmlDeviceGetHandleByIndex(device)
285
+ mem_info = nvmlDeviceGetMemoryInfo(dev_handle)
286
+ gb_used = mem_info.used / 2**30
287
+ output_raw[device] = gb_used
288
+ output[device] = f'{gb_used:.02f}'
289
+
290
+ print('gpu memory used (GB): ', end='')
291
+ for k, v in output.items():
292
+ print(f'{k}={v}; ', end='')
293
+ print('')
294
+
295
+ dur_s = time.time() - start_time
296
+ if all(v <= (threshold_bytes / 2**30) for v in output_raw.values()):
297
+ print(f'Done waiting for free GPU memory on devices {devices=} '
298
+ f'({threshold_bytes/2**30=}) {dur_s=:.02f}')
299
+ break
300
+
301
+ if dur_s >= timeout_s:
302
+ raise ValueError(f'Memory of devices {devices=} not free after '
303
+ f'{dur_s=:.02f} ({threshold_bytes/2**30=})')
304
+
305
+ time.sleep(5)
tests/spec_decode/e2e/test_logprobs.py
new
+335
@@ -0,0 +1,335 @@
1
+import math
2
+from itertools import cycle
3
+
4
+import pytest
5
+
6
+from vllm import SamplingParams
7
+
8
+from .conftest import get_logprobs_from_llm_generator
9
+
10
+
11
+@pytest.mark.parametrize(
12
+ "common_llm_kwargs",
13
+ [{
14
+ "model": "JackFram/llama-68m",
15
+
16
+ # Skip cuda graph recording for fast test.
17
+ "enforce_eager": True,
18
+
19
+ # Required for spec decode.
20
+ "use_v2_block_manager": True,
21
+ "max_logprobs": 6,
22
+ }])
23
+@pytest.mark.parametrize("per_test_common_llm_kwargs", [{}])
24
+@pytest.mark.parametrize("baseline_llm_kwargs", [{}])
25
+@pytest.mark.parametrize("test_llm_kwargs", [{
26
+ "speculative_model": "JackFram/llama-160m",
27
+ "num_speculative_tokens": 3,
28
+}])
29
+@pytest.mark.parametrize("batch_size", [8])
30
+@pytest.mark.parametrize(
31
+ "output_len",
32
+ [
33
+ # Use smaller output len for fast test.
34
+ 7,
35
+ ])
36
+@pytest.mark.parametrize("seed", [1])
37
+def test_logprobs_equality(baseline_llm_generator, test_llm_generator,
38
+ batch_size: int, output_len: int):
39
+ """Verify output logprobs are equal with and without speculative decoding.
40
+ """
41
+ run_greedy_logprobs_correctness_test(baseline_llm_generator,
42
+ test_llm_generator,
43
+ batch_size,
44
+ max_output_len=output_len,
45
+ force_output_len=True)
46
+
47
+
48
+@pytest.mark.parametrize(
49
+ "common_llm_kwargs",
50
+ [{
51
+ "model": "JackFram/llama-68m",
52
+
53
+ # Skip cuda graph recording for fast test.
54
+ "enforce_eager": True,
55
+
56
+ # Required for spec decode.
57
+ "use_v2_block_manager": True,
58
+ "max_logprobs": 6,
59
+ }])
60
+@pytest.mark.parametrize("per_test_common_llm_kwargs", [{}])
61
+@pytest.mark.parametrize("baseline_llm_kwargs", [{}])
62
+@pytest.mark.parametrize("test_llm_kwargs", [{
63
+ "speculative_model": "JackFram/llama-160m",
64
+ "num_speculative_tokens": 3,
65
+}])
66
+@pytest.mark.parametrize("batch_size", [1])
67
+@pytest.mark.parametrize("num_logprobs", [6])
68
+@pytest.mark.parametrize(
69
+ "output_len",
70
+ [
71
+ # Use smaller output len for fast test.
72
+ 7,
73
+ ])
74
+@pytest.mark.parametrize("seed", [1])
75
+def test_diff_num_logprobs(baseline_llm_generator, test_llm_generator,
76
+ batch_size: int, output_len: int,
77
+ num_logprobs: int):
78
+ """Verify output logprobs are equal with and without spec decode.
79
+ This specifies a number of logprobs >1.
80
+ """
81
+ run_greedy_logprobs_correctness_test(baseline_llm_generator,
82
+ test_llm_generator,
83
+ batch_size,
84
+ max_output_len=output_len,
85
+ force_output_len=True,
86
+ logprob_rank=num_logprobs)
87
+
88
+
89
+@pytest.mark.parametrize(
90
+ "common_llm_kwargs",
91
+ [{
92
+ "model": "JackFram/llama-68m",
93
+
94
+ # Skip cuda graph recording for fast test.
95
+ "enforce_eager": True,
96
+
97
+ # Required for spec decode.
98
+ "use_v2_block_manager": True
99
+ }])
100
+@pytest.mark.parametrize("per_test_common_llm_kwargs", [{}])
101
+@pytest.mark.parametrize("baseline_llm_kwargs", [{}])
102
+@pytest.mark.parametrize("test_llm_kwargs", [{
103
+ "speculative_model": "JackFram/llama-160m",
104
+ "num_speculative_tokens": 3,
105
+}, {
106
+ "speculative_model": "JackFram/llama-160m",
107
+ "num_speculative_tokens": 6,
108
+}])
109
+@pytest.mark.parametrize("batch_size", [8])
110
+@pytest.mark.parametrize(
111
+ "output_len",
112
+ [
113
+ # Use smaller output len for fast test.
114
+ 32,
115
+ ])
116
+@pytest.mark.parametrize("seed", [1])
117
+def test_logprobs_different_k(baseline_llm_generator, test_llm_generator,
118
+ batch_size: int, output_len: int):
119
+ """Veriy logprob greedy equality with different speculation lens.
120
+ """
121
+ run_greedy_logprobs_correctness_test(baseline_llm_generator,
122
+ test_llm_generator,
123
+ batch_size,
124
+ max_output_len=output_len,
125
+ force_output_len=True)
126
+
127
+
128
+@pytest.mark.parametrize(
129
+ "common_llm_kwargs",
130
+ [{
131
+ "model": "JackFram/llama-68m",
132
+
133
+ # Skip cuda graph recording for fast test.
134
+ "enforce_eager": True,
135
+
136
+ # Required for spec decode.
137
+ "use_v2_block_manager": True
138
+ }])
139
+@pytest.mark.parametrize("per_test_common_llm_kwargs", [{}])
140
+@pytest.mark.parametrize("baseline_llm_kwargs", [{}])
141
+@pytest.mark.parametrize(
142
+ "test_llm_kwargs",
143
+ [{
144
+ "speculative_model": "JackFram/llama-160m",
145
+ "num_speculative_tokens": 3,
146
+
147
+ # Artificially limit the draft model max model len; this forces vLLM
148
+ # to skip speculation once the sequences grow beyond 32-k tokens.
149
+ "speculative_max_model_len": 32,
150
+ }])
151
+@pytest.mark.parametrize("batch_size", [8])
152
+@pytest.mark.parametrize(
153
+ "output_len",
154
+ [
155
+ # Use smaller output len for fast test.
156
+ 32,
157
+ ])
158
+@pytest.mark.parametrize("seed", [1])
159
+def test_logprobs_when_skip_speculation(baseline_llm_generator,
160
+ test_llm_generator, batch_size: int,
161
+ output_len: int):
162
+ """Verify logprobs greedy equality when some sequences skip speculation.
163
+ """
164
+ run_greedy_logprobs_correctness_test(baseline_llm_generator,
165
+ test_llm_generator,
166
+ batch_size,
167
+ max_output_len=output_len,
168
+ force_output_len=True)
169
+
170
+
171
+@pytest.mark.parametrize(
172
+ "common_llm_kwargs",
173
+ [{
174
+ "model": "JackFram/llama-68m",
175
+
176
+ # Skip cuda graph recording for fast test.
177
+ "enforce_eager": True,
178
+
179
+ # Required for spec decode.
180
+ "use_v2_block_manager": True
181
+ }])
182
+@pytest.mark.parametrize("per_test_common_llm_kwargs", [{}])
183
+@pytest.mark.parametrize("baseline_llm_kwargs", [{}])
184
+@pytest.mark.parametrize("test_llm_kwargs", [{
185
+ "speculative_model": "JackFram/llama-160m",
186
+ "num_speculative_tokens": 3,
187
+}])
188
+@pytest.mark.parametrize("batch_size", [1])
189
+@pytest.mark.parametrize(
190
+ "output_len",
191
+ [
192
+ # Use smaller output len for fast test.
193
+ 32,
194
+ ])
195
+@pytest.mark.parametrize("seed", [1])
196
+def test_logprobs_temp_1(baseline_llm_generator, test_llm_generator,
197
+ batch_size: int, output_len: int):
198
+ """Verify at least one logprob result has num_logprobs+1, which tests the
199
+ case where the sampled token is not in top-k logprobs.
200
+
201
+ Ideally, this test should validate equality with non-spec by getting
202
+ logprobs. This is left as future improvement.
203
+ """
204
+ batch_size = 8
205
+ max_output_len = output_len
206
+ force_output_len = True
207
+ logprob_rank = 5
208
+
209
+ temperature = 1.0
210
+
211
+ prompts = [
212
+ "Hello, my name is",
213
+ "The president of the United States is",
214
+ "The capital of France is",
215
+ "The future of AI is",
216
+ "San Francisco is know for its",
217
+ "Facebook was created in 2004 by",
218
+ "Curious George is a",
219
+ "Python 3.11 brings improvements to its",
220
+ ]
221
+
222
+ prompts = [prompt for prompt, _ in zip(cycle(prompts), range(batch_size))]
223
+
224
+ # If the test requires that we generated max_output_len tokens, then set the
225
+ # sampling params to ignore eos token.
226
+ ignore_eos = force_output_len
227
+
228
+ sampling_params = SamplingParams(
229
+ max_tokens=max_output_len,
230
+ ignore_eos=ignore_eos,
231
+ temperature=temperature,
232
+ logprobs=logprob_rank,
233
+ )
234
+
235
+ spec_batch_logprobs = get_logprobs_from_llm_generator(
236
+ test_llm_generator, prompts, sampling_params)
237
+
238
+ num_returned_logprobs = [
239
+ len(logprob_dict) for seq_logprobs in spec_batch_logprobs
240
+ for logprob_dict in seq_logprobs
241
+ ]
242
+
243
+ # Assert one of the returned logprobs has > num_logprobs (indicating the
244
+ # sampled token is not in top-k).
245
+ assert any([
246
+ num_returned > logprob_rank for num_returned in num_returned_logprobs
247
+ ])
248
+
249
+
250
+def run_greedy_logprobs_correctness_test(baseline_llm_generator,
251
+ test_llm_generator,
252
+ batch_size,
253
+ max_output_len,
254
+ force_output_len: bool,
255
+ logprob_rank: int = 1):
256
+ """Helper method that compares the logprobs outputs of both the baseline LLM
257
+ and the test LLM. It asserts greedy equality of the logprobs when the
258
+ temperature is zero.
259
+ """
260
+ temperature = 0.0
261
+
262
+ prompts = [
263
+ "Hello, my name is",
264
+ "The president of the United States is",
265
+ "The capital of France is",
266
+ "The future of AI is",
267
+ "San Francisco is know for its",
268
+ "Facebook was created in 2004 by",
269
+ "Curious George is a",
270
+ "Python 3.11 brings improvements to its",
271
+ ]
272
+
273
+ prompts = [prompt for prompt, _ in zip(cycle(prompts), range(batch_size))]
274
+
275
+ # If the test requires that we generated max_output_len tokens, then set the
276
+ # sampling params to ignore eos token.
277
+ ignore_eos = force_output_len
278
+
279
+ sampling_params = SamplingParams(
280
+ max_tokens=max_output_len,
281
+ ignore_eos=ignore_eos,
282
+ temperature=temperature,
283
+ logprobs=logprob_rank,
284
+ )
285
+
286
+ spec_batch_logprobs = get_logprobs_from_llm_generator(
287
+ test_llm_generator, prompts, sampling_params)
288
+ baseline_batch_logprobs = get_logprobs_from_llm_generator(
289
+ baseline_llm_generator, prompts, sampling_params)
290
+
291
+ assert len(baseline_batch_logprobs) == len(prompts)
292
+ assert len(spec_batch_logprobs) == len(prompts)
293
+
294
+ # For each sequence in the batch.
295
+ for i, (baseline_logprobs, spec_logprobs) in enumerate(
296
+ zip(baseline_batch_logprobs, spec_batch_logprobs)):
297
+ assert len(spec_logprobs) == len(baseline_logprobs)
298
+
299
+ # For each generated position of the sequence.
300
+ for pos, (spec_pos_logprobs, baseline_pos_logprobs) in enumerate(
301
+ zip(spec_logprobs, baseline_logprobs)):
302
+
303
+ # Map rank to token/logprob in spec output.
304
+ spec_rank_to_token_id = {
305
+ value.rank: key
306
+ for key, value in spec_pos_logprobs.items()
307
+ }
308
+ spec_rank_to_logprob = {
309
+ value.rank: value.logprob
310
+ for key, value in spec_pos_logprobs.items()
311
+ }
312
+
313
+ # Map rank to token/logprob in baseline output.
314
+ baseline_rank_to_token_id = {
315
+ value.rank: key
316
+ for key, value in baseline_pos_logprobs.items()
317
+ }
318
+ baseline_rank_to_logprob = {
319
+ value.rank: value.logprob
320
+ for key, value in baseline_pos_logprobs.items()
321
+ }
322
+
323
+ # Assert set of ranks returned is equal.
324
+ assert set(spec_rank_to_token_id.keys()) == set(
325
+ baseline_rank_to_token_id.keys())
326
+
327
+ # Assert each logprob/token id is correct, keyed by rank.
328
+ for rank in sorted(set(spec_rank_to_token_id.keys())):
329
+ assert spec_rank_to_token_id[
330
+ rank] == baseline_rank_to_token_id[rank], f"{rank}"
331
+ assert math.isclose(
332
+ a=spec_rank_to_logprob[rank],
333
+ b=baseline_rank_to_logprob[rank],
334
+ abs_tol=1e-1,
335
+ )
tests/spec_decode/e2e/test_multistep_correctness.py
+47
-16
@@ -41,24 +41,17 @@ from .conftest import (get_output_from_llm_generator,
41
42
@pytest.mark.parametrize(
43
"common_llm_kwargs",
44
- [
45
- {
46
- # Use a small model for a fast test.
47
- # Note this is repeated in the test body; to initialize a tokenizer.
48
- "model": "JackFram/llama-68m",
49
-
50
- # Skip cuda graph recording for fast test.
51
- "enforce_eager": True,
44
+ [{
45
+ # Use a small model for a fast test.
46
+ # Note this is repeated in the test body; to initialize a tokenizer.
47
+ "model": "JackFram/llama-68m",
48
53
- # Required for spec decode.
54
- "use_v2_block_manager": True,
49
+ # Skip cuda graph recording for fast test.
50
+ "enforce_eager": True,
51
56
- # whether use AsyncLLM engine
57
- "use_async": async_mode,
58
- }
59
- # Try both async and sync engine execution
60
- for async_mode in [True, False]
61
- ])
52
+ # Required for spec decode.
53
+ "use_v2_block_manager": True,
54
+ }])
55
@pytest.mark.parametrize(
56
"per_test_common_llm_kwargs",
57
[
@@ -117,6 +110,44 @@ def test_spec_decode_e2e_with_detokenization(test_llm_generator,
110
assert actual_tokens.strip() == expected_tokens.strip()
111
112
113
+@pytest.mark.parametrize(
114
+ "common_llm_kwargs",
115
+ [{
116
+ # Use a small model for a fast test.
117
+ # Note this is repeated in the test body; to initialize a tokenizer.
118
+ "model": "JackFram/llama-68m",
119
+
120
+ # Skip cuda graph recording for fast test.
121
+ "enforce_eager": True,
122
+
123
+ # Required for spec decode.
124
+ "use_v2_block_manager": True,
125
+
126
+ # Use AsyncLLM engine
127
+ "use_async": True,
128
+ }])
129
+@pytest.mark.parametrize("baseline_llm_kwargs", [{}])
130
+@pytest.mark.parametrize("per_test_common_llm_kwargs", [
131
+ {
132
+ "speculative_model": "JackFram/llama-68m",
133
+ "num_speculative_tokens": 5,
134
+ },
135
+])
136
+@pytest.mark.parametrize("test_llm_kwargs", [{}])
137
+@pytest.mark.parametrize("batch_size", [2])
138
+@pytest.mark.parametrize("seed", [1])
139
+def test_spec_decode_e2e_with_async_engine(test_llm_generator,
140
+ baseline_llm_generator,
141
+ batch_size: int):
142
+ """Verify spec decode works well with async LLM engine.
143
+ """
144
+ run_greedy_equality_correctness_test(baseline_llm_generator,
145
+ test_llm_generator,
146
+ batch_size,
147
+ max_output_len=32,
148
+ force_output_len=True)
149
+
150
+
151
@pytest.mark.parametrize(
152
"common_llm_kwargs",
153
[{
tests/spec_decode/test_multi_step_worker.py
+8
@@ -292,6 +292,10 @@ def test_draft_proposals_full_speculation_len():
292
vocab_size,
293
device=device,
294
dtype=torch.float32),
295
+ logprobs=torch.rand(batch_size,
296
+ vocab_size,
297
+ device=device,
298
+ dtype=torch.float32),
299
sampled_token_ids=torch.randint(low=0,
300
high=vocab_size,
301
size=(batch_size, ),
@@ -392,6 +396,10 @@ def test_draft_proposals_mixed_k():
396
vocab_size,
397
device=device,
398
dtype=torch.float32),
399
+ logprobs=torch.rand(expected_num_proposal_seqs,
400
+ vocab_size,
401
+ device=device,
402
+ dtype=torch.float32),
403
sampled_token_ids=torch.randint(
404
low=0,
405
high=vocab_size,
tests/spec_decode/test_spec_decode_worker.py
+24
-5
@@ -192,8 +192,14 @@ def test_correctly_calls_rejection_sampler(k: int, batch_size: int):
192
vocab_size,
193
dtype=torch.float32,
194
device='cuda')
195
+ target_token_logprobs = torch.rand(1,
196
+ batch_size * (k + 1),
197
+ vocab_size,
198
+ dtype=torch.float32,
199
+ device='cuda')
200
target_output = create_sampler_output_list(target_token_ids,
196
- target_token_probs)
201
+ target_token_probs,
202
+ target_token_logprobs)
203
204
target_worker.execute_model.return_value = [target_output[0]]
205
@@ -273,8 +279,14 @@ def test_correctly_formats_output(k: int, batch_size: int):
279
vocab_size,
280
dtype=torch.float32,
281
device='cuda')
282
+ target_token_logprobs = torch.rand(1,
283
+ batch_size * (k + 1),
284
+ vocab_size,
285
+ dtype=torch.float32,
286
+ device='cuda')
287
target_output = create_sampler_output_list(target_token_ids,
277
- target_token_probs)
288
+ target_token_probs,
289
+ target_token_logprobs)
290
291
target_worker.execute_model.return_value = [target_output[0]]
292
@@ -294,7 +306,9 @@ def test_correctly_formats_output(k: int, batch_size: int):
306
num_lookahead_slots=k)
307
308
expected_output = create_sampler_output_list(
297
- rejection_sampler_output.transpose(0, 1), [None for _ in range(k + 1)])
309
+ token_ids=rejection_sampler_output.transpose(0, 1),
310
+ probs=[None for _ in range(k + 1)],
311
+ logprobs=[None for _ in range(k + 1)])
312
313
seq_ids = [
314
next(iter(seq_group_metadata.seq_data.keys()))
@@ -328,7 +342,6 @@ def test_correctly_formats_output(k: int, batch_size: int):
342
continue
343
assert actual_by_step[i].output_token == expected_by_step[
344
i].output_token
331
- assert actual_by_step[i].logprobs == expected_by_step[i].logprobs
345
346
347
@pytest.mark.parametrize('k', [1, 2])
@@ -387,8 +400,14 @@ def test_collects_metrics(k: int, batch_size: int, returns_metrics: bool):
400
vocab_size,
401
dtype=torch.float32,
402
device='cuda')
403
+ target_token_logprobs = torch.rand(1,
404
+ batch_size * (k + 1),
405
+ vocab_size,
406
+ dtype=torch.float32,
407
+ device='cuda')
408
target_output = create_sampler_output_list(target_token_ids,
391
- target_token_probs)
409
+ target_token_probs,
410
+ target_token_logprobs)
411
412
target_worker.execute_model.return_value = [target_output[0]]
413
tests/spec_decode/utils.py
+2
@@ -201,6 +201,7 @@ def assert_logprobs_dict_allclose(
201
def create_sampler_output_list(
202
token_ids: torch.Tensor,
203
probs: Iterable[Optional[torch.Tensor]],
204
+ logprobs: Iterable[Optional[torch.Tensor]],
205
seq_ids: Optional[List[int]] = None) -> List[SamplerOutput]:
206
num_steps, batch_size = token_ids.shape
207
token_ids_by_step = token_ids.tolist()
@@ -222,6 +223,7 @@ def create_sampler_output_list(
223
) for seq_index, token_id in enumerate(token_ids_by_step[step])
224
],
225
sampled_token_probs=probs[step],
226
+ logprobs=logprobs[step],
227
sampled_token_ids=token_ids[step])
228
for step in range(num_steps)
229
]
vllm/engine/output_processor/multi_step.py
+12
-6
@@ -1,3 +1,4 @@
1
+import functools
2
from typing import Callable, List
3
4
from transformers import PreTrainedTokenizer
@@ -8,8 +9,8 @@ from vllm.engine.output_processor.interfaces import (
9
from vllm.engine.output_processor.stop_checker import StopChecker
10
from vllm.logger import init_logger
11
from vllm.sampling_params import SamplingParams
11
-from vllm.sequence import (Logprob, Sequence, SequenceGroup,
12
- SequenceGroupOutput, SequenceOutput, SequenceStatus)
12
+from vllm.sequence import (Sequence, SequenceGroup, SequenceGroupOutput,
13
+ SequenceOutput, SequenceStatus)
14
from vllm.transformers_utils.detokenizer import Detokenizer
15
from vllm.utils import Counter
16
@@ -48,10 +49,14 @@ class MultiStepOutputProcessor(SequenceGroupOutputProcessor):
49
outputs: List[SequenceGroupOutput]) -> None:
50
# TODO(sang): Prompt logprob currently not implemented in multi step
51
# workers.
52
+ self._log_prompt_logprob_unsupported_warning_once()
53
+
54
+ @staticmethod
55
+ @functools.lru_cache()
56
+ def _log_prompt_logprob_unsupported_warning_once():
57
logger.warning(
58
"Prompt logprob is not supported by multi step workers. "
59
"(e.g., speculative decode uses multi step workers).")
54
- pass
60
61
def process_outputs(self, sequence_group: SequenceGroup,
62
outputs: List[SequenceGroupOutput]) -> None:
@@ -89,6 +94,7 @@ class MultiStepOutputProcessor(SequenceGroupOutputProcessor):
94
valid_samples: List[SequenceOutput],
95
sampling_params: SamplingParams) -> None:
96
output_token_ids = [sample.output_token for sample in valid_samples]
97
+ output_logprobs = [sample.logprobs for sample in valid_samples]
98
99
# Truncate to max_tokens if necessary.
100
remaining_tokens = sampling_params.max_tokens - (seq.get_output_len() +
@@ -113,11 +119,11 @@ class MultiStepOutputProcessor(SequenceGroupOutputProcessor):
119
120
# Incrementally append tokens to the sequence, as if we had only one new
121
# token.
116
- for output_token_id in output_token_ids:
122
+ for output_token_id, output_logprob in zip(output_token_ids,
123
+ output_logprobs):
124
seq.append_token_id(
125
token_id=output_token_id,
119
- # TODO emit logprobs in multi-step decoding.
120
- logprobs={output_token_id: Logprob(0.0)},
126
+ logprobs=output_logprob,
127
)
128
129
new_char_count = 0
vllm/model_executor/layers/sampler.py
+9
-7
@@ -103,8 +103,7 @@ class Sampler(nn.Module):
103
104
if self.include_gpu_probs_tensor:
105
assert maybe_sampled_tokens_tensor is not None
106
- sampled_tokens_tensor = maybe_sampled_tokens_tensor
107
- on_device_tensors = (probs, sampled_tokens_tensor)
106
+ on_device_tensors = (probs, logprobs, maybe_sampled_tokens_tensor)
107
else:
108
on_device_tensors = None
109
@@ -965,8 +964,7 @@ def _modify_greedy_probs_inplace(logprobs: torch.Tensor, probs: torch.Tensor,
964
has implications on the overall design of the sampler, e.g. how to record
965
accurate logprobs for the user, so this improvement is deferred to later.
966
"""
968
- logprobs[sample_indices, :] = -float('inf')
969
- logprobs[sample_indices, greedy_samples] = 0.0
967
+ # NOTE: logprobs are not modified so they can be returned to the user.
968
probs[sample_indices, :] = 0
969
probs[sample_indices, greedy_samples] = 1.0
970
@@ -976,7 +974,8 @@ def _build_sampler_output(
974
sampling_metadata: SamplingMetadata,
975
prompt_logprobs: List[Optional[PromptLogprobs]],
976
sample_logprobs: List[SampleLogprobs],
979
- on_device_tensors: Optional[Tuple[torch.Tensor, torch.Tensor]],
977
+ on_device_tensors: Optional[Tuple[torch.Tensor, torch.Tensor,
978
+ torch.Tensor]],
979
) -> SamplerOutput:
980
"""Construct Python objects with the output of sampling.
981
@@ -1005,14 +1004,17 @@ def _build_sampler_output(
1004
1005
# If not specified, store None values in SamplerOutput.
1006
if on_device_tensors is not None:
1008
- sampled_token_probs, sampled_token_ids = on_device_tensors
1007
+ (sampled_token_probs, logprobs_tensor,
1008
+ sampled_token_ids) = on_device_tensors
1009
else:
1010
- sampled_token_probs, sampled_token_ids = (None, None)
1010
+ sampled_token_probs, logprobs_tensor, sampled_token_ids = (None, None,
1011
+ None)
1012
1013
return SamplerOutput(
1014
outputs=sampler_output,
1015
sampled_token_probs=sampled_token_probs,
1016
sampled_token_ids=sampled_token_ids,
1017
+ logprobs=logprobs_tensor,
1018
)
1019
1020
vllm/sequence.py
+3
@@ -700,6 +700,9 @@ class SamplerOutput:
700
# On-device tensor containing probabilities of each token.
701
sampled_token_probs: Optional["torch.Tensor"] = None
702
703
+ # On-device tensor containing the logprobs of each token.
704
+ logprobs: Optional["torch.Tensor"] = None
705
+
706
# On-device tensor containing the sampled token ids.
707
sampled_token_ids: Optional["torch.Tensor"] = None
708
vllm/spec_decode/batch_expansion.py
+41
-18
@@ -94,7 +94,7 @@ class BatchExpansionTop1Scorer(SpeculativeScorer):
94
assert len(target_sampler_output) == 1, "expected single-step output"
95
target_sampler_output = target_sampler_output[0]
96
97
- all_tokens, all_probs = self._contract_batch(
97
+ all_tokens, all_probs, spec_logprobs = self._contract_batch(
98
contracted_bs=len(seq_group_metadata_list),
99
target_sampler_output=target_sampler_output,
100
proposals=proposals,
@@ -107,6 +107,7 @@ class BatchExpansionTop1Scorer(SpeculativeScorer):
107
return SpeculativeScores(
108
probs=all_probs,
109
token_ids=all_tokens,
110
+ logprobs=spec_logprobs,
111
)
112
113
def _expand_batch(
@@ -148,12 +149,12 @@ class BatchExpansionTop1Scorer(SpeculativeScorer):
149
return (spec_indices, non_spec_indices, target_seq_group_metadata_list,
150
num_scoring_tokens)
151
151
- def _contract_batch(self, contracted_bs: int,
152
- target_sampler_output: List[SamplerOutput],
153
- proposals: SpeculativeProposals,
154
- num_scoring_tokens: int, non_spec_indices: List[int],
155
- spec_indices: List[int],
156
- k: int) -> Tuple[torch.Tensor, torch.Tensor]:
152
+ def _contract_batch(
153
+ self, contracted_bs: int,
154
+ target_sampler_output: List[SamplerOutput],
155
+ proposals: SpeculativeProposals, num_scoring_tokens: int,
156
+ non_spec_indices: List[int], spec_indices: List[int],
157
+ k: int) -> Tuple[torch.Tensor, torch.Tensor, torch.Tensor]:
158
"""Contract the expanded batch back into its original size.
159
This maps the scores of speculative tokens back to their original
160
sequences.
@@ -161,8 +162,9 @@ class BatchExpansionTop1Scorer(SpeculativeScorer):
162
contracted_bs is the original batch size, and the batch size that the
163
target_sampler_output will be contracted to.
164
"""
164
- (target_token_ids, target_probs, non_spec_target_token_ids,
165
- non_spec_target_probs) = self._split_scoring_output(
165
+ (target_token_ids, target_probs, target_logprobs,
166
+ non_spec_target_token_ids, non_spec_target_probs,
167
+ non_spec_target_logprobs) = self._split_scoring_output(
168
target_sampler_output, num_scoring_tokens)
169
170
# Map distinct sequences used to score each token
@@ -179,6 +181,8 @@ class BatchExpansionTop1Scorer(SpeculativeScorer):
181
spec_expanded_bs, k + 1)
182
target_probs = target_probs.squeeze().reshape(spec_expanded_bs, k + 1,
183
self._vocab_size)
184
+ target_logprobs = target_logprobs.squeeze().reshape(
185
+ spec_expanded_bs, k + 1, self._vocab_size)
186
187
all_tokens = torch.full(size=(contracted_bs, k + 1),
188
fill_value=-1,
@@ -189,16 +193,26 @@ class BatchExpansionTop1Scorer(SpeculativeScorer):
193
self._vocab_size,
194
device=self._device,
195
dtype=torch.float32)
196
+ all_logprobs = torch.full(size=(
197
+ contracted_bs,
198
+ k + 1,
199
+ self._vocab_size,
200
+ ),
201
+ fill_value=-float("inf"),
202
+ device=self._device,
203
+ dtype=torch.float32)
204
205
if non_spec_indices:
206
all_tokens[non_spec_indices, :1] = non_spec_target_token_ids
207
all_probs[non_spec_indices, :1, :] = non_spec_target_probs
208
+ all_logprobs[non_spec_indices, :1, :] = non_spec_target_logprobs
209
210
if spec_indices:
211
all_tokens[spec_indices] = target_token_ids
212
all_probs[spec_indices] = target_probs
213
+ all_logprobs[spec_indices] = target_logprobs
214
201
- return all_tokens, all_probs
215
+ return all_tokens, all_probs, all_logprobs
216
217
def _create_scoring_model_input(
218
self,
@@ -308,7 +322,8 @@ class BatchExpansionTop1Scorer(SpeculativeScorer):
322
323
def _split_scoring_output(
324
self, sampler_output: SamplerOutput, num_scoring_tokens: int
311
- ) -> Tuple[torch.Tensor, torch.Tensor, torch.Tensor, torch.Tensor]:
325
+ ) -> Tuple[torch.Tensor, torch.Tensor, torch.Tensor, torch.Tensor,
326
+ torch.Tensor, torch.Tensor]:
327
"""Split the target model output into speculative and non-speculative
328
output.
329
"""
@@ -328,21 +343,29 @@ class BatchExpansionTop1Scorer(SpeculativeScorer):
343
) = sampler_output.sampled_token_probs.split(split_sizes)
344
(spec_sampled_tokens, non_spec_sampled_tokens
345
) = sampler_output.sampled_token_ids.flatten().split(split_sizes)
346
+ (
347
+ spec_logprobs,
348
+ non_spec_logprobs,
349
+ ) = sampler_output.logprobs.split(split_sizes)
350
351
# Convert scores to tensors.
352
sampler_output.sampled_token_probs = spec_probs
353
sampler_output.sampled_token_ids = spec_sampled_tokens
335
- target_token_ids, target_probs = sampler_output_to_torch(
336
- [sampler_output], True)
354
+ sampler_output.logprobs = spec_logprobs
355
+ (target_token_ids, target_probs,
356
+ target_logprobs) = sampler_output_to_torch([sampler_output], True)
357
358
# Convert non-speculative output tokens to tensors.
359
sampler_output.sampled_token_probs = non_spec_probs
360
sampler_output.sampled_token_ids = non_spec_sampled_tokens
341
- non_spec_target_token_ids, non_spec_target_probs = (
342
- sampler_output_to_torch([sampler_output], True))
343
-
344
- return (target_token_ids, target_probs, non_spec_target_token_ids,
345
- non_spec_target_probs)
361
+ sampler_output.logprobs = non_spec_logprobs
362
+ (non_spec_target_token_ids, non_spec_target_probs,
363
+ non_spec_target_logprobs) = sampler_output_to_torch([sampler_output],
364
+ True)
365
+
366
+ return (target_token_ids, target_probs, target_logprobs,
367
+ non_spec_target_token_ids, non_spec_target_probs,
368
+ non_spec_target_logprobs)
369
370
def _create_target_seq_id_iterator(
371
self, seq_ids: List[SeqId]) -> Iterator[TargetSeqId]:
vllm/spec_decode/interfaces.py
+5
@@ -38,6 +38,11 @@ class SpeculativeScores:
38
# Probabilities of the speculative tokens according to the scoring model.
39
probs: torch.Tensor
40
41
+ # Log-probabilities of the speculative tokens according to the scoring
42
+ # model. These values can be used to generate Logprob objects that are
43
+ # returned to the user.
44
+ logprobs: torch.Tensor
45
+
46
# Token ids sampled from the scoring model. Used for speculative bonus
47
# tokens and also non-speculative normal decoding.
48
token_ids: torch.Tensor
vllm/spec_decode/ngram_worker.py
+6
@@ -140,11 +140,17 @@ class NGramWorker(LoraNotSupportedWorkerBase):
140
device=self.device,
141
)
142
token_probs.scatter_(2, indices, 1)
143
+ token_logprobs = torch.zeros(
144
+ (len(seq_group_metadata_list), sample_len, self.vocab_size),
145
+ dtype=torch.float32,
146
+ device=self.device,
147
+ )
148
for i in range(len(seq_group_metadata_list)):
149
outputs.append(
150
SamplerOutput(
151
outputs=None,
152
sampled_token_probs=token_probs[i],
153
+ logprobs=token_logprobs,
154
sampled_token_ids=token_ids[i],
155
))
156
return outputs, False
vllm/spec_decode/spec_decode_worker.py
+74
-26
@@ -5,15 +5,16 @@ import torch
5
6
from vllm.logger import init_logger
7
from vllm.model_executor.layers.rejection_sampler import RejectionSampler
8
-from vllm.sequence import (Logprob, SamplerOutput, SequenceGroupMetadata,
9
- SequenceGroupOutput, SequenceOutput)
8
+from vllm.sequence import SamplerOutput, SequenceGroupMetadata
9
from vllm.spec_decode.batch_expansion import BatchExpansionTop1Scorer
10
from vllm.spec_decode.interfaces import (SpeculativeProposals,
11
SpeculativeScorer, SpeculativeScores)
12
from vllm.spec_decode.metrics import AsyncMetricsCollector
13
from vllm.spec_decode.multi_step_worker import MultiStepWorker
14
from vllm.spec_decode.ngram_worker import NGramWorker
16
-from vllm.spec_decode.util import (get_all_seq_ids, nvtx_range,
15
+from vllm.spec_decode.util import (create_sequence_group_output,
16
+ get_all_num_logprobs, get_all_seq_ids,
17
+ get_sampled_token_logprobs, nvtx_range,
18
split_batch_by_proposal_len)
19
from vllm.worker.worker_base import LoraNotSupportedWorkerBase, WorkerBase
20
@@ -258,6 +259,7 @@ class SpecDecodeWorker(LoraNotSupportedWorkerBase):
259
# overhead when the engine runs in a different process than the workers.
260
sampler_output.probs = None
261
sampler_output.sampled_tokens = None
262
+ sampler_output.logprobs = None
263
return [sampler_output]
264
265
@nvtx_range("spec_decode_worker._run_speculative_decoding_step")
@@ -298,12 +300,15 @@ class SpecDecodeWorker(LoraNotSupportedWorkerBase):
300
)
301
302
#logger.info("verify proposals")
301
- accepted_token_ids = self._verify_tokens(seq_group_metadata_list,
302
- proposal_scores, proposals, k)
303
+ accepted_token_ids, target_logprobs = self._verify_tokens(
304
+ seq_group_metadata_list, proposal_scores, proposals, k)
305
306
#logger.info("create output list")
305
- return self._create_output_sampler_list(seq_group_metadata_list,
306
- accepted_token_ids, k)
307
+ return self._create_output_sampler_list(
308
+ seq_group_metadata_list,
309
+ accepted_token_ids,
310
+ target_logprobs=target_logprobs,
311
+ k=k)
312
313
@nvtx_range("spec_decode_worker._verify_tokens")
314
def _verify_tokens(
@@ -312,9 +317,12 @@ class SpecDecodeWorker(LoraNotSupportedWorkerBase):
317
proposal_scores: SpeculativeScores,
318
proposals: SpeculativeProposals,
319
max_proposal_len: int,
315
- ) -> torch.Tensor:
320
+ ) -> Tuple[torch.Tensor, torch.Tensor]:
321
"""Determine which speculative tokens are accepted using the
322
probabilities of each token according to the proposer and scorer models.
323
+
324
+ Returns a tuple of Tensors, one for the accepted token ids and one for
325
+ the logprobs according to the scoring model.
326
"""
327
proposal_lens_list = proposals.proposal_lens.tolist()
328
@@ -361,17 +369,19 @@ class SpecDecodeWorker(LoraNotSupportedWorkerBase):
369
non_spec_token_ids[:, 1:] = -1
370
accepted_token_ids = torch.cat(
371
[accepted_token_ids, non_spec_token_ids])
372
+ logprobs = proposal_scores.logprobs
373
374
# Rearrange so that results are in the order of the original seq group
375
# metadata.
376
accepted_token_ids[original_indices] = accepted_token_ids.clone()
377
369
- return accepted_token_ids
378
+ return accepted_token_ids, logprobs
379
380
def _create_output_sampler_list(
381
self,
382
seq_group_metadata_list: List[SequenceGroupMetadata],
383
accepted_token_ids: torch.Tensor, # shape: [batch_size, k+1]
384
+ target_logprobs: torch.Tensor, # shape: [batch_size, k+1, vocab_size]
385
k: int,
386
) -> List[SamplerOutput]:
387
"""Given the accepted token ids, create a list of SamplerOutput.
@@ -379,30 +389,68 @@ class SpecDecodeWorker(LoraNotSupportedWorkerBase):
389
The output is padded with -1 tokens such that each sequence has
390
the same number of outputs.
391
"""
392
+ batch_size, num_steps = accepted_token_ids.shape
393
+
394
+ # Organize input tensors by step instead of by sequence.
395
+ target_logprobs_by_step = target_logprobs.transpose(0, 1)
396
+ accepted_token_ids_by_step = accepted_token_ids.transpose(0, 1)
397
+
398
+ # Get the logprobs/rank of the accepted tokens.
399
+ (accepted_token_id_ranks_by_step,
400
+ accepted_token_id_logprobs_by_step) = get_sampled_token_logprobs(
401
+ logprob_tensor=target_logprobs_by_step,
402
+ sampled_token_ids=accepted_token_ids_by_step,
403
+ )
404
+
405
+ # Get the top-k logprobs (which may or may not include the logprob of
406
+ # the accepted token).
407
+ (topk_logprobs_by_step,
408
+ topk_indices_by_step) = target_logprobs_by_step.topk(
409
+ k=self.scorer_worker.model_config.max_logprobs,
410
+ dim=-1,
411
+ )
412
+
413
+ # Get the sequence ids and num_logprobs (sampling parameter) in the
414
+ # batch.
415
seq_ids = get_all_seq_ids(seq_group_metadata_list)
383
-
384
- # shape: [k+1, batch_size]
385
- accepted_token_ids_by_step = accepted_token_ids.transpose(0,
386
- 1).tolist()
416
+ num_logprobs_per_seq = get_all_num_logprobs(seq_group_metadata_list)
417
+
418
+ # Serialize all tensors to CPU Python lists.
419
+ accepted_token_ids_by_step = accepted_token_ids_by_step.tolist()
420
+ accepted_token_id_ranks_by_step = (
421
+ accepted_token_id_ranks_by_step.tolist())
422
+ accepted_token_id_logprobs_by_step = (
423
+ accepted_token_id_logprobs_by_step.tolist())
424
+ topk_logprobs_by_step = topk_logprobs_by_step.tolist()
425
+ topk_indices_by_step = topk_indices_by_step.tolist()
426
+
427
+ # Construct the output on a per-step, per-sequence basis.
428
sampler_output_list = []
388
- for token_ids_by_step in accepted_token_ids_by_step:
389
- if all(token_id == -1 for token_id in token_ids_by_step):
429
+ for step_index in range(num_steps):
430
+ if all(token_id == -1
431
+ for token_id in accepted_token_ids_by_step[step_index]):
432
break
433
434
step_output_token_ids = []
393
- for token_id, seq_id in zip(token_ids_by_step, seq_ids):
435
+ for sequence_index in range(batch_size):
436
+ # Each sequence may have a different num_logprobs; retrieve it.
437
+ num_logprobs = num_logprobs_per_seq[sequence_index]
438
+
439
step_output_token_ids.append(
395
- SequenceGroupOutput(
396
- samples=[
397
- SequenceOutput(
398
- parent_seq_id=seq_id,
399
- output_token=token_id,
400
- # TODO Add verifier logprobs.
401
- logprobs={token_id: Logprob(0.0)},
402
- )
403
- ],
404
- prompt_logprobs=None,
440
+ create_sequence_group_output(
441
+ token_id=accepted_token_ids_by_step[step_index]
442
+ [sequence_index],
443
+ token_id_logprob_rank=accepted_token_id_ranks_by_step[
444
+ step_index][sequence_index],
445
+ token_id_logprob=accepted_token_id_logprobs_by_step[
446
+ step_index][sequence_index],
447
+ seq_id=seq_ids[sequence_index],
448
+ topk_token_ids=topk_indices_by_step[step_index]
449
+ [sequence_index][:num_logprobs],
450
+ topk_logprobs=topk_logprobs_by_step[step_index]
451
+ [sequence_index][:num_logprobs],
452
))
453
+
454
sampler_output_list.append(
455
SamplerOutput(outputs=step_output_token_ids))
456
vllm/spec_decode/top1_proposer.py
+1
-1
@@ -166,7 +166,7 @@ class Top1Proposer(SpeculativeProposer):
166
return proposal_tokens, proposal_probs, proposal_lens_tensor
167
168
sampler_output = maybe_sampler_output
169
- proposal_tokens, proposal_probs = sampler_output_to_torch(
169
+ proposal_tokens, proposal_probs, _ = sampler_output_to_torch(
170
sampler_output, sampler_transposed)
171
172
# Now, reformat the output GPU tensors such that each sequence has
vllm/spec_decode/util.py
+98
-5
@@ -1,10 +1,11 @@
1
from contextlib import contextmanager
2
from itertools import chain
3
-from typing import List, Tuple
3
+from typing import Dict, List, Tuple
4
5
import torch
6
7
-from vllm.sequence import SamplerOutput, SequenceGroupMetadata
7
+from vllm.sequence import (Logprob, SamplerOutput, SequenceGroupMetadata,
8
+ SequenceGroupOutput, SequenceOutput)
9
10
SeqId = int
11
@@ -21,6 +22,89 @@ def get_all_seq_ids(
22
]))
23
24
25
+def get_all_num_logprobs(
26
+ seq_group_metadata_list: List[SequenceGroupMetadata]) -> List[int]:
27
+ """Given a list of SequenceGroupMetadata, create a list of all num_logprobs.
28
+
29
+ If the sampling params do not call for any logprobs, return 0 for that
30
+ sequence.
31
+ """
32
+
33
+ all_num_logprobs = []
34
+ for seq_group_metadata in seq_group_metadata_list:
35
+ num_logprobs = seq_group_metadata.sampling_params.logprobs
36
+ if seq_group_metadata.sampling_params.logprobs is None:
37
+ num_logprobs = 0
38
+ all_num_logprobs.append(num_logprobs)
39
+
40
+ return all_num_logprobs
41
+
42
+
43
+def get_sampled_token_logprobs(
44
+ # shape [num_steps, batch_size, vocab_size]
45
+ logprob_tensor: torch.Tensor,
46
+ sampled_token_ids: torch.Tensor, # shape [num_steps, batch_size]
47
+) -> Tuple[torch.Tensor, torch.Tensor]:
48
+ """Get the logprobs for the sampled tokens. Returns the ranks and logprobs.
49
+ """
50
+ num_steps, batch_size, vocab_size = logprob_tensor.shape
51
+
52
+ selected_logprobs = logprob_tensor[torch.arange(num_steps).unsqueeze(1),
53
+ torch.arange(batch_size),
54
+ sampled_token_ids, ]
55
+ expanded_selected_logprobs = selected_logprobs.unsqueeze(-1).expand(
56
+ -1, -1, vocab_size)
57
+ sampled_token_ids_ranks = (logprob_tensor >=
58
+ expanded_selected_logprobs).sum(-1)
59
+
60
+ return sampled_token_ids_ranks, selected_logprobs
61
+
62
+
63
+def create_sequence_group_output(
64
+ token_id: int,
65
+ token_id_logprob_rank: int,
66
+ token_id_logprob: float,
67
+ seq_id: SeqId,
68
+ topk_token_ids: List[int],
69
+ topk_logprobs: List[float],
70
+) -> SequenceGroupOutput:
71
+ """Create a SequenceGroupOutput given the sampling results.
72
+
73
+ Args:
74
+ token_id (int): The sampled token for the sequence.
75
+ token_id_logprob_rank (int): The logprob rank of the sampled token.
76
+ token_id_logprob (float): The logprob value of the sampled token.
77
+ seq_id (int): The sequence id.
78
+ topk_token_ids (List[int]): The list of top-k token ids.
79
+ topk_logprobs (List[float]): The list of top-k logprobs.
80
+ """
81
+ # vLLM logprobs always include the sampled token. In addition, the user may
82
+ # request topk-logprobs (where top-k varies per user up to max_logprobs).
83
+ logprobs: Dict[int, Logprob] = {
84
+ token_id: Logprob(
85
+ logprob=token_id_logprob,
86
+ rank=token_id_logprob_rank,
87
+ ),
88
+ }
89
+ logprobs.update({
90
+ topk_token_ids[topk_logprob_index]: Logprob(
91
+ logprob=topk_logprobs[topk_logprob_index],
92
+ rank=topk_logprob_index + 1,
93
+ )
94
+ for topk_logprob_index, _ in enumerate(topk_token_ids)
95
+ })
96
+
97
+ return SequenceGroupOutput(
98
+ samples=[
99
+ SequenceOutput(parent_seq_id=seq_id,
100
+ output_token=token_id,
101
+ logprobs=logprobs)
102
+ ],
103
+ # TODO add prompt logprobs support.
104
+ prompt_logprobs=None,
105
+ )
106
+
107
+
108
def split_batch_by_proposal_len(
109
seq_group_metadata_list: List[SequenceGroupMetadata],
110
proposal_lens: List[int], select_proposal_len_zero: bool
@@ -49,8 +133,8 @@ def split_batch_by_proposal_len(
133
134
135
def sampler_output_to_torch(
52
- sampler_output_list: List[SamplerOutput],
53
- sampler_transposed: bool) -> Tuple[torch.Tensor, torch.Tensor]:
136
+ sampler_output_list: List[SamplerOutput], sampler_transposed: bool
137
+) -> Tuple[torch.Tensor, torch.Tensor, torch.Tensor]:
138
"""Utility function which converts a list of SamplerOutput to tensors.
139
140
sampler_transposed here is used as the indicator for whether
@@ -76,6 +160,15 @@ def sampler_output_to_torch(
160
if sampler_transposed:
161
sampled_token_probs = sampled_token_probs.transpose(0, 1)
162
163
+ # shape: [batch_size, num_sampler_output, vocab_size]
164
+ sampled_token_logprobs = torch.stack(
165
+ [sampler_output.logprobs for sampler_output in sampler_output_list],
166
+ dim=0,
167
+ )
168
+
169
+ if sampler_transposed:
170
+ sampled_token_logprobs = sampled_token_logprobs.transpose(0, 1)
171
+
172
# shape: [batch_size, num_sampler_output]
173
sampled_token_ids = torch.stack(
174
[
@@ -87,7 +180,7 @@ def sampler_output_to_torch(
180
if sampler_transposed:
181
sampled_token_ids = sampled_token_ids.transpose(0, 1)
182
90
- return sampled_token_ids, sampled_token_probs
183
+ return sampled_token_ids, sampled_token_probs, sampled_token_logprobs
184
185
186
def maybe_mock_device_tensors(sampler_output: SamplerOutput, batch_size: int,