@tduncan / Nvidia-Optimization-Stack-VLLM Mirror · read-only / commits / ab50275111
Fork 0

A high-throughput and memory-efficient inference and serving engine for LLMs

Mirrored from https://github.com/tduncan-lum/Nvidia-Optimization-Stack---VLLM.git

Synced Nov 17, 01:45 UTC

[Speculative decoding] Support target-model logprobs (#4378)

Cade Daniel committed May 3, 2024 at 15:52 UTC ab502751117d3785384b9c33ee88e0aff93bbf05
15 files changed +728 -87
tests/spec_decode/e2e/conftest.py
+63 -3
@@ -1,9 +1,13 @@
1 import asyncio
2 +import time
3 from itertools import cycle
3 -from typing import List, Optional, Tuple, Union
4 +from typing import Dict, List, Optional, Tuple, Union
5
6 import pytest
7 import ray
8 +import torch
9 +from pynvml import (nvmlDeviceGetHandleByIndex, nvmlDeviceGetMemoryInfo,
10 + nvmlInit)
11
12 from tests.conftest import cleanup
13 from vllm import LLM
@@ -13,7 +17,7 @@ from vllm.lora.request import LoRARequest
17 from vllm.model_executor.utils import set_random_seed
18 from vllm.outputs import RequestOutput
19 from vllm.sampling_params import SamplingParams
16 -from vllm.sequence import MultiModalData
20 +from vllm.sequence import Logprob, MultiModalData
21 from vllm.usage.usage_lib import UsageContext
22 from vllm.utils import Counter, random_uuid
23
@@ -153,12 +157,19 @@ def create_llm_generator(baseline_or_test, request, common_llm_kwargs,
157 test_name = request.node.name
158
159 def generator_inner():
156 - print(f'Creating {baseline_or_test=} LLM for {test_name=}. {kwargs=}')
160 +
161 + wait_for_gpu_memory_to_clear(
162 + devices=list(range(torch.cuda.device_count())),
163 + threshold_bytes=2 * 2**30,
164 + timeout_s=60,
165 + )
166
167 use_async = False
168 if "use_async" in kwargs:
169 use_async = kwargs.pop("use_async")
170 + print(f'{use_async=}')
171
172 + print(f'Creating {baseline_or_test=} LLM for {test_name=}. {kwargs=}')
173 llm = AsyncLLM(**kwargs) if use_async else LLM(**kwargs)
174 set_random_seed(seed)
175
@@ -188,6 +199,20 @@ def get_output_from_llm_generator(
199 return tokens, token_ids
200
201
202 +def get_logprobs_from_llm_generator(
203 + llm_generator, prompts,
204 + sampling_params) -> List[List[Dict[int, Logprob]]]:
205 + """Returns a dict of (token_id: Logprob) for each generated position, for
206 + each sequence in the batch.
207 + """
208 + for llm in llm_generator():
209 + outputs = llm.generate(prompts, sampling_params, use_tqdm=True)
210 + logprobs = [output.outputs[0].logprobs[:] for output in outputs]
211 + del llm
212 +
213 + return logprobs
214 +
215 +
216 def run_greedy_equality_correctness_test(baseline_llm_generator,
217 test_llm_generator,
218 batch_size,
@@ -243,3 +268,38 @@ def run_greedy_equality_correctness_test(baseline_llm_generator,
268 print(f'{i=} {baseline_token_ids=}')
269 print(f'{i=} {spec_token_ids=}')
270 assert baseline_token_ids == spec_token_ids
271 +
272 +
273 +def wait_for_gpu_memory_to_clear(devices: List[int],
274 + threshold_bytes: int,
275 + timeout_s: float = 120) -> None:
276 + # Use nvml instead of pytorch to reduce measurement error from torch cuda
277 + # context.
278 + nvmlInit()
279 + start_time = time.time()
280 + while True:
281 + output = {}
282 + output_raw = {}
283 + for device in devices:
284 + dev_handle = nvmlDeviceGetHandleByIndex(device)
285 + mem_info = nvmlDeviceGetMemoryInfo(dev_handle)
286 + gb_used = mem_info.used / 2**30
287 + output_raw[device] = gb_used
288 + output[device] = f'{gb_used:.02f}'
289 +
290 + print('gpu memory used (GB): ', end='')
291 + for k, v in output.items():
292 + print(f'{k}={v}; ', end='')
293 + print('')
294 +
295 + dur_s = time.time() - start_time
296 + if all(v <= (threshold_bytes / 2**30) for v in output_raw.values()):
297 + print(f'Done waiting for free GPU memory on devices {devices=} '
298 + f'({threshold_bytes/2**30=}) {dur_s=:.02f}')
299 + break
300 +
301 + if dur_s >= timeout_s:
302 + raise ValueError(f'Memory of devices {devices=} not free after '
303 + f'{dur_s=:.02f} ({threshold_bytes/2**30=})')
304 +
305 + time.sleep(5)
tests/spec_decode/e2e/test_logprobs.py new
+335
@@ -0,0 +1,335 @@
1 +import math
2 +from itertools import cycle
3 +
4 +import pytest
5 +
6 +from vllm import SamplingParams
7 +
8 +from .conftest import get_logprobs_from_llm_generator
9 +
10 +
11 +@pytest.mark.parametrize(
12 + "common_llm_kwargs",
13 + [{
14 + "model": "JackFram/llama-68m",
15 +
16 + # Skip cuda graph recording for fast test.
17 + "enforce_eager": True,
18 +
19 + # Required for spec decode.
20 + "use_v2_block_manager": True,
21 + "max_logprobs": 6,
22 + }])
23 +@pytest.mark.parametrize("per_test_common_llm_kwargs", [{}])
24 +@pytest.mark.parametrize("baseline_llm_kwargs", [{}])
25 +@pytest.mark.parametrize("test_llm_kwargs", [{
26 + "speculative_model": "JackFram/llama-160m",
27 + "num_speculative_tokens": 3,
28 +}])
29 +@pytest.mark.parametrize("batch_size", [8])
30 +@pytest.mark.parametrize(
31 + "output_len",
32 + [
33 + # Use smaller output len for fast test.
34 + 7,
35 + ])
36 +@pytest.mark.parametrize("seed", [1])
37 +def test_logprobs_equality(baseline_llm_generator, test_llm_generator,
38 + batch_size: int, output_len: int):
39 + """Verify output logprobs are equal with and without speculative decoding.
40 + """
41 + run_greedy_logprobs_correctness_test(baseline_llm_generator,
42 + test_llm_generator,
43 + batch_size,
44 + max_output_len=output_len,
45 + force_output_len=True)
46 +
47 +
48 +@pytest.mark.parametrize(
49 + "common_llm_kwargs",
50 + [{
51 + "model": "JackFram/llama-68m",
52 +
53 + # Skip cuda graph recording for fast test.
54 + "enforce_eager": True,
55 +
56 + # Required for spec decode.
57 + "use_v2_block_manager": True,
58 + "max_logprobs": 6,
59 + }])
60 +@pytest.mark.parametrize("per_test_common_llm_kwargs", [{}])
61 +@pytest.mark.parametrize("baseline_llm_kwargs", [{}])
62 +@pytest.mark.parametrize("test_llm_kwargs", [{
63 + "speculative_model": "JackFram/llama-160m",
64 + "num_speculative_tokens": 3,
65 +}])
66 +@pytest.mark.parametrize("batch_size", [1])
67 +@pytest.mark.parametrize("num_logprobs", [6])
68 +@pytest.mark.parametrize(
69 + "output_len",
70 + [
71 + # Use smaller output len for fast test.
72 + 7,
73 + ])
74 +@pytest.mark.parametrize("seed", [1])
75 +def test_diff_num_logprobs(baseline_llm_generator, test_llm_generator,
76 + batch_size: int, output_len: int,
77 + num_logprobs: int):
78 + """Verify output logprobs are equal with and without spec decode.
79 + This specifies a number of logprobs >1.
80 + """
81 + run_greedy_logprobs_correctness_test(baseline_llm_generator,
82 + test_llm_generator,
83 + batch_size,
84 + max_output_len=output_len,
85 + force_output_len=True,
86 + logprob_rank=num_logprobs)
87 +
88 +
89 +@pytest.mark.parametrize(
90 + "common_llm_kwargs",
91 + [{
92 + "model": "JackFram/llama-68m",
93 +
94 + # Skip cuda graph recording for fast test.
95 + "enforce_eager": True,
96 +
97 + # Required for spec decode.
98 + "use_v2_block_manager": True
99 + }])
100 +@pytest.mark.parametrize("per_test_common_llm_kwargs", [{}])
101 +@pytest.mark.parametrize("baseline_llm_kwargs", [{}])
102 +@pytest.mark.parametrize("test_llm_kwargs", [{
103 + "speculative_model": "JackFram/llama-160m",
104 + "num_speculative_tokens": 3,
105 +}, {
106 + "speculative_model": "JackFram/llama-160m",
107 + "num_speculative_tokens": 6,
108 +}])
109 +@pytest.mark.parametrize("batch_size", [8])
110 +@pytest.mark.parametrize(
111 + "output_len",
112 + [
113 + # Use smaller output len for fast test.
114 + 32,
115 + ])
116 +@pytest.mark.parametrize("seed", [1])
117 +def test_logprobs_different_k(baseline_llm_generator, test_llm_generator,
118 + batch_size: int, output_len: int):
119 + """Veriy logprob greedy equality with different speculation lens.
120 + """
121 + run_greedy_logprobs_correctness_test(baseline_llm_generator,
122 + test_llm_generator,
123 + batch_size,
124 + max_output_len=output_len,
125 + force_output_len=True)
126 +
127 +
128 +@pytest.mark.parametrize(
129 + "common_llm_kwargs",
130 + [{
131 + "model": "JackFram/llama-68m",
132 +
133 + # Skip cuda graph recording for fast test.
134 + "enforce_eager": True,
135 +
136 + # Required for spec decode.
137 + "use_v2_block_manager": True
138 + }])
139 +@pytest.mark.parametrize("per_test_common_llm_kwargs", [{}])
140 +@pytest.mark.parametrize("baseline_llm_kwargs", [{}])
141 +@pytest.mark.parametrize(
142 + "test_llm_kwargs",
143 + [{
144 + "speculative_model": "JackFram/llama-160m",
145 + "num_speculative_tokens": 3,
146 +
147 + # Artificially limit the draft model max model len; this forces vLLM
148 + # to skip speculation once the sequences grow beyond 32-k tokens.
149 + "speculative_max_model_len": 32,
150 + }])
151 +@pytest.mark.parametrize("batch_size", [8])
152 +@pytest.mark.parametrize(
153 + "output_len",
154 + [
155 + # Use smaller output len for fast test.
156 + 32,
157 + ])
158 +@pytest.mark.parametrize("seed", [1])
159 +def test_logprobs_when_skip_speculation(baseline_llm_generator,
160 + test_llm_generator, batch_size: int,
161 + output_len: int):
162 + """Verify logprobs greedy equality when some sequences skip speculation.
163 + """
164 + run_greedy_logprobs_correctness_test(baseline_llm_generator,
165 + test_llm_generator,
166 + batch_size,
167 + max_output_len=output_len,
168 + force_output_len=True)
169 +
170 +
171 +@pytest.mark.parametrize(
172 + "common_llm_kwargs",
173 + [{
174 + "model": "JackFram/llama-68m",
175 +
176 + # Skip cuda graph recording for fast test.
177 + "enforce_eager": True,
178 +
179 + # Required for spec decode.
180 + "use_v2_block_manager": True
181 + }])
182 +@pytest.mark.parametrize("per_test_common_llm_kwargs", [{}])
183 +@pytest.mark.parametrize("baseline_llm_kwargs", [{}])
184 +@pytest.mark.parametrize("test_llm_kwargs", [{
185 + "speculative_model": "JackFram/llama-160m",
186 + "num_speculative_tokens": 3,
187 +}])
188 +@pytest.mark.parametrize("batch_size", [1])
189 +@pytest.mark.parametrize(
190 + "output_len",
191 + [
192 + # Use smaller output len for fast test.
193 + 32,
194 + ])
195 +@pytest.mark.parametrize("seed", [1])
196 +def test_logprobs_temp_1(baseline_llm_generator, test_llm_generator,
197 + batch_size: int, output_len: int):
198 + """Verify at least one logprob result has num_logprobs+1, which tests the
199 + case where the sampled token is not in top-k logprobs.
200 +
201 + Ideally, this test should validate equality with non-spec by getting
202 + logprobs. This is left as future improvement.
203 + """
204 + batch_size = 8
205 + max_output_len = output_len
206 + force_output_len = True
207 + logprob_rank = 5
208 +
209 + temperature = 1.0
210 +
211 + prompts = [
212 + "Hello, my name is",
213 + "The president of the United States is",
214 + "The capital of France is",
215 + "The future of AI is",
216 + "San Francisco is know for its",
217 + "Facebook was created in 2004 by",
218 + "Curious George is a",
219 + "Python 3.11 brings improvements to its",
220 + ]
221 +
222 + prompts = [prompt for prompt, _ in zip(cycle(prompts), range(batch_size))]
223 +
224 + # If the test requires that we generated max_output_len tokens, then set the
225 + # sampling params to ignore eos token.
226 + ignore_eos = force_output_len
227 +
228 + sampling_params = SamplingParams(
229 + max_tokens=max_output_len,
230 + ignore_eos=ignore_eos,
231 + temperature=temperature,
232 + logprobs=logprob_rank,
233 + )
234 +
235 + spec_batch_logprobs = get_logprobs_from_llm_generator(
236 + test_llm_generator, prompts, sampling_params)
237 +
238 + num_returned_logprobs = [
239 + len(logprob_dict) for seq_logprobs in spec_batch_logprobs
240 + for logprob_dict in seq_logprobs
241 + ]
242 +
243 + # Assert one of the returned logprobs has > num_logprobs (indicating the
244 + # sampled token is not in top-k).
245 + assert any([
246 + num_returned > logprob_rank for num_returned in num_returned_logprobs
247 + ])
248 +
249 +
250 +def run_greedy_logprobs_correctness_test(baseline_llm_generator,
251 + test_llm_generator,
252 + batch_size,
253 + max_output_len,
254 + force_output_len: bool,
255 + logprob_rank: int = 1):
256 + """Helper method that compares the logprobs outputs of both the baseline LLM
257 + and the test LLM. It asserts greedy equality of the logprobs when the
258 + temperature is zero.
259 + """
260 + temperature = 0.0
261 +
262 + prompts = [
263 + "Hello, my name is",
264 + "The president of the United States is",
265 + "The capital of France is",
266 + "The future of AI is",
267 + "San Francisco is know for its",
268 + "Facebook was created in 2004 by",
269 + "Curious George is a",
270 + "Python 3.11 brings improvements to its",
271 + ]
272 +
273 + prompts = [prompt for prompt, _ in zip(cycle(prompts), range(batch_size))]
274 +
275 + # If the test requires that we generated max_output_len tokens, then set the
276 + # sampling params to ignore eos token.
277 + ignore_eos = force_output_len
278 +
279 + sampling_params = SamplingParams(
280 + max_tokens=max_output_len,
281 + ignore_eos=ignore_eos,
282 + temperature=temperature,
283 + logprobs=logprob_rank,
284 + )
285 +
286 + spec_batch_logprobs = get_logprobs_from_llm_generator(
287 + test_llm_generator, prompts, sampling_params)
288 + baseline_batch_logprobs = get_logprobs_from_llm_generator(
289 + baseline_llm_generator, prompts, sampling_params)
290 +
291 + assert len(baseline_batch_logprobs) == len(prompts)
292 + assert len(spec_batch_logprobs) == len(prompts)
293 +
294 + # For each sequence in the batch.
295 + for i, (baseline_logprobs, spec_logprobs) in enumerate(
296 + zip(baseline_batch_logprobs, spec_batch_logprobs)):
297 + assert len(spec_logprobs) == len(baseline_logprobs)
298 +
299 + # For each generated position of the sequence.
300 + for pos, (spec_pos_logprobs, baseline_pos_logprobs) in enumerate(
301 + zip(spec_logprobs, baseline_logprobs)):
302 +
303 + # Map rank to token/logprob in spec output.
304 + spec_rank_to_token_id = {
305 + value.rank: key
306 + for key, value in spec_pos_logprobs.items()
307 + }
308 + spec_rank_to_logprob = {
309 + value.rank: value.logprob
310 + for key, value in spec_pos_logprobs.items()
311 + }
312 +
313 + # Map rank to token/logprob in baseline output.
314 + baseline_rank_to_token_id = {
315 + value.rank: key
316 + for key, value in baseline_pos_logprobs.items()
317 + }
318 + baseline_rank_to_logprob = {
319 + value.rank: value.logprob
320 + for key, value in baseline_pos_logprobs.items()
321 + }
322 +
323 + # Assert set of ranks returned is equal.
324 + assert set(spec_rank_to_token_id.keys()) == set(
325 + baseline_rank_to_token_id.keys())
326 +
327 + # Assert each logprob/token id is correct, keyed by rank.
328 + for rank in sorted(set(spec_rank_to_token_id.keys())):
329 + assert spec_rank_to_token_id[
330 + rank] == baseline_rank_to_token_id[rank], f"{rank}"
331 + assert math.isclose(
332 + a=spec_rank_to_logprob[rank],
333 + b=baseline_rank_to_logprob[rank],
334 + abs_tol=1e-1,
335 + )
tests/spec_decode/e2e/test_multistep_correctness.py
+47 -16
@@ -41,24 +41,17 @@ from .conftest import (get_output_from_llm_generator,
41
42 @pytest.mark.parametrize(
43 "common_llm_kwargs",
44 - [
45 - {
46 - # Use a small model for a fast test.
47 - # Note this is repeated in the test body; to initialize a tokenizer.
48 - "model": "JackFram/llama-68m",
49 -
50 - # Skip cuda graph recording for fast test.
51 - "enforce_eager": True,
44 + [{
45 + # Use a small model for a fast test.
46 + # Note this is repeated in the test body; to initialize a tokenizer.
47 + "model": "JackFram/llama-68m",
48
53 - # Required for spec decode.
54 - "use_v2_block_manager": True,
49 + # Skip cuda graph recording for fast test.
50 + "enforce_eager": True,
51
56 - # whether use AsyncLLM engine
57 - "use_async": async_mode,
58 - }
59 - # Try both async and sync engine execution
60 - for async_mode in [True, False]
61 - ])
52 + # Required for spec decode.
53 + "use_v2_block_manager": True,
54 + }])
55 @pytest.mark.parametrize(
56 "per_test_common_llm_kwargs",
57 [
@@ -117,6 +110,44 @@ def test_spec_decode_e2e_with_detokenization(test_llm_generator,
110 assert actual_tokens.strip() == expected_tokens.strip()
111
112
113 +@pytest.mark.parametrize(
114 + "common_llm_kwargs",
115 + [{
116 + # Use a small model for a fast test.
117 + # Note this is repeated in the test body; to initialize a tokenizer.
118 + "model": "JackFram/llama-68m",
119 +
120 + # Skip cuda graph recording for fast test.
121 + "enforce_eager": True,
122 +
123 + # Required for spec decode.
124 + "use_v2_block_manager": True,
125 +
126 + # Use AsyncLLM engine
127 + "use_async": True,
128 + }])
129 +@pytest.mark.parametrize("baseline_llm_kwargs", [{}])
130 +@pytest.mark.parametrize("per_test_common_llm_kwargs", [
131 + {
132 + "speculative_model": "JackFram/llama-68m",
133 + "num_speculative_tokens": 5,
134 + },
135 +])
136 +@pytest.mark.parametrize("test_llm_kwargs", [{}])
137 +@pytest.mark.parametrize("batch_size", [2])
138 +@pytest.mark.parametrize("seed", [1])
139 +def test_spec_decode_e2e_with_async_engine(test_llm_generator,
140 + baseline_llm_generator,
141 + batch_size: int):
142 + """Verify spec decode works well with async LLM engine.
143 + """
144 + run_greedy_equality_correctness_test(baseline_llm_generator,
145 + test_llm_generator,
146 + batch_size,
147 + max_output_len=32,
148 + force_output_len=True)
149 +
150 +
151 @pytest.mark.parametrize(
152 "common_llm_kwargs",
153 [{
tests/spec_decode/test_multi_step_worker.py
+8
@@ -292,6 +292,10 @@ def test_draft_proposals_full_speculation_len():
292 vocab_size,
293 device=device,
294 dtype=torch.float32),
295 + logprobs=torch.rand(batch_size,
296 + vocab_size,
297 + device=device,
298 + dtype=torch.float32),
299 sampled_token_ids=torch.randint(low=0,
300 high=vocab_size,
301 size=(batch_size, ),
@@ -392,6 +396,10 @@ def test_draft_proposals_mixed_k():
396 vocab_size,
397 device=device,
398 dtype=torch.float32),
399 + logprobs=torch.rand(expected_num_proposal_seqs,
400 + vocab_size,
401 + device=device,
402 + dtype=torch.float32),
403 sampled_token_ids=torch.randint(
404 low=0,
405 high=vocab_size,
tests/spec_decode/test_spec_decode_worker.py
+24 -5
@@ -192,8 +192,14 @@ def test_correctly_calls_rejection_sampler(k: int, batch_size: int):
192 vocab_size,
193 dtype=torch.float32,
194 device='cuda')
195 + target_token_logprobs = torch.rand(1,
196 + batch_size * (k + 1),
197 + vocab_size,
198 + dtype=torch.float32,
199 + device='cuda')
200 target_output = create_sampler_output_list(target_token_ids,
196 - target_token_probs)
201 + target_token_probs,
202 + target_token_logprobs)
203
204 target_worker.execute_model.return_value = [target_output[0]]
205
@@ -273,8 +279,14 @@ def test_correctly_formats_output(k: int, batch_size: int):
279 vocab_size,
280 dtype=torch.float32,
281 device='cuda')
282 + target_token_logprobs = torch.rand(1,
283 + batch_size * (k + 1),
284 + vocab_size,
285 + dtype=torch.float32,
286 + device='cuda')
287 target_output = create_sampler_output_list(target_token_ids,
277 - target_token_probs)
288 + target_token_probs,
289 + target_token_logprobs)
290
291 target_worker.execute_model.return_value = [target_output[0]]
292
@@ -294,7 +306,9 @@ def test_correctly_formats_output(k: int, batch_size: int):
306 num_lookahead_slots=k)
307
308 expected_output = create_sampler_output_list(
297 - rejection_sampler_output.transpose(0, 1), [None for _ in range(k + 1)])
309 + token_ids=rejection_sampler_output.transpose(0, 1),
310 + probs=[None for _ in range(k + 1)],
311 + logprobs=[None for _ in range(k + 1)])
312
313 seq_ids = [
314 next(iter(seq_group_metadata.seq_data.keys()))
@@ -328,7 +342,6 @@ def test_correctly_formats_output(k: int, batch_size: int):
342 continue
343 assert actual_by_step[i].output_token == expected_by_step[
344 i].output_token
331 - assert actual_by_step[i].logprobs == expected_by_step[i].logprobs
345
346
347 @pytest.mark.parametrize('k', [1, 2])
@@ -387,8 +400,14 @@ def test_collects_metrics(k: int, batch_size: int, returns_metrics: bool):
400 vocab_size,
401 dtype=torch.float32,
402 device='cuda')
403 + target_token_logprobs = torch.rand(1,
404 + batch_size * (k + 1),
405 + vocab_size,
406 + dtype=torch.float32,
407 + device='cuda')
408 target_output = create_sampler_output_list(target_token_ids,
391 - target_token_probs)
409 + target_token_probs,
410 + target_token_logprobs)
411
412 target_worker.execute_model.return_value = [target_output[0]]
413
tests/spec_decode/utils.py
+2
@@ -201,6 +201,7 @@ def assert_logprobs_dict_allclose(
201 def create_sampler_output_list(
202 token_ids: torch.Tensor,
203 probs: Iterable[Optional[torch.Tensor]],
204 + logprobs: Iterable[Optional[torch.Tensor]],
205 seq_ids: Optional[List[int]] = None) -> List[SamplerOutput]:
206 num_steps, batch_size = token_ids.shape
207 token_ids_by_step = token_ids.tolist()
@@ -222,6 +223,7 @@ def create_sampler_output_list(
223 ) for seq_index, token_id in enumerate(token_ids_by_step[step])
224 ],
225 sampled_token_probs=probs[step],
226 + logprobs=logprobs[step],
227 sampled_token_ids=token_ids[step])
228 for step in range(num_steps)
229 ]
vllm/engine/output_processor/multi_step.py
+12 -6
@@ -1,3 +1,4 @@
1 +import functools
2 from typing import Callable, List
3
4 from transformers import PreTrainedTokenizer
@@ -8,8 +9,8 @@ from vllm.engine.output_processor.interfaces import (
9 from vllm.engine.output_processor.stop_checker import StopChecker
10 from vllm.logger import init_logger
11 from vllm.sampling_params import SamplingParams
11 -from vllm.sequence import (Logprob, Sequence, SequenceGroup,
12 - SequenceGroupOutput, SequenceOutput, SequenceStatus)
12 +from vllm.sequence import (Sequence, SequenceGroup, SequenceGroupOutput,
13 + SequenceOutput, SequenceStatus)
14 from vllm.transformers_utils.detokenizer import Detokenizer
15 from vllm.utils import Counter
16
@@ -48,10 +49,14 @@ class MultiStepOutputProcessor(SequenceGroupOutputProcessor):
49 outputs: List[SequenceGroupOutput]) -> None:
50 # TODO(sang): Prompt logprob currently not implemented in multi step
51 # workers.
52 + self._log_prompt_logprob_unsupported_warning_once()
53 +
54 + @staticmethod
55 + @functools.lru_cache()
56 + def _log_prompt_logprob_unsupported_warning_once():
57 logger.warning(
58 "Prompt logprob is not supported by multi step workers. "
59 "(e.g., speculative decode uses multi step workers).")
54 - pass
60
61 def process_outputs(self, sequence_group: SequenceGroup,
62 outputs: List[SequenceGroupOutput]) -> None:
@@ -89,6 +94,7 @@ class MultiStepOutputProcessor(SequenceGroupOutputProcessor):
94 valid_samples: List[SequenceOutput],
95 sampling_params: SamplingParams) -> None:
96 output_token_ids = [sample.output_token for sample in valid_samples]
97 + output_logprobs = [sample.logprobs for sample in valid_samples]
98
99 # Truncate to max_tokens if necessary.
100 remaining_tokens = sampling_params.max_tokens - (seq.get_output_len() +
@@ -113,11 +119,11 @@ class MultiStepOutputProcessor(SequenceGroupOutputProcessor):
119
120 # Incrementally append tokens to the sequence, as if we had only one new
121 # token.
116 - for output_token_id in output_token_ids:
122 + for output_token_id, output_logprob in zip(output_token_ids,
123 + output_logprobs):
124 seq.append_token_id(
125 token_id=output_token_id,
119 - # TODO emit logprobs in multi-step decoding.
120 - logprobs={output_token_id: Logprob(0.0)},
126 + logprobs=output_logprob,
127 )
128
129 new_char_count = 0
vllm/model_executor/layers/sampler.py
+9 -7
@@ -103,8 +103,7 @@ class Sampler(nn.Module):
103
104 if self.include_gpu_probs_tensor:
105 assert maybe_sampled_tokens_tensor is not None
106 - sampled_tokens_tensor = maybe_sampled_tokens_tensor
107 - on_device_tensors = (probs, sampled_tokens_tensor)
106 + on_device_tensors = (probs, logprobs, maybe_sampled_tokens_tensor)
107 else:
108 on_device_tensors = None
109
@@ -965,8 +964,7 @@ def _modify_greedy_probs_inplace(logprobs: torch.Tensor, probs: torch.Tensor,
964 has implications on the overall design of the sampler, e.g. how to record
965 accurate logprobs for the user, so this improvement is deferred to later.
966 """
968 - logprobs[sample_indices, :] = -float('inf')
969 - logprobs[sample_indices, greedy_samples] = 0.0
967 + # NOTE: logprobs are not modified so they can be returned to the user.
968 probs[sample_indices, :] = 0
969 probs[sample_indices, greedy_samples] = 1.0
970
@@ -976,7 +974,8 @@ def _build_sampler_output(
974 sampling_metadata: SamplingMetadata,
975 prompt_logprobs: List[Optional[PromptLogprobs]],
976 sample_logprobs: List[SampleLogprobs],
979 - on_device_tensors: Optional[Tuple[torch.Tensor, torch.Tensor]],
977 + on_device_tensors: Optional[Tuple[torch.Tensor, torch.Tensor,
978 + torch.Tensor]],
979 ) -> SamplerOutput:
980 """Construct Python objects with the output of sampling.
981
@@ -1005,14 +1004,17 @@ def _build_sampler_output(
1004
1005 # If not specified, store None values in SamplerOutput.
1006 if on_device_tensors is not None:
1008 - sampled_token_probs, sampled_token_ids = on_device_tensors
1007 + (sampled_token_probs, logprobs_tensor,
1008 + sampled_token_ids) = on_device_tensors
1009 else:
1010 - sampled_token_probs, sampled_token_ids = (None, None)
1010 + sampled_token_probs, logprobs_tensor, sampled_token_ids = (None, None,
1011 + None)
1012
1013 return SamplerOutput(
1014 outputs=sampler_output,
1015 sampled_token_probs=sampled_token_probs,
1016 sampled_token_ids=sampled_token_ids,
1017 + logprobs=logprobs_tensor,
1018 )
1019
1020
vllm/sequence.py
+3
@@ -700,6 +700,9 @@ class SamplerOutput:
700 # On-device tensor containing probabilities of each token.
701 sampled_token_probs: Optional["torch.Tensor"] = None
702
703 + # On-device tensor containing the logprobs of each token.
704 + logprobs: Optional["torch.Tensor"] = None
705 +
706 # On-device tensor containing the sampled token ids.
707 sampled_token_ids: Optional["torch.Tensor"] = None
708
vllm/spec_decode/batch_expansion.py
+41 -18
@@ -94,7 +94,7 @@ class BatchExpansionTop1Scorer(SpeculativeScorer):
94 assert len(target_sampler_output) == 1, "expected single-step output"
95 target_sampler_output = target_sampler_output[0]
96
97 - all_tokens, all_probs = self._contract_batch(
97 + all_tokens, all_probs, spec_logprobs = self._contract_batch(
98 contracted_bs=len(seq_group_metadata_list),
99 target_sampler_output=target_sampler_output,
100 proposals=proposals,
@@ -107,6 +107,7 @@ class BatchExpansionTop1Scorer(SpeculativeScorer):
107 return SpeculativeScores(
108 probs=all_probs,
109 token_ids=all_tokens,
110 + logprobs=spec_logprobs,
111 )
112
113 def _expand_batch(
@@ -148,12 +149,12 @@ class BatchExpansionTop1Scorer(SpeculativeScorer):
149 return (spec_indices, non_spec_indices, target_seq_group_metadata_list,
150 num_scoring_tokens)
151
151 - def _contract_batch(self, contracted_bs: int,
152 - target_sampler_output: List[SamplerOutput],
153 - proposals: SpeculativeProposals,
154 - num_scoring_tokens: int, non_spec_indices: List[int],
155 - spec_indices: List[int],
156 - k: int) -> Tuple[torch.Tensor, torch.Tensor]:
152 + def _contract_batch(
153 + self, contracted_bs: int,
154 + target_sampler_output: List[SamplerOutput],
155 + proposals: SpeculativeProposals, num_scoring_tokens: int,
156 + non_spec_indices: List[int], spec_indices: List[int],
157 + k: int) -> Tuple[torch.Tensor, torch.Tensor, torch.Tensor]:
158 """Contract the expanded batch back into its original size.
159 This maps the scores of speculative tokens back to their original
160 sequences.
@@ -161,8 +162,9 @@ class BatchExpansionTop1Scorer(SpeculativeScorer):
162 contracted_bs is the original batch size, and the batch size that the
163 target_sampler_output will be contracted to.
164 """
164 - (target_token_ids, target_probs, non_spec_target_token_ids,
165 - non_spec_target_probs) = self._split_scoring_output(
165 + (target_token_ids, target_probs, target_logprobs,
166 + non_spec_target_token_ids, non_spec_target_probs,
167 + non_spec_target_logprobs) = self._split_scoring_output(
168 target_sampler_output, num_scoring_tokens)
169
170 # Map distinct sequences used to score each token
@@ -179,6 +181,8 @@ class BatchExpansionTop1Scorer(SpeculativeScorer):
181 spec_expanded_bs, k + 1)
182 target_probs = target_probs.squeeze().reshape(spec_expanded_bs, k + 1,
183 self._vocab_size)
184 + target_logprobs = target_logprobs.squeeze().reshape(
185 + spec_expanded_bs, k + 1, self._vocab_size)
186
187 all_tokens = torch.full(size=(contracted_bs, k + 1),
188 fill_value=-1,
@@ -189,16 +193,26 @@ class BatchExpansionTop1Scorer(SpeculativeScorer):
193 self._vocab_size,
194 device=self._device,
195 dtype=torch.float32)
196 + all_logprobs = torch.full(size=(
197 + contracted_bs,
198 + k + 1,
199 + self._vocab_size,
200 + ),
201 + fill_value=-float("inf"),
202 + device=self._device,
203 + dtype=torch.float32)
204
205 if non_spec_indices:
206 all_tokens[non_spec_indices, :1] = non_spec_target_token_ids
207 all_probs[non_spec_indices, :1, :] = non_spec_target_probs
208 + all_logprobs[non_spec_indices, :1, :] = non_spec_target_logprobs
209
210 if spec_indices:
211 all_tokens[spec_indices] = target_token_ids
212 all_probs[spec_indices] = target_probs
213 + all_logprobs[spec_indices] = target_logprobs
214
201 - return all_tokens, all_probs
215 + return all_tokens, all_probs, all_logprobs
216
217 def _create_scoring_model_input(
218 self,
@@ -308,7 +322,8 @@ class BatchExpansionTop1Scorer(SpeculativeScorer):
322
323 def _split_scoring_output(
324 self, sampler_output: SamplerOutput, num_scoring_tokens: int
311 - ) -> Tuple[torch.Tensor, torch.Tensor, torch.Tensor, torch.Tensor]:
325 + ) -> Tuple[torch.Tensor, torch.Tensor, torch.Tensor, torch.Tensor,
326 + torch.Tensor, torch.Tensor]:
327 """Split the target model output into speculative and non-speculative
328 output.
329 """
@@ -328,21 +343,29 @@ class BatchExpansionTop1Scorer(SpeculativeScorer):
343 ) = sampler_output.sampled_token_probs.split(split_sizes)
344 (spec_sampled_tokens, non_spec_sampled_tokens
345 ) = sampler_output.sampled_token_ids.flatten().split(split_sizes)
346 + (
347 + spec_logprobs,
348 + non_spec_logprobs,
349 + ) = sampler_output.logprobs.split(split_sizes)
350
351 # Convert scores to tensors.
352 sampler_output.sampled_token_probs = spec_probs
353 sampler_output.sampled_token_ids = spec_sampled_tokens
335 - target_token_ids, target_probs = sampler_output_to_torch(
336 - [sampler_output], True)
354 + sampler_output.logprobs = spec_logprobs
355 + (target_token_ids, target_probs,
356 + target_logprobs) = sampler_output_to_torch([sampler_output], True)
357
358 # Convert non-speculative output tokens to tensors.
359 sampler_output.sampled_token_probs = non_spec_probs
360 sampler_output.sampled_token_ids = non_spec_sampled_tokens
341 - non_spec_target_token_ids, non_spec_target_probs = (
342 - sampler_output_to_torch([sampler_output], True))
343 -
344 - return (target_token_ids, target_probs, non_spec_target_token_ids,
345 - non_spec_target_probs)
361 + sampler_output.logprobs = non_spec_logprobs
362 + (non_spec_target_token_ids, non_spec_target_probs,
363 + non_spec_target_logprobs) = sampler_output_to_torch([sampler_output],
364 + True)
365 +
366 + return (target_token_ids, target_probs, target_logprobs,
367 + non_spec_target_token_ids, non_spec_target_probs,
368 + non_spec_target_logprobs)
369
370 def _create_target_seq_id_iterator(
371 self, seq_ids: List[SeqId]) -> Iterator[TargetSeqId]:
vllm/spec_decode/interfaces.py
+5
@@ -38,6 +38,11 @@ class SpeculativeScores:
38 # Probabilities of the speculative tokens according to the scoring model.
39 probs: torch.Tensor
40
41 + # Log-probabilities of the speculative tokens according to the scoring
42 + # model. These values can be used to generate Logprob objects that are
43 + # returned to the user.
44 + logprobs: torch.Tensor
45 +
46 # Token ids sampled from the scoring model. Used for speculative bonus
47 # tokens and also non-speculative normal decoding.
48 token_ids: torch.Tensor
vllm/spec_decode/ngram_worker.py
+6
@@ -140,11 +140,17 @@ class NGramWorker(LoraNotSupportedWorkerBase):
140 device=self.device,
141 )
142 token_probs.scatter_(2, indices, 1)
143 + token_logprobs = torch.zeros(
144 + (len(seq_group_metadata_list), sample_len, self.vocab_size),
145 + dtype=torch.float32,
146 + device=self.device,
147 + )
148 for i in range(len(seq_group_metadata_list)):
149 outputs.append(
150 SamplerOutput(
151 outputs=None,
152 sampled_token_probs=token_probs[i],
153 + logprobs=token_logprobs,
154 sampled_token_ids=token_ids[i],
155 ))
156 return outputs, False
vllm/spec_decode/spec_decode_worker.py
+74 -26
@@ -5,15 +5,16 @@ import torch
5
6 from vllm.logger import init_logger
7 from vllm.model_executor.layers.rejection_sampler import RejectionSampler
8 -from vllm.sequence import (Logprob, SamplerOutput, SequenceGroupMetadata,
9 - SequenceGroupOutput, SequenceOutput)
8 +from vllm.sequence import SamplerOutput, SequenceGroupMetadata
9 from vllm.spec_decode.batch_expansion import BatchExpansionTop1Scorer
10 from vllm.spec_decode.interfaces import (SpeculativeProposals,
11 SpeculativeScorer, SpeculativeScores)
12 from vllm.spec_decode.metrics import AsyncMetricsCollector
13 from vllm.spec_decode.multi_step_worker import MultiStepWorker
14 from vllm.spec_decode.ngram_worker import NGramWorker
16 -from vllm.spec_decode.util import (get_all_seq_ids, nvtx_range,
15 +from vllm.spec_decode.util import (create_sequence_group_output,
16 + get_all_num_logprobs, get_all_seq_ids,
17 + get_sampled_token_logprobs, nvtx_range,
18 split_batch_by_proposal_len)
19 from vllm.worker.worker_base import LoraNotSupportedWorkerBase, WorkerBase
20
@@ -258,6 +259,7 @@ class SpecDecodeWorker(LoraNotSupportedWorkerBase):
259 # overhead when the engine runs in a different process than the workers.
260 sampler_output.probs = None
261 sampler_output.sampled_tokens = None
262 + sampler_output.logprobs = None
263 return [sampler_output]
264
265 @nvtx_range("spec_decode_worker._run_speculative_decoding_step")
@@ -298,12 +300,15 @@ class SpecDecodeWorker(LoraNotSupportedWorkerBase):
300 )
301
302 #logger.info("verify proposals")
301 - accepted_token_ids = self._verify_tokens(seq_group_metadata_list,
302 - proposal_scores, proposals, k)
303 + accepted_token_ids, target_logprobs = self._verify_tokens(
304 + seq_group_metadata_list, proposal_scores, proposals, k)
305
306 #logger.info("create output list")
305 - return self._create_output_sampler_list(seq_group_metadata_list,
306 - accepted_token_ids, k)
307 + return self._create_output_sampler_list(
308 + seq_group_metadata_list,
309 + accepted_token_ids,
310 + target_logprobs=target_logprobs,
311 + k=k)
312
313 @nvtx_range("spec_decode_worker._verify_tokens")
314 def _verify_tokens(
@@ -312,9 +317,12 @@ class SpecDecodeWorker(LoraNotSupportedWorkerBase):
317 proposal_scores: SpeculativeScores,
318 proposals: SpeculativeProposals,
319 max_proposal_len: int,
315 - ) -> torch.Tensor:
320 + ) -> Tuple[torch.Tensor, torch.Tensor]:
321 """Determine which speculative tokens are accepted using the
322 probabilities of each token according to the proposer and scorer models.
323 +
324 + Returns a tuple of Tensors, one for the accepted token ids and one for
325 + the logprobs according to the scoring model.
326 """
327 proposal_lens_list = proposals.proposal_lens.tolist()
328
@@ -361,17 +369,19 @@ class SpecDecodeWorker(LoraNotSupportedWorkerBase):
369 non_spec_token_ids[:, 1:] = -1
370 accepted_token_ids = torch.cat(
371 [accepted_token_ids, non_spec_token_ids])
372 + logprobs = proposal_scores.logprobs
373
374 # Rearrange so that results are in the order of the original seq group
375 # metadata.
376 accepted_token_ids[original_indices] = accepted_token_ids.clone()
377
369 - return accepted_token_ids
378 + return accepted_token_ids, logprobs
379
380 def _create_output_sampler_list(
381 self,
382 seq_group_metadata_list: List[SequenceGroupMetadata],
383 accepted_token_ids: torch.Tensor, # shape: [batch_size, k+1]
384 + target_logprobs: torch.Tensor, # shape: [batch_size, k+1, vocab_size]
385 k: int,
386 ) -> List[SamplerOutput]:
387 """Given the accepted token ids, create a list of SamplerOutput.
@@ -379,30 +389,68 @@ class SpecDecodeWorker(LoraNotSupportedWorkerBase):
389 The output is padded with -1 tokens such that each sequence has
390 the same number of outputs.
391 """
392 + batch_size, num_steps = accepted_token_ids.shape
393 +
394 + # Organize input tensors by step instead of by sequence.
395 + target_logprobs_by_step = target_logprobs.transpose(0, 1)
396 + accepted_token_ids_by_step = accepted_token_ids.transpose(0, 1)
397 +
398 + # Get the logprobs/rank of the accepted tokens.
399 + (accepted_token_id_ranks_by_step,
400 + accepted_token_id_logprobs_by_step) = get_sampled_token_logprobs(
401 + logprob_tensor=target_logprobs_by_step,
402 + sampled_token_ids=accepted_token_ids_by_step,
403 + )
404 +
405 + # Get the top-k logprobs (which may or may not include the logprob of
406 + # the accepted token).
407 + (topk_logprobs_by_step,
408 + topk_indices_by_step) = target_logprobs_by_step.topk(
409 + k=self.scorer_worker.model_config.max_logprobs,
410 + dim=-1,
411 + )
412 +
413 + # Get the sequence ids and num_logprobs (sampling parameter) in the
414 + # batch.
415 seq_ids = get_all_seq_ids(seq_group_metadata_list)
383 -
384 - # shape: [k+1, batch_size]
385 - accepted_token_ids_by_step = accepted_token_ids.transpose(0,
386 - 1).tolist()
416 + num_logprobs_per_seq = get_all_num_logprobs(seq_group_metadata_list)
417 +
418 + # Serialize all tensors to CPU Python lists.
419 + accepted_token_ids_by_step = accepted_token_ids_by_step.tolist()
420 + accepted_token_id_ranks_by_step = (
421 + accepted_token_id_ranks_by_step.tolist())
422 + accepted_token_id_logprobs_by_step = (
423 + accepted_token_id_logprobs_by_step.tolist())
424 + topk_logprobs_by_step = topk_logprobs_by_step.tolist()
425 + topk_indices_by_step = topk_indices_by_step.tolist()
426 +
427 + # Construct the output on a per-step, per-sequence basis.
428 sampler_output_list = []
388 - for token_ids_by_step in accepted_token_ids_by_step:
389 - if all(token_id == -1 for token_id in token_ids_by_step):
429 + for step_index in range(num_steps):
430 + if all(token_id == -1
431 + for token_id in accepted_token_ids_by_step[step_index]):
432 break
433
434 step_output_token_ids = []
393 - for token_id, seq_id in zip(token_ids_by_step, seq_ids):
435 + for sequence_index in range(batch_size):
436 + # Each sequence may have a different num_logprobs; retrieve it.
437 + num_logprobs = num_logprobs_per_seq[sequence_index]
438 +
439 step_output_token_ids.append(
395 - SequenceGroupOutput(
396 - samples=[
397 - SequenceOutput(
398 - parent_seq_id=seq_id,
399 - output_token=token_id,
400 - # TODO Add verifier logprobs.
401 - logprobs={token_id: Logprob(0.0)},
402 - )
403 - ],
404 - prompt_logprobs=None,
440 + create_sequence_group_output(
441 + token_id=accepted_token_ids_by_step[step_index]
442 + [sequence_index],
443 + token_id_logprob_rank=accepted_token_id_ranks_by_step[
444 + step_index][sequence_index],
445 + token_id_logprob=accepted_token_id_logprobs_by_step[
446 + step_index][sequence_index],
447 + seq_id=seq_ids[sequence_index],
448 + topk_token_ids=topk_indices_by_step[step_index]
449 + [sequence_index][:num_logprobs],
450 + topk_logprobs=topk_logprobs_by_step[step_index]
451 + [sequence_index][:num_logprobs],
452 ))
453 +
454 sampler_output_list.append(
455 SamplerOutput(outputs=step_output_token_ids))
456
vllm/spec_decode/top1_proposer.py
+1 -1
@@ -166,7 +166,7 @@ class Top1Proposer(SpeculativeProposer):
166 return proposal_tokens, proposal_probs, proposal_lens_tensor
167
168 sampler_output = maybe_sampler_output
169 - proposal_tokens, proposal_probs = sampler_output_to_torch(
169 + proposal_tokens, proposal_probs, _ = sampler_output_to_torch(
170 sampler_output, sampler_transposed)
171
172 # Now, reformat the output GPU tensors such that each sequence has
vllm/spec_decode/util.py
+98 -5
@@ -1,10 +1,11 @@
1 from contextlib import contextmanager
2 from itertools import chain
3 -from typing import List, Tuple
3 +from typing import Dict, List, Tuple
4
5 import torch
6
7 -from vllm.sequence import SamplerOutput, SequenceGroupMetadata
7 +from vllm.sequence import (Logprob, SamplerOutput, SequenceGroupMetadata,
8 + SequenceGroupOutput, SequenceOutput)
9
10 SeqId = int
11
@@ -21,6 +22,89 @@ def get_all_seq_ids(
22 ]))
23
24
25 +def get_all_num_logprobs(
26 + seq_group_metadata_list: List[SequenceGroupMetadata]) -> List[int]:
27 + """Given a list of SequenceGroupMetadata, create a list of all num_logprobs.
28 +
29 + If the sampling params do not call for any logprobs, return 0 for that
30 + sequence.
31 + """
32 +
33 + all_num_logprobs = []
34 + for seq_group_metadata in seq_group_metadata_list:
35 + num_logprobs = seq_group_metadata.sampling_params.logprobs
36 + if seq_group_metadata.sampling_params.logprobs is None:
37 + num_logprobs = 0
38 + all_num_logprobs.append(num_logprobs)
39 +
40 + return all_num_logprobs
41 +
42 +
43 +def get_sampled_token_logprobs(
44 + # shape [num_steps, batch_size, vocab_size]
45 + logprob_tensor: torch.Tensor,
46 + sampled_token_ids: torch.Tensor, # shape [num_steps, batch_size]
47 +) -> Tuple[torch.Tensor, torch.Tensor]:
48 + """Get the logprobs for the sampled tokens. Returns the ranks and logprobs.
49 + """
50 + num_steps, batch_size, vocab_size = logprob_tensor.shape
51 +
52 + selected_logprobs = logprob_tensor[torch.arange(num_steps).unsqueeze(1),
53 + torch.arange(batch_size),
54 + sampled_token_ids, ]
55 + expanded_selected_logprobs = selected_logprobs.unsqueeze(-1).expand(
56 + -1, -1, vocab_size)
57 + sampled_token_ids_ranks = (logprob_tensor >=
58 + expanded_selected_logprobs).sum(-1)
59 +
60 + return sampled_token_ids_ranks, selected_logprobs
61 +
62 +
63 +def create_sequence_group_output(
64 + token_id: int,
65 + token_id_logprob_rank: int,
66 + token_id_logprob: float,
67 + seq_id: SeqId,
68 + topk_token_ids: List[int],
69 + topk_logprobs: List[float],
70 +) -> SequenceGroupOutput:
71 + """Create a SequenceGroupOutput given the sampling results.
72 +
73 + Args:
74 + token_id (int): The sampled token for the sequence.
75 + token_id_logprob_rank (int): The logprob rank of the sampled token.
76 + token_id_logprob (float): The logprob value of the sampled token.
77 + seq_id (int): The sequence id.
78 + topk_token_ids (List[int]): The list of top-k token ids.
79 + topk_logprobs (List[float]): The list of top-k logprobs.
80 + """
81 + # vLLM logprobs always include the sampled token. In addition, the user may
82 + # request topk-logprobs (where top-k varies per user up to max_logprobs).
83 + logprobs: Dict[int, Logprob] = {
84 + token_id: Logprob(
85 + logprob=token_id_logprob,
86 + rank=token_id_logprob_rank,
87 + ),
88 + }
89 + logprobs.update({
90 + topk_token_ids[topk_logprob_index]: Logprob(
91 + logprob=topk_logprobs[topk_logprob_index],
92 + rank=topk_logprob_index + 1,
93 + )
94 + for topk_logprob_index, _ in enumerate(topk_token_ids)
95 + })
96 +
97 + return SequenceGroupOutput(
98 + samples=[
99 + SequenceOutput(parent_seq_id=seq_id,
100 + output_token=token_id,
101 + logprobs=logprobs)
102 + ],
103 + # TODO add prompt logprobs support.
104 + prompt_logprobs=None,
105 + )
106 +
107 +
108 def split_batch_by_proposal_len(
109 seq_group_metadata_list: List[SequenceGroupMetadata],
110 proposal_lens: List[int], select_proposal_len_zero: bool
@@ -49,8 +133,8 @@ def split_batch_by_proposal_len(
133
134
135 def sampler_output_to_torch(
52 - sampler_output_list: List[SamplerOutput],
53 - sampler_transposed: bool) -> Tuple[torch.Tensor, torch.Tensor]:
136 + sampler_output_list: List[SamplerOutput], sampler_transposed: bool
137 +) -> Tuple[torch.Tensor, torch.Tensor, torch.Tensor]:
138 """Utility function which converts a list of SamplerOutput to tensors.
139
140 sampler_transposed here is used as the indicator for whether
@@ -76,6 +160,15 @@ def sampler_output_to_torch(
160 if sampler_transposed:
161 sampled_token_probs = sampled_token_probs.transpose(0, 1)
162
163 + # shape: [batch_size, num_sampler_output, vocab_size]
164 + sampled_token_logprobs = torch.stack(
165 + [sampler_output.logprobs for sampler_output in sampler_output_list],
166 + dim=0,
167 + )
168 +
169 + if sampler_transposed:
170 + sampled_token_logprobs = sampled_token_logprobs.transpose(0, 1)
171 +
172 # shape: [batch_size, num_sampler_output]
173 sampled_token_ids = torch.stack(
174 [
@@ -87,7 +180,7 @@ def sampler_output_to_torch(
180 if sampler_transposed:
181 sampled_token_ids = sampled_token_ids.transpose(0, 1)
182
90 - return sampled_token_ids, sampled_token_probs
183 + return sampled_token_ids, sampled_token_probs, sampled_token_logprobs
184
185
186 def maybe_mock_device_tensors(sampler_output: SamplerOutput, batch_size: int,