@tduncan / Nvidia-Optimization-Stack-VLLM Mirror · read-only / commits / 947b794146
Fork 0

A high-throughput and memory-efficient inference and serving engine for LLMs

Mirrored from https://github.com/tduncan-lum/Nvidia-Optimization-Stack---VLLM.git

Synced Nov 17, 06:45 UTC

[Sampler] Vectorized sampling (simplified) (#1048)

Co-authored-by: Antoni Baum <antoni.baum@protonmail.com>

Zhuohan Li committed Sep 22, 2023 at 17:48 UTC 947b794146aeae41ea17cbbe8bf9e53abc9c3f53
3 files changed +481 -180
tests/samplers/test_sampler.py new
+184
@@ -0,0 +1,184 @@
1 +import pytest
2 +import random
3 +from typing import Tuple
4 +from unittest.mock import patch
5 +
6 +import torch
7 +
8 +from vllm.model_executor.layers.sampler import Sampler
9 +from vllm.model_executor.utils import set_random_seed
10 +from vllm.sequence import SamplingParams, SequenceData, SequenceGroupMetadata
11 +from vllm.worker.worker import Worker
12 +
13 +
14 +class MockLogitsSampler(Sampler):
15 +
16 + def __init__(self, vocab_size: int, fake_logits: torch.Tensor):
17 + super().__init__(vocab_size=vocab_size)
18 + self.fake_logits = fake_logits
19 +
20 + def forward(self, *args, **kwargs):
21 + with patch("vllm.model_executor.layers.sampler._prune_hidden_states",
22 + lambda x, y: x):
23 + with patch("vllm.model_executor.layers.sampler._get_logits",
24 + lambda *args, **kwargs: self.fake_logits):
25 + return super().forward(*args, **kwargs)
26 +
27 +
28 +def _prepare_test(
29 + batch_size: int
30 +) -> Tuple[torch.Tensor, torch.Tensor, MockLogitsSampler, Worker]:
31 + vocab_size = 32000
32 + input_tensor = torch.rand((batch_size, 1024),
33 + device="cuda",
34 + dtype=torch.float16)
35 + fake_logits = torch.full((batch_size, vocab_size),
36 + 1e-2,
37 + device=input_tensor.device,
38 + dtype=input_tensor.dtype)
39 + sampler = MockLogitsSampler(32000, fake_logits)
40 + worker = Worker(None, None, None)
41 + worker.block_size = 16
42 + return input_tensor, fake_logits, sampler, worker
43 +
44 +
45 +RANDOM_SEEDS = list(range(128))
46 +
47 +
48 +@pytest.mark.parametrize("seed", RANDOM_SEEDS)
49 +def test_sampler_all_greedy(seed: int):
50 + set_random_seed(seed)
51 + batch_size = random.randint(1, 256)
52 + input_tensor, fake_logits, sampler, worker = _prepare_test(batch_size)
53 +
54 + seq_group_metadata_list = []
55 + for i in range(batch_size):
56 + seq_group_metadata_list.append(
57 + SequenceGroupMetadata(
58 + request_id=f"test_{i}",
59 + is_prompt=True,
60 + seq_data={0: SequenceData([1, 2, 3])},
61 + sampling_params=SamplingParams(temperature=0, ),
62 + block_tables={0: [1]},
63 + ))
64 +
65 + _, _, input_metadata = worker._prepare_inputs(seq_group_metadata_list)
66 + sampler_output = sampler(embedding=None,
67 + hidden_states=input_tensor,
68 + input_metadata=input_metadata)
69 + expected = torch.argmax(fake_logits, dim=-1)
70 + for i, sequence_output in enumerate(sampler_output):
71 + for nth_output in sequence_output:
72 + assert nth_output.output_token == expected[i].item()
73 +
74 +
75 +@pytest.mark.parametrize("seed", RANDOM_SEEDS)
76 +def test_sampler_all_random(seed: int):
77 + set_random_seed(seed)
78 + batch_size = random.randint(1, 256)
79 + input_tensor, fake_logits, sampler, worker = _prepare_test(batch_size)
80 +
81 + for i in range(batch_size):
82 + fake_logits[i, i] = 1e2
83 +
84 + seq_group_metadata_list = []
85 + for i in range(batch_size):
86 + seq_group_metadata_list.append(
87 + SequenceGroupMetadata(
88 + request_id=f"test_{i}",
89 + is_prompt=True,
90 + seq_data={0: SequenceData([1, 2, 3])},
91 + sampling_params=SamplingParams(
92 + temperature=1.0,
93 + n=random.randint(1, 10),
94 + ),
95 + block_tables={0: [1]},
96 + ))
97 +
98 + _, _, input_metadata = worker._prepare_inputs(seq_group_metadata_list)
99 + sampler_output = sampler(embedding=None,
100 + hidden_states=input_tensor,
101 + input_metadata=input_metadata)
102 + for i, sequence_output in enumerate(sampler_output):
103 + for nth_output in sequence_output:
104 + assert nth_output.output_token == i
105 +
106 +
107 +@pytest.mark.parametrize("seed", RANDOM_SEEDS)
108 +def test_sampler_all_beam(seed: int):
109 + set_random_seed(seed)
110 + batch_size = random.randint(1, 256)
111 + input_tensor, fake_logits, sampler, worker = _prepare_test(batch_size)
112 +
113 + seq_group_metadata_list = []
114 + for i in range(batch_size):
115 + seq_group_metadata_list.append(
116 + SequenceGroupMetadata(
117 + request_id=f"test_{i}",
118 + is_prompt=True,
119 + seq_data={0: SequenceData([1, 2, 3])},
120 + sampling_params=SamplingParams(
121 + temperature=0,
122 + best_of=2,
123 + use_beam_search=True,
124 + ),
125 + block_tables={0: [1]},
126 + ))
127 +
128 + _, _, input_metadata = worker._prepare_inputs(seq_group_metadata_list)
129 + sampler(embedding=None,
130 + hidden_states=input_tensor,
131 + input_metadata=input_metadata)
132 + # no assertion here as I am not sure how to determine whether
133 + # the outputs are expected - in other words, this just tests
134 + # whether there are no exceptions in the sampler
135 + # when handling an all-beam search case.
136 +
137 +
138 +@pytest.mark.parametrize("seed", RANDOM_SEEDS)
139 +def test_sampler_mixed(seed: int):
140 + set_random_seed(seed)
141 + batch_size = random.randint(1, 256)
142 + input_tensor, fake_logits, sampler, worker = _prepare_test(batch_size)
143 +
144 + seq_group_metadata_list = []
145 + expected_tokens = []
146 + for i in range(batch_size):
147 + n = 1
148 + sampling_type = random.randint(0, 2)
149 + if sampling_type == 0:
150 + sampling_params = SamplingParams(temperature=0)
151 + elif sampling_type == 1:
152 + n = random.randint(1, 10)
153 + sampling_params = SamplingParams(
154 + temperature=random.random() + 0.1,
155 + top_p=min(random.random() + 0.1, 1),
156 + top_k=random.randint(0, 10) or -1,
157 + n=n,
158 + presence_penalty=random.randint(0, 1),
159 + )
160 + else:
161 + sampling_params = SamplingParams(temperature=0,
162 + use_beam_search=True,
163 + best_of=2)
164 + for idx in range(n):
165 + fake_logits[i, i + idx] = 1e2
166 + expected_tokens.append(i + idx)
167 + seq_group_metadata_list.append(
168 + SequenceGroupMetadata(
169 + request_id=f"test_{i}",
170 + is_prompt=True,
171 + seq_data={0: SequenceData([1, 2, 3])},
172 + sampling_params=sampling_params,
173 + block_tables={0: [1]},
174 + ))
175 +
176 + _, _, input_metadata = worker._prepare_inputs(seq_group_metadata_list)
177 + sampler_output = sampler(embedding=None,
178 + hidden_states=input_tensor,
179 + input_metadata=input_metadata)
180 + for i, sequence_output in enumerate(sampler_output):
181 + if seq_group_metadata_list[i].sampling_params.use_beam_search:
182 + continue
183 + for nth_output in sequence_output:
184 + assert nth_output.output_token in expected_tokens
vllm/model_executor/layers/sampler.py
+281 -180
@@ -1,15 +1,14 @@
1 """A layer that samples the next tokens from the model's outputs."""
2 -from typing import Dict, List, Tuple, Optional
2 +from typing import Dict, List, Optional, Tuple
3
4 -import numpy as np
4 import torch
5 import torch.nn as nn
6
7 from vllm.model_executor.input_metadata import InputMetadata
8 from vllm.model_executor.parallel_utils.tensor_parallel import (
9 gather_from_tensor_model_parallel_region)
11 -from vllm.sampling_params import SamplingParams
12 -from vllm.sequence import SamplerOutput, SequenceOutputs
10 +from vllm.sampling_params import SamplingParams, SamplingType
11 +from vllm.sequence import SamplerOutput, SequenceData, SequenceOutputs
12
13 _SAMPLING_EPS = 1e-5
14
@@ -44,12 +43,8 @@ class Sampler(nn.Module):
43 hidden_states = _prune_hidden_states(hidden_states, input_metadata)
44
45 # Get the logits for the next tokens.
47 - logits = torch.matmul(hidden_states, embedding.t())
48 - if embedding_bias is not None:
49 - logits += embedding_bias
50 - logits = gather_from_tensor_model_parallel_region(logits)
51 - # Remove paddings in vocab (if any).
52 - logits = logits[:, :self.vocab_size]
46 + logits = _get_logits(hidden_states, embedding, embedding_bias,
47 + self.vocab_size)
48
49 # Apply presence and frequency penalties.
50 output_tokens = _get_output_tokens(input_metadata)
@@ -59,7 +54,7 @@ class Sampler(nn.Module):
54 assert len(presence_penalties) == logits.shape[0]
55 assert len(frequency_penalties) == logits.shape[0]
56 logits = _apply_penalties(logits, output_tokens, presence_penalties,
62 - frequency_penalties, self.vocab_size)
57 + frequency_penalties)
58
59 # Apply temperature scaling.
60 temperatures = _get_temperatures(input_metadata)
@@ -90,19 +85,47 @@ class Sampler(nn.Module):
85 return _sample(probs, logprobs, input_metadata)
86
87
88 +def _get_logits(hidden_states: torch.Tensor, embedding: torch.Tensor,
89 + embedding_bias: Optional[torch.Tensor],
90 + vocab_size: int) -> torch.Tensor:
91 + # Get the logits for the next tokens.
92 + logits = torch.matmul(hidden_states, embedding.t())
93 + if embedding_bias is not None:
94 + logits += embedding_bias
95 + logits = gather_from_tensor_model_parallel_region(logits)
96 + # Remove paddings in vocab (if any).
97 + logits = logits[:, :vocab_size]
98 + return logits
99 +
100 +
101 def _prune_hidden_states(
102 hidden_states: torch.Tensor,
103 input_metadata: InputMetadata,
104 ) -> torch.Tensor:
105 + last_token_indices = {t: [] for t in SamplingType}
106 start_idx = 0
98 - last_token_indicies: List[int] = []
99 - for prompt_len in input_metadata.prompt_lens:
100 - last_token_indicies.append(start_idx + prompt_len - 1)
101 - start_idx += prompt_len
102 - last_token_indicies.extend(
103 - range(start_idx, start_idx + input_metadata.num_generation_tokens))
104 - return hidden_states.index_select(
105 - 0, torch.tensor(last_token_indicies, device=hidden_states.device))
107 + for i, seq_group in enumerate(input_metadata.seq_groups):
108 + seq_ids, sampling_params = seq_group
109 + sampling_type = sampling_params.sampling_type
110 + if i < input_metadata.num_prompts:
111 + assert len(seq_ids) == 1, "Prompt input should have only one seq."
112 + prompt_len = input_metadata.prompt_lens[i]
113 + last_token_indices[sampling_type].append(start_idx + prompt_len -
114 + 1)
115 + start_idx += prompt_len
116 + else:
117 + num_seqs = len(seq_ids)
118 + last_token_indices[sampling_type].extend(
119 + range(start_idx, start_idx + num_seqs))
120 + start_idx += num_seqs
121 +
122 + all_last_token_indices = []
123 + for sampling_type in SamplingType:
124 + all_last_token_indices.extend(last_token_indices[sampling_type])
125 + all_last_token_indices = torch.tensor(all_last_token_indices,
126 + dtype=torch.long,
127 + device=hidden_states.device)
128 + return hidden_states.index_select(0, all_last_token_indices)
129
130
131 def _get_penalties(
@@ -149,11 +172,8 @@ def _apply_penalties(
172 output_tokens: List[List[int]],
173 presence_penalties: List[float],
174 frequency_penalties: List[float],
152 - vocab_size: int,
175 ) -> torch.Tensor:
154 - num_seqs = logits.shape[0]
155 - # Collect the indices of sequences that have non-zero penalties.
156 - indices = []
176 + num_seqs, vocab_size = logits.shape
177 for i in range(num_seqs):
178 if not output_tokens[i]:
179 continue
@@ -161,33 +181,40 @@ def _apply_penalties(
181 f = frequency_penalties[i]
182 if abs(p) < _SAMPLING_EPS and abs(f) < _SAMPLING_EPS:
183 continue
164 - indices.append(i)
165 -
166 - # Return early if all sequences have zero penalties.
167 - if not indices:
184 + break
185 + else:
186 + # Return early if all sequences have zero penalties.
187 return logits
188
170 - bin_counts = []
171 - for i in indices:
172 - bin_counts.append(np.bincount(output_tokens[i], minlength=vocab_size))
173 - bin_counts = np.stack(bin_counts, axis=0)
174 - bin_counts = torch.from_numpy(bin_counts).to(dtype=logits.dtype,
175 - device=logits.device)
189 + max_output_len = max(len(tokens) for tokens in output_tokens)
190 + padded_output_tokens = [
191 + tokens + [vocab_size] * (max_output_len - len(tokens))
192 + for tokens in output_tokens
193 + ]
194 + output_tokens_tensor = torch.tensor(padded_output_tokens,
195 + dtype=torch.long,
196 + device=logits.device)
197 +
198 + # Compute the bin counts for the output tokens.
199 + # vocab_size + 1 for padding.
200 + bin_counts = torch.zeros((num_seqs, vocab_size + 1),
201 + dtype=torch.long,
202 + device=logits.device)
203 + bin_counts.scatter_add_(1, output_tokens_tensor,
204 + torch.ones_like(output_tokens_tensor))
205 + bin_counts = bin_counts[:, :vocab_size] # Remove the padding bin.
206
177 - frequency_penalties = [frequency_penalties[i] for i in indices]
207 frequency_penalties = torch.tensor(frequency_penalties,
208 dtype=logits.dtype,
209 device=logits.device)
181 - presence_penalties = [presence_penalties[i] for i in indices]
210 presence_penalties = torch.tensor(presence_penalties,
211 dtype=logits.dtype,
212 device=logits.device)
213
214 # We follow the definition in OpenAI API.
215 # Refer to https://platform.openai.com/docs/api-reference/parameter-details
188 - logits[indices] -= frequency_penalties.unsqueeze(dim=1) * bin_counts
189 - presence_mask = (bin_counts > 0.0).to(dtype=logits.dtype)
190 - logits[indices] -= presence_penalties.unsqueeze(dim=1) * presence_mask
216 + logits -= frequency_penalties.unsqueeze(dim=1) * bin_counts
217 + logits -= presence_penalties.unsqueeze(dim=1) * (bin_counts > 0)
218 return logits
219
220
@@ -268,95 +295,154 @@ def _apply_top_p_top_k(
295 def _get_topk_logprobs(
296 logprobs: torch.Tensor,
297 num_logprobs: Optional[int],
271 -) -> Dict[int, float]:
298 +) -> List[Dict[int, float]]:
299 + num_seqs = logprobs.size(0)
300 if num_logprobs is None or num_logprobs == 0:
273 - return {}
274 -
275 - topk_logprobs, topk_ids = torch.topk(logprobs, num_logprobs)
276 - if num_logprobs == 1:
277 - topk_logprobs = [topk_logprobs.item()]
278 - topk_ids = [topk_ids.item()]
279 - else:
280 - topk_logprobs = topk_logprobs.tolist()
281 - topk_ids = topk_ids.tolist()
282 -
283 - token_to_logprob: Dict[int, float] = {}
284 - for token_id, logprob in zip(topk_ids, topk_logprobs):
285 - token_to_logprob[token_id] = logprob
286 - return token_to_logprob
301 + return [{} for _ in range(num_seqs)]
302 +
303 + all_topk_logprobs, all_topk_ids = torch.topk(logprobs,
304 + num_logprobs,
305 + dim=-1)
306 + all_topk_logprobs = all_topk_logprobs.cpu()
307 + all_topk_ids = all_topk_ids.cpu()
308 + all_token_to_logprob = []
309 + for topk_logprobs, topk_ids in zip(all_topk_logprobs, all_topk_ids):
310 + token_to_logprob: Dict[int, float] = {}
311 + for token_id, logprob in zip(topk_ids, topk_logprobs):
312 + token_to_logprob[token_id.item()] = logprob.item()
313 + all_token_to_logprob.append(token_to_logprob)
314 + return all_token_to_logprob
315 +
316 +
317 +def _build_sequence_outputs(
318 + parent_ids: List[int],
319 + next_token_ids: List[int],
320 + selected_token_logprobs: torch.Tensor,
321 + parent_seq_ids: List[int],
322 + parent_logprobs: torch.Tensor,
323 + num_output_logprobs: Optional[int],
324 +) -> List[SequenceOutputs]:
325 + # Get top-k log probabilities for the next tokens.
326 + next_logprobs = _get_topk_logprobs(parent_logprobs, num_output_logprobs)
327 + seq_outputs: List[SequenceOutputs] = []
328 + for parent_id, next_token_id, token_logprob in zip(
329 + parent_ids, next_token_ids, selected_token_logprobs):
330 + output_logprobs = next_logprobs[parent_id].copy()
331 + output_logprobs[next_token_id] = token_logprob
332 + seq_outputs.append(
333 + SequenceOutputs(parent_seq_ids[parent_id], next_token_id,
334 + output_logprobs))
335 + return seq_outputs
336
337
289 -def _sample_from_prompt(
290 - prob: torch.Tensor,
291 - sampling_params: SamplingParams,
292 -) -> List[int]:
293 - if sampling_params.use_beam_search:
294 - # Beam search.
295 - beam_width = sampling_params.best_of
296 - # Sample 2 * beam_width candidates to make sure that with high
297 - # probability we can get `beam_width` candidates in addition to
298 - # the finished sequences for the next iteration. See
299 - # https://github.com/tensorflow/tensor2tensor/blob/bafdc1b67730430d38d6ab802cbd51f9d053ba2e/tensor2tensor/utils/beam_search.py#L557-L563
300 - # for details. See also HF reference:
301 - # https://github.com/huggingface/transformers/blob/a4dd53d88e4852f023332d284ff07a01afcd5681/src/transformers/generation/utils.py#L3063-L3065
302 - _, next_token_ids = torch.topk(prob, 2 * beam_width)
303 - next_token_ids = next_token_ids.tolist()
304 - elif sampling_params.temperature < _SAMPLING_EPS:
305 - # Greedy sampling.
306 - assert sampling_params.best_of == 1
307 - next_token_id = torch.argmax(prob)
308 - next_token_ids = [next_token_id.item()]
309 - else:
310 - # Random sampling.
311 - # Sample `best_of` tokens for the prompt.
312 - num_seqs = sampling_params.best_of
313 - next_token_ids = torch.multinomial(prob,
314 - num_samples=num_seqs,
315 - replacement=True)
316 - next_token_ids = next_token_ids.tolist()
317 - return next_token_ids
318 -
319 -
320 -def _sample_from_generation_tokens(
321 - seq_ids: List[int],
338 +def _greedy_sample(
339 + selected_seq_groups: List[Tuple[List[int], SamplingParams]],
340 + logprobs: torch.Tensor,
341 +) -> List[Tuple[List[int], List[int]]]:
342 + samples = torch.argmax(logprobs, dim=-1).cpu()
343 + sample_idx = 0
344 + results = []
345 + for seq_group in selected_seq_groups:
346 + seq_ids, _ = seq_group
347 + num_parent_seqs = len(seq_ids)
348 + assert num_parent_seqs == 1, (
349 + "Greedy sampling should have only one seq.")
350 + parent_ids = list(range(num_parent_seqs))
351 + next_token_ids = [samples[sample_idx].item()]
352 + results.append((next_token_ids, parent_ids))
353 + sample_idx += num_parent_seqs
354 + assert sample_idx == logprobs.size(0)
355 + return results
356 +
357 +
358 +def _random_sample(
359 + selected_seq_groups: List[Tuple[List[int], SamplingParams]],
360 + is_prompts: List[bool],
361 probs: torch.Tensor,
362 +) -> List[Tuple[List[int], List[int]]]:
363 + # Find the maximum best_of value of the prompt phase requests.
364 + max_best_of = 1
365 + for seq_group, is_prompt in zip(selected_seq_groups, is_prompts):
366 + if is_prompt:
367 + seq_ids, sampling_params = seq_group
368 + max_best_of = max(max_best_of, sampling_params.best_of)
369 + random_samples = torch.multinomial(probs,
370 + num_samples=max_best_of,
371 + replacement=True).cpu()
372 + sample_idx = 0
373 + results = []
374 + for seq_group, is_prompt in zip(selected_seq_groups, is_prompts):
375 + seq_ids, sampling_params = seq_group
376 + num_parent_seqs = len(seq_ids)
377 + if is_prompt:
378 + # Prompt phase.
379 + assert num_parent_seqs == 1, (
380 + "Prompt input should have only one seq.")
381 + parent_ids = [0] * sampling_params.best_of
382 + next_token_ids = random_samples[
383 + sample_idx, :sampling_params.best_of].tolist()
384 + else:
385 + # Generation phase.
386 + parent_ids = list(range(num_parent_seqs))
387 + next_token_ids = random_samples[sample_idx:sample_idx +
388 + num_parent_seqs, 0].tolist()
389 + results.append((next_token_ids, parent_ids))
390 + sample_idx += num_parent_seqs
391 + assert sample_idx == probs.size(0)
392 + return results
393 +
394 +
395 +def _beam_search_sample(
396 + selected_seq_groups: List[Tuple[List[int], SamplingParams]],
397 + is_prompts: List[bool],
398 + seq_data: Dict[int, SequenceData],
399 logprobs: torch.Tensor,
324 - seq_logprobs: List[float],
325 - sampling_params: SamplingParams,
326 -) -> Tuple[List[int], List[int]]:
327 - # NOTE(woosuk): sampling_params.best_of can be greater than
328 - # len(seq_ids) because some sequences in the group might have
329 - # been already terminated.
330 - if sampling_params.use_beam_search:
331 - # Beam search.
332 - # Add cumulative logprobs for the sequences in the group.
333 - seq_logprobs = torch.tensor(seq_logprobs,
334 - dtype=torch.float,
335 - device=logprobs.device)
336 - logprobs = logprobs + seq_logprobs.unsqueeze(dim=1)
337 -
338 - vocab_size = logprobs.size(-1)
339 - beam_width = len(seq_ids)
340 - _, topk_ids = torch.topk(logprobs.flatten(), 2 * beam_width)
341 - topk_ids = topk_ids.tolist()
342 - seq_idx = [i // vocab_size for i in topk_ids]
343 - parent_seq_ids = [seq_ids[i] for i in seq_idx]
344 - next_token_ids = [i % vocab_size for i in topk_ids]
345 - elif sampling_params.temperature < _SAMPLING_EPS:
346 - # Greedy sampling.
347 - assert len(seq_ids) == 1
348 - next_token_id = torch.argmax(probs, dim=-1)
349 - next_token_ids = [int(next_token_id.item())]
350 - parent_seq_ids = seq_ids
351 - else:
352 - # Random sampling.
353 - # Sample 1 token for each sequence in the group.
354 - next_token_ids = torch.multinomial(probs,
355 - num_samples=1,
356 - replacement=True)
357 - next_token_ids = next_token_ids.squeeze(dim=-1).tolist()
358 - parent_seq_ids = seq_ids
359 - return parent_seq_ids, next_token_ids
400 +) -> List[Tuple[List[int], List[int]]]:
401 + # We sample 2 * beam_width candidates to make sure that with high
402 + # probability we can get `beam_width` candidates in addition to
403 + # the finished sequences for the next iteration. See
404 + # https://github.com/tensorflow/tensor2tensor/blob/bafdc1b67730430d38d6ab802cbd51f9d053ba2e/tensor2tensor/utils/beam_search.py#L557-L563
405 + # for details. See also HF reference:
406 + # https://github.com/huggingface/transformers/blob/a4dd53d88e4852f023332d284ff07a01afcd5681/src/transformers/generation/utils.py#L3063-L3065
407 + #
408 + # Note: Beam search is not vectorized, so its speed can be slower than
409 + # other sampling methods.
410 + sample_idx = 0
411 + results = []
412 + for seq_group, is_prompt in zip(selected_seq_groups, is_prompts):
413 + seq_ids, sampling_params = seq_group
414 + num_parent_seqs = len(seq_ids)
415 + beam_width = sampling_params.best_of
416 + seq_group_logprobs = logprobs[sample_idx:sample_idx + num_parent_seqs]
417 + if is_prompt:
418 + # Prompt phase.
419 + assert num_parent_seqs == 1, (
420 + "Prompt input should have only one seq.")
421 + parent_ids = [0] * (2 * beam_width)
422 + _, next_token_ids = torch.topk(seq_group_logprobs[0],
423 + 2 * beam_width)
424 + next_token_ids = next_token_ids.tolist()
425 + else:
426 + # Generation phase.
427 + cumulative_logprobs = [
428 + seq_data[seq_id].cumulative_logprob for seq_id in seq_ids
429 + ]
430 + cumulative_logprobs = torch.tensor(
431 + cumulative_logprobs,
432 + dtype=torch.float,
433 + device=seq_group_logprobs.device)
434 + seq_group_logprobs = (seq_group_logprobs +
435 + cumulative_logprobs.unsqueeze(dim=1))
436 + _, topk_ids = torch.topk(seq_group_logprobs.flatten(),
437 + 2 * beam_width)
438 + topk_ids = topk_ids.tolist()
439 + vocab_size = seq_group_logprobs.size(-1)
440 + parent_ids = [i // vocab_size for i in topk_ids]
441 + next_token_ids = [i % vocab_size for i in topk_ids]
442 + results.append((next_token_ids, parent_ids))
443 + sample_idx += num_parent_seqs
444 + assert sample_idx == logprobs.size(0)
445 + return results
446
447
448 def _sample(
@@ -364,65 +450,80 @@ def _sample(
450 logprobs: torch.Tensor,
451 input_metadata: InputMetadata,
452 ) -> SamplerOutput:
367 - seq_outputs: SamplerOutput = []
368 -
369 - # TODO(woosuk): Optimize.
370 - idx = 0
453 + categorized_seq_group_ids = {t: [] for t in SamplingType}
454 + category_num_tokens = {t: 0 for t in SamplingType}
455 for i, seq_group in enumerate(input_metadata.seq_groups):
372 - seq_group_outputs: List[SequenceOutputs] = []
456 seq_ids, sampling_params = seq_group
374 - if i < input_metadata.num_prompts:
375 - # Generate the next tokens for a prompt input.
376 - assert len(seq_ids) == 1, "Prompt input should have only one seq."
377 - parent_seq_id = seq_ids[0]
378 - prob = probs[idx]
379 - logprob = logprobs[idx]
380 - idx += 1
381 -
382 - # Sample the next tokens.
383 - next_token_ids = _sample_from_prompt(prob, sampling_params)
384 - # Get top-k log probabilities for the next tokens.
385 - next_logprobs = _get_topk_logprobs(logprob,
386 - sampling_params.logprobs)
387 -
388 - # Build the output.
389 - for next_token_id in next_token_ids:
390 - output_logprobs = next_logprobs.copy()
391 - output_logprobs[next_token_id] = logprob[next_token_id].item()
392 - seq_group_outputs.append(
393 - SequenceOutputs(parent_seq_id, next_token_id,
394 - output_logprobs))
457 + sampling_type = sampling_params.sampling_type
458 + categorized_seq_group_ids[sampling_type].append(i)
459 + num_seqs = len(seq_ids)
460 + category_num_tokens[sampling_type] += num_seqs
461 +
462 + seq_outputs_dict: Dict[int, List[SequenceOutputs]] = {}
463 + category_start_idx = 0
464 + for sampling_type in SamplingType:
465 + seq_group_ids = categorized_seq_group_ids[sampling_type]
466 + seq_groups = [input_metadata.seq_groups[i] for i in seq_group_ids]
467 + is_prompts = [i < input_metadata.num_prompts for i in seq_group_ids]
468 + num_tokens = category_num_tokens[sampling_type]
469 + if num_tokens == 0:
470 + continue
471 + category_logprobs = logprobs[category_start_idx:category_start_idx +
472 + num_tokens]
473 + category_probs = probs[category_start_idx:category_start_idx +
474 + num_tokens]
475 + if sampling_type == SamplingType.GREEDY:
476 + sample_results = _greedy_sample(seq_groups, category_logprobs)
477 + elif sampling_type == SamplingType.RANDOM:
478 + sample_results = _random_sample(seq_groups, is_prompts,
479 + category_probs)
480 + elif sampling_type == SamplingType.BEAM:
481 + sample_results = _beam_search_sample(seq_groups, is_prompts,
482 + input_metadata.seq_data,
483 + category_logprobs)
484 else:
396 - # Generate the next tokens for generation tokens.
485 + raise ValueError(f"Unsupported sampling type: {sampling_type}")
486 +
487 + # Batched query for logprobs of selected token
488 + batched_logprobs_query_seq_indices: List[int] = []
489 + batched_logprobs_query_token_indices: List[int] = []
490 + sample_idx = 0
491 + for seq_group_id, seq_group, sample_result in zip(
492 + seq_group_ids, seq_groups, sample_results):
493 + seq_ids, sampling_params = seq_group
494 + next_token_ids, parent_ids = sample_result
495 num_parent_seqs = len(seq_ids)
398 - prob = probs[idx:idx + num_parent_seqs]
399 - logprob = logprobs[idx:idx + num_parent_seqs]
400 - idx += num_parent_seqs
401 -
402 - # Sample the next tokens.
403 - seq_logprobs = [
404 - input_metadata.seq_data[seq_id].cumulative_logprob
405 - for seq_id in seq_ids
406 - ]
407 - parent_seq_ids, next_token_ids = _sample_from_generation_tokens(
408 - seq_ids, prob, logprob, seq_logprobs, sampling_params)
409 -
410 - # Get top-k log probabilities for the next tokens.
411 - next_logprobs: Dict[int, Dict[int, float]] = {}
412 - for j, seq_id in enumerate(seq_ids):
413 - next_logprobs[seq_id] = _get_topk_logprobs(
414 - logprob[j], sampling_params.logprobs)
415 -
416 - # Build the output.
417 - for parent_seq_id, next_token_id in zip(parent_seq_ids,
418 - next_token_ids):
419 - j = seq_ids.index(parent_seq_id)
420 - output_logprobs = next_logprobs[parent_seq_id].copy()
421 - output_logprobs[next_token_id] = logprob[j,
422 - next_token_id].item()
423 - seq_group_outputs.append(
424 - SequenceOutputs(parent_seq_id, next_token_id,
425 - output_logprobs))
426 - seq_outputs.append(seq_group_outputs)
427 -
428 - return seq_outputs
496 + batched_logprobs_query_seq_indices.extend(
497 + [sample_idx + parent_id for parent_id in parent_ids])
498 + batched_logprobs_query_token_indices.extend(next_token_ids)
499 + sample_idx += num_parent_seqs
500 + assert sample_idx == num_tokens
501 + batched_logprobs_query_result = category_logprobs[[
502 + batched_logprobs_query_seq_indices,
503 + batched_logprobs_query_token_indices
504 + ]].tolist()
505 +
506 + # Build the sequence outputs.
507 + sample_idx = 0
508 + result_idx = 0
509 + for seq_group_id, seq_group, sample_result in zip(
510 + seq_group_ids, seq_groups, sample_results):
511 + seq_ids, sampling_params = seq_group
512 + next_token_ids, parent_ids = sample_result
513 + num_results = len(next_token_ids)
514 + num_parent_seqs = len(seq_ids)
515 + parent_logprobs = category_logprobs[sample_idx:sample_idx +
516 + num_parent_seqs]
517 + selected_token_logprobs = batched_logprobs_query_result[
518 + result_idx:result_idx + num_results]
519 + seq_output = _build_sequence_outputs(parent_ids, next_token_ids,
520 + selected_token_logprobs,
521 + seq_ids, parent_logprobs,
522 + sampling_params.logprobs)
523 + seq_outputs_dict[seq_group_id] = seq_output
524 + sample_idx += num_parent_seqs
525 + result_idx += num_results
526 + assert sample_idx == num_tokens
527 + category_start_idx += num_tokens
528 +
529 + return [seq_outputs_dict[i] for i in range(len(input_metadata.seq_groups))]
vllm/sampling_params.py
+16
@@ -1,9 +1,17 @@
1 """Sampling parameters for text generation."""
2 +from enum import IntEnum
3 +from functools import cached_property
4 from typing import List, Optional, Union
5
6 _SAMPLING_EPS = 1e-5
7
8
9 +class SamplingType(IntEnum):
10 + GREEDY = 0
11 + RANDOM = 1
12 + BEAM = 2
13 +
14 +
15 class SamplingParams:
16 """Sampling parameters for text generation.
17
@@ -166,6 +174,14 @@ class SamplingParams:
174 if self.top_k != -1:
175 raise ValueError("top_k must be -1 when using greedy sampling.")
176
177 + @cached_property
178 + def sampling_type(self) -> SamplingType:
179 + if self.use_beam_search:
180 + return SamplingType.BEAM
181 + if self.temperature < _SAMPLING_EPS:
182 + return SamplingType.GREEDY
183 + return SamplingType.RANDOM
184 +
185 def __repr__(self) -> str:
186 return (f"SamplingParams(n={self.n}, "
187 f"best_of={self.best_of}, "