A high-throughput and memory-efficient inference and serving engine for LLMs
Mirrored from https://github.com/tduncan-lum/Nvidia-Optimization-Stack---VLLM.git
Synced Nov 17, 06:45 UTC
[Sampler] Vectorized sampling (simplified) (#1048)
Co-authored-by: Antoni Baum <antoni.baum@protonmail.com>
Zhuohan Li committed
Sep 22, 2023 at 17:48 UTC
947b794146aeae41ea17cbbe8bf9e53abc9c3f53
3 files changed
+481
-180
tests/samplers/test_sampler.py
new
+184
@@ -0,0 +1,184 @@
1
+import pytest
2
+import random
3
+from typing import Tuple
4
+from unittest.mock import patch
5
+
6
+import torch
7
+
8
+from vllm.model_executor.layers.sampler import Sampler
9
+from vllm.model_executor.utils import set_random_seed
10
+from vllm.sequence import SamplingParams, SequenceData, SequenceGroupMetadata
11
+from vllm.worker.worker import Worker
12
+
13
+
14
+class MockLogitsSampler(Sampler):
15
+
16
+ def __init__(self, vocab_size: int, fake_logits: torch.Tensor):
17
+ super().__init__(vocab_size=vocab_size)
18
+ self.fake_logits = fake_logits
19
+
20
+ def forward(self, *args, **kwargs):
21
+ with patch("vllm.model_executor.layers.sampler._prune_hidden_states",
22
+ lambda x, y: x):
23
+ with patch("vllm.model_executor.layers.sampler._get_logits",
24
+ lambda *args, **kwargs: self.fake_logits):
25
+ return super().forward(*args, **kwargs)
26
+
27
+
28
+def _prepare_test(
29
+ batch_size: int
30
+) -> Tuple[torch.Tensor, torch.Tensor, MockLogitsSampler, Worker]:
31
+ vocab_size = 32000
32
+ input_tensor = torch.rand((batch_size, 1024),
33
+ device="cuda",
34
+ dtype=torch.float16)
35
+ fake_logits = torch.full((batch_size, vocab_size),
36
+ 1e-2,
37
+ device=input_tensor.device,
38
+ dtype=input_tensor.dtype)
39
+ sampler = MockLogitsSampler(32000, fake_logits)
40
+ worker = Worker(None, None, None)
41
+ worker.block_size = 16
42
+ return input_tensor, fake_logits, sampler, worker
43
+
44
+
45
+RANDOM_SEEDS = list(range(128))
46
+
47
+
48
+@pytest.mark.parametrize("seed", RANDOM_SEEDS)
49
+def test_sampler_all_greedy(seed: int):
50
+ set_random_seed(seed)
51
+ batch_size = random.randint(1, 256)
52
+ input_tensor, fake_logits, sampler, worker = _prepare_test(batch_size)
53
+
54
+ seq_group_metadata_list = []
55
+ for i in range(batch_size):
56
+ seq_group_metadata_list.append(
57
+ SequenceGroupMetadata(
58
+ request_id=f"test_{i}",
59
+ is_prompt=True,
60
+ seq_data={0: SequenceData([1, 2, 3])},
61
+ sampling_params=SamplingParams(temperature=0, ),
62
+ block_tables={0: [1]},
63
+ ))
64
+
65
+ _, _, input_metadata = worker._prepare_inputs(seq_group_metadata_list)
66
+ sampler_output = sampler(embedding=None,
67
+ hidden_states=input_tensor,
68
+ input_metadata=input_metadata)
69
+ expected = torch.argmax(fake_logits, dim=-1)
70
+ for i, sequence_output in enumerate(sampler_output):
71
+ for nth_output in sequence_output:
72
+ assert nth_output.output_token == expected[i].item()
73
+
74
+
75
+@pytest.mark.parametrize("seed", RANDOM_SEEDS)
76
+def test_sampler_all_random(seed: int):
77
+ set_random_seed(seed)
78
+ batch_size = random.randint(1, 256)
79
+ input_tensor, fake_logits, sampler, worker = _prepare_test(batch_size)
80
+
81
+ for i in range(batch_size):
82
+ fake_logits[i, i] = 1e2
83
+
84
+ seq_group_metadata_list = []
85
+ for i in range(batch_size):
86
+ seq_group_metadata_list.append(
87
+ SequenceGroupMetadata(
88
+ request_id=f"test_{i}",
89
+ is_prompt=True,
90
+ seq_data={0: SequenceData([1, 2, 3])},
91
+ sampling_params=SamplingParams(
92
+ temperature=1.0,
93
+ n=random.randint(1, 10),
94
+ ),
95
+ block_tables={0: [1]},
96
+ ))
97
+
98
+ _, _, input_metadata = worker._prepare_inputs(seq_group_metadata_list)
99
+ sampler_output = sampler(embedding=None,
100
+ hidden_states=input_tensor,
101
+ input_metadata=input_metadata)
102
+ for i, sequence_output in enumerate(sampler_output):
103
+ for nth_output in sequence_output:
104
+ assert nth_output.output_token == i
105
+
106
+
107
+@pytest.mark.parametrize("seed", RANDOM_SEEDS)
108
+def test_sampler_all_beam(seed: int):
109
+ set_random_seed(seed)
110
+ batch_size = random.randint(1, 256)
111
+ input_tensor, fake_logits, sampler, worker = _prepare_test(batch_size)
112
+
113
+ seq_group_metadata_list = []
114
+ for i in range(batch_size):
115
+ seq_group_metadata_list.append(
116
+ SequenceGroupMetadata(
117
+ request_id=f"test_{i}",
118
+ is_prompt=True,
119
+ seq_data={0: SequenceData([1, 2, 3])},
120
+ sampling_params=SamplingParams(
121
+ temperature=0,
122
+ best_of=2,
123
+ use_beam_search=True,
124
+ ),
125
+ block_tables={0: [1]},
126
+ ))
127
+
128
+ _, _, input_metadata = worker._prepare_inputs(seq_group_metadata_list)
129
+ sampler(embedding=None,
130
+ hidden_states=input_tensor,
131
+ input_metadata=input_metadata)
132
+ # no assertion here as I am not sure how to determine whether
133
+ # the outputs are expected - in other words, this just tests
134
+ # whether there are no exceptions in the sampler
135
+ # when handling an all-beam search case.
136
+
137
+
138
+@pytest.mark.parametrize("seed", RANDOM_SEEDS)
139
+def test_sampler_mixed(seed: int):
140
+ set_random_seed(seed)
141
+ batch_size = random.randint(1, 256)
142
+ input_tensor, fake_logits, sampler, worker = _prepare_test(batch_size)
143
+
144
+ seq_group_metadata_list = []
145
+ expected_tokens = []
146
+ for i in range(batch_size):
147
+ n = 1
148
+ sampling_type = random.randint(0, 2)
149
+ if sampling_type == 0:
150
+ sampling_params = SamplingParams(temperature=0)
151
+ elif sampling_type == 1:
152
+ n = random.randint(1, 10)
153
+ sampling_params = SamplingParams(
154
+ temperature=random.random() + 0.1,
155
+ top_p=min(random.random() + 0.1, 1),
156
+ top_k=random.randint(0, 10) or -1,
157
+ n=n,
158
+ presence_penalty=random.randint(0, 1),
159
+ )
160
+ else:
161
+ sampling_params = SamplingParams(temperature=0,
162
+ use_beam_search=True,
163
+ best_of=2)
164
+ for idx in range(n):
165
+ fake_logits[i, i + idx] = 1e2
166
+ expected_tokens.append(i + idx)
167
+ seq_group_metadata_list.append(
168
+ SequenceGroupMetadata(
169
+ request_id=f"test_{i}",
170
+ is_prompt=True,
171
+ seq_data={0: SequenceData([1, 2, 3])},
172
+ sampling_params=sampling_params,
173
+ block_tables={0: [1]},
174
+ ))
175
+
176
+ _, _, input_metadata = worker._prepare_inputs(seq_group_metadata_list)
177
+ sampler_output = sampler(embedding=None,
178
+ hidden_states=input_tensor,
179
+ input_metadata=input_metadata)
180
+ for i, sequence_output in enumerate(sampler_output):
181
+ if seq_group_metadata_list[i].sampling_params.use_beam_search:
182
+ continue
183
+ for nth_output in sequence_output:
184
+ assert nth_output.output_token in expected_tokens
vllm/model_executor/layers/sampler.py
+281
-180
@@ -1,15 +1,14 @@
1
"""A layer that samples the next tokens from the model's outputs."""
2
-from typing import Dict, List, Tuple, Optional
2
+from typing import Dict, List, Optional, Tuple
3
4
-import numpy as np
4
import torch
5
import torch.nn as nn
6
7
from vllm.model_executor.input_metadata import InputMetadata
8
from vllm.model_executor.parallel_utils.tensor_parallel import (
9
gather_from_tensor_model_parallel_region)
11
-from vllm.sampling_params import SamplingParams
12
-from vllm.sequence import SamplerOutput, SequenceOutputs
10
+from vllm.sampling_params import SamplingParams, SamplingType
11
+from vllm.sequence import SamplerOutput, SequenceData, SequenceOutputs
12
13
_SAMPLING_EPS = 1e-5
14
@@ -44,12 +43,8 @@ class Sampler(nn.Module):
43
hidden_states = _prune_hidden_states(hidden_states, input_metadata)
44
45
# Get the logits for the next tokens.
47
- logits = torch.matmul(hidden_states, embedding.t())
48
- if embedding_bias is not None:
49
- logits += embedding_bias
50
- logits = gather_from_tensor_model_parallel_region(logits)
51
- # Remove paddings in vocab (if any).
52
- logits = logits[:, :self.vocab_size]
46
+ logits = _get_logits(hidden_states, embedding, embedding_bias,
47
+ self.vocab_size)
48
49
# Apply presence and frequency penalties.
50
output_tokens = _get_output_tokens(input_metadata)
@@ -59,7 +54,7 @@ class Sampler(nn.Module):
54
assert len(presence_penalties) == logits.shape[0]
55
assert len(frequency_penalties) == logits.shape[0]
56
logits = _apply_penalties(logits, output_tokens, presence_penalties,
62
- frequency_penalties, self.vocab_size)
57
+ frequency_penalties)
58
59
# Apply temperature scaling.
60
temperatures = _get_temperatures(input_metadata)
@@ -90,19 +85,47 @@ class Sampler(nn.Module):
85
return _sample(probs, logprobs, input_metadata)
86
87
88
+def _get_logits(hidden_states: torch.Tensor, embedding: torch.Tensor,
89
+ embedding_bias: Optional[torch.Tensor],
90
+ vocab_size: int) -> torch.Tensor:
91
+ # Get the logits for the next tokens.
92
+ logits = torch.matmul(hidden_states, embedding.t())
93
+ if embedding_bias is not None:
94
+ logits += embedding_bias
95
+ logits = gather_from_tensor_model_parallel_region(logits)
96
+ # Remove paddings in vocab (if any).
97
+ logits = logits[:, :vocab_size]
98
+ return logits
99
+
100
+
101
def _prune_hidden_states(
102
hidden_states: torch.Tensor,
103
input_metadata: InputMetadata,
104
) -> torch.Tensor:
105
+ last_token_indices = {t: [] for t in SamplingType}
106
start_idx = 0
98
- last_token_indicies: List[int] = []
99
- for prompt_len in input_metadata.prompt_lens:
100
- last_token_indicies.append(start_idx + prompt_len - 1)
101
- start_idx += prompt_len
102
- last_token_indicies.extend(
103
- range(start_idx, start_idx + input_metadata.num_generation_tokens))
104
- return hidden_states.index_select(
105
- 0, torch.tensor(last_token_indicies, device=hidden_states.device))
107
+ for i, seq_group in enumerate(input_metadata.seq_groups):
108
+ seq_ids, sampling_params = seq_group
109
+ sampling_type = sampling_params.sampling_type
110
+ if i < input_metadata.num_prompts:
111
+ assert len(seq_ids) == 1, "Prompt input should have only one seq."
112
+ prompt_len = input_metadata.prompt_lens[i]
113
+ last_token_indices[sampling_type].append(start_idx + prompt_len -
114
+ 1)
115
+ start_idx += prompt_len
116
+ else:
117
+ num_seqs = len(seq_ids)
118
+ last_token_indices[sampling_type].extend(
119
+ range(start_idx, start_idx + num_seqs))
120
+ start_idx += num_seqs
121
+
122
+ all_last_token_indices = []
123
+ for sampling_type in SamplingType:
124
+ all_last_token_indices.extend(last_token_indices[sampling_type])
125
+ all_last_token_indices = torch.tensor(all_last_token_indices,
126
+ dtype=torch.long,
127
+ device=hidden_states.device)
128
+ return hidden_states.index_select(0, all_last_token_indices)
129
130
131
def _get_penalties(
@@ -149,11 +172,8 @@ def _apply_penalties(
172
output_tokens: List[List[int]],
173
presence_penalties: List[float],
174
frequency_penalties: List[float],
152
- vocab_size: int,
175
) -> torch.Tensor:
154
- num_seqs = logits.shape[0]
155
- # Collect the indices of sequences that have non-zero penalties.
156
- indices = []
176
+ num_seqs, vocab_size = logits.shape
177
for i in range(num_seqs):
178
if not output_tokens[i]:
179
continue
@@ -161,33 +181,40 @@ def _apply_penalties(
181
f = frequency_penalties[i]
182
if abs(p) < _SAMPLING_EPS and abs(f) < _SAMPLING_EPS:
183
continue
164
- indices.append(i)
165
-
166
- # Return early if all sequences have zero penalties.
167
- if not indices:
184
+ break
185
+ else:
186
+ # Return early if all sequences have zero penalties.
187
return logits
188
170
- bin_counts = []
171
- for i in indices:
172
- bin_counts.append(np.bincount(output_tokens[i], minlength=vocab_size))
173
- bin_counts = np.stack(bin_counts, axis=0)
174
- bin_counts = torch.from_numpy(bin_counts).to(dtype=logits.dtype,
175
- device=logits.device)
189
+ max_output_len = max(len(tokens) for tokens in output_tokens)
190
+ padded_output_tokens = [
191
+ tokens + [vocab_size] * (max_output_len - len(tokens))
192
+ for tokens in output_tokens
193
+ ]
194
+ output_tokens_tensor = torch.tensor(padded_output_tokens,
195
+ dtype=torch.long,
196
+ device=logits.device)
197
+
198
+ # Compute the bin counts for the output tokens.
199
+ # vocab_size + 1 for padding.
200
+ bin_counts = torch.zeros((num_seqs, vocab_size + 1),
201
+ dtype=torch.long,
202
+ device=logits.device)
203
+ bin_counts.scatter_add_(1, output_tokens_tensor,
204
+ torch.ones_like(output_tokens_tensor))
205
+ bin_counts = bin_counts[:, :vocab_size] # Remove the padding bin.
206
177
- frequency_penalties = [frequency_penalties[i] for i in indices]
207
frequency_penalties = torch.tensor(frequency_penalties,
208
dtype=logits.dtype,
209
device=logits.device)
181
- presence_penalties = [presence_penalties[i] for i in indices]
210
presence_penalties = torch.tensor(presence_penalties,
211
dtype=logits.dtype,
212
device=logits.device)
213
214
# We follow the definition in OpenAI API.
215
# Refer to https://platform.openai.com/docs/api-reference/parameter-details
188
- logits[indices] -= frequency_penalties.unsqueeze(dim=1) * bin_counts
189
- presence_mask = (bin_counts > 0.0).to(dtype=logits.dtype)
190
- logits[indices] -= presence_penalties.unsqueeze(dim=1) * presence_mask
216
+ logits -= frequency_penalties.unsqueeze(dim=1) * bin_counts
217
+ logits -= presence_penalties.unsqueeze(dim=1) * (bin_counts > 0)
218
return logits
219
220
@@ -268,95 +295,154 @@ def _apply_top_p_top_k(
295
def _get_topk_logprobs(
296
logprobs: torch.Tensor,
297
num_logprobs: Optional[int],
271
-) -> Dict[int, float]:
298
+) -> List[Dict[int, float]]:
299
+ num_seqs = logprobs.size(0)
300
if num_logprobs is None or num_logprobs == 0:
273
- return {}
274
-
275
- topk_logprobs, topk_ids = torch.topk(logprobs, num_logprobs)
276
- if num_logprobs == 1:
277
- topk_logprobs = [topk_logprobs.item()]
278
- topk_ids = [topk_ids.item()]
279
- else:
280
- topk_logprobs = topk_logprobs.tolist()
281
- topk_ids = topk_ids.tolist()
282
-
283
- token_to_logprob: Dict[int, float] = {}
284
- for token_id, logprob in zip(topk_ids, topk_logprobs):
285
- token_to_logprob[token_id] = logprob
286
- return token_to_logprob
301
+ return [{} for _ in range(num_seqs)]
302
+
303
+ all_topk_logprobs, all_topk_ids = torch.topk(logprobs,
304
+ num_logprobs,
305
+ dim=-1)
306
+ all_topk_logprobs = all_topk_logprobs.cpu()
307
+ all_topk_ids = all_topk_ids.cpu()
308
+ all_token_to_logprob = []
309
+ for topk_logprobs, topk_ids in zip(all_topk_logprobs, all_topk_ids):
310
+ token_to_logprob: Dict[int, float] = {}
311
+ for token_id, logprob in zip(topk_ids, topk_logprobs):
312
+ token_to_logprob[token_id.item()] = logprob.item()
313
+ all_token_to_logprob.append(token_to_logprob)
314
+ return all_token_to_logprob
315
+
316
+
317
+def _build_sequence_outputs(
318
+ parent_ids: List[int],
319
+ next_token_ids: List[int],
320
+ selected_token_logprobs: torch.Tensor,
321
+ parent_seq_ids: List[int],
322
+ parent_logprobs: torch.Tensor,
323
+ num_output_logprobs: Optional[int],
324
+) -> List[SequenceOutputs]:
325
+ # Get top-k log probabilities for the next tokens.
326
+ next_logprobs = _get_topk_logprobs(parent_logprobs, num_output_logprobs)
327
+ seq_outputs: List[SequenceOutputs] = []
328
+ for parent_id, next_token_id, token_logprob in zip(
329
+ parent_ids, next_token_ids, selected_token_logprobs):
330
+ output_logprobs = next_logprobs[parent_id].copy()
331
+ output_logprobs[next_token_id] = token_logprob
332
+ seq_outputs.append(
333
+ SequenceOutputs(parent_seq_ids[parent_id], next_token_id,
334
+ output_logprobs))
335
+ return seq_outputs
336
337
289
-def _sample_from_prompt(
290
- prob: torch.Tensor,
291
- sampling_params: SamplingParams,
292
-) -> List[int]:
293
- if sampling_params.use_beam_search:
294
- # Beam search.
295
- beam_width = sampling_params.best_of
296
- # Sample 2 * beam_width candidates to make sure that with high
297
- # probability we can get `beam_width` candidates in addition to
298
- # the finished sequences for the next iteration. See
299
- # https://github.com/tensorflow/tensor2tensor/blob/bafdc1b67730430d38d6ab802cbd51f9d053ba2e/tensor2tensor/utils/beam_search.py#L557-L563
300
- # for details. See also HF reference:
301
- # https://github.com/huggingface/transformers/blob/a4dd53d88e4852f023332d284ff07a01afcd5681/src/transformers/generation/utils.py#L3063-L3065
302
- _, next_token_ids = torch.topk(prob, 2 * beam_width)
303
- next_token_ids = next_token_ids.tolist()
304
- elif sampling_params.temperature < _SAMPLING_EPS:
305
- # Greedy sampling.
306
- assert sampling_params.best_of == 1
307
- next_token_id = torch.argmax(prob)
308
- next_token_ids = [next_token_id.item()]
309
- else:
310
- # Random sampling.
311
- # Sample `best_of` tokens for the prompt.
312
- num_seqs = sampling_params.best_of
313
- next_token_ids = torch.multinomial(prob,
314
- num_samples=num_seqs,
315
- replacement=True)
316
- next_token_ids = next_token_ids.tolist()
317
- return next_token_ids
318
-
319
-
320
-def _sample_from_generation_tokens(
321
- seq_ids: List[int],
338
+def _greedy_sample(
339
+ selected_seq_groups: List[Tuple[List[int], SamplingParams]],
340
+ logprobs: torch.Tensor,
341
+) -> List[Tuple[List[int], List[int]]]:
342
+ samples = torch.argmax(logprobs, dim=-1).cpu()
343
+ sample_idx = 0
344
+ results = []
345
+ for seq_group in selected_seq_groups:
346
+ seq_ids, _ = seq_group
347
+ num_parent_seqs = len(seq_ids)
348
+ assert num_parent_seqs == 1, (
349
+ "Greedy sampling should have only one seq.")
350
+ parent_ids = list(range(num_parent_seqs))
351
+ next_token_ids = [samples[sample_idx].item()]
352
+ results.append((next_token_ids, parent_ids))
353
+ sample_idx += num_parent_seqs
354
+ assert sample_idx == logprobs.size(0)
355
+ return results
356
+
357
+
358
+def _random_sample(
359
+ selected_seq_groups: List[Tuple[List[int], SamplingParams]],
360
+ is_prompts: List[bool],
361
probs: torch.Tensor,
362
+) -> List[Tuple[List[int], List[int]]]:
363
+ # Find the maximum best_of value of the prompt phase requests.
364
+ max_best_of = 1
365
+ for seq_group, is_prompt in zip(selected_seq_groups, is_prompts):
366
+ if is_prompt:
367
+ seq_ids, sampling_params = seq_group
368
+ max_best_of = max(max_best_of, sampling_params.best_of)
369
+ random_samples = torch.multinomial(probs,
370
+ num_samples=max_best_of,
371
+ replacement=True).cpu()
372
+ sample_idx = 0
373
+ results = []
374
+ for seq_group, is_prompt in zip(selected_seq_groups, is_prompts):
375
+ seq_ids, sampling_params = seq_group
376
+ num_parent_seqs = len(seq_ids)
377
+ if is_prompt:
378
+ # Prompt phase.
379
+ assert num_parent_seqs == 1, (
380
+ "Prompt input should have only one seq.")
381
+ parent_ids = [0] * sampling_params.best_of
382
+ next_token_ids = random_samples[
383
+ sample_idx, :sampling_params.best_of].tolist()
384
+ else:
385
+ # Generation phase.
386
+ parent_ids = list(range(num_parent_seqs))
387
+ next_token_ids = random_samples[sample_idx:sample_idx +
388
+ num_parent_seqs, 0].tolist()
389
+ results.append((next_token_ids, parent_ids))
390
+ sample_idx += num_parent_seqs
391
+ assert sample_idx == probs.size(0)
392
+ return results
393
+
394
+
395
+def _beam_search_sample(
396
+ selected_seq_groups: List[Tuple[List[int], SamplingParams]],
397
+ is_prompts: List[bool],
398
+ seq_data: Dict[int, SequenceData],
399
logprobs: torch.Tensor,
324
- seq_logprobs: List[float],
325
- sampling_params: SamplingParams,
326
-) -> Tuple[List[int], List[int]]:
327
- # NOTE(woosuk): sampling_params.best_of can be greater than
328
- # len(seq_ids) because some sequences in the group might have
329
- # been already terminated.
330
- if sampling_params.use_beam_search:
331
- # Beam search.
332
- # Add cumulative logprobs for the sequences in the group.
333
- seq_logprobs = torch.tensor(seq_logprobs,
334
- dtype=torch.float,
335
- device=logprobs.device)
336
- logprobs = logprobs + seq_logprobs.unsqueeze(dim=1)
337
-
338
- vocab_size = logprobs.size(-1)
339
- beam_width = len(seq_ids)
340
- _, topk_ids = torch.topk(logprobs.flatten(), 2 * beam_width)
341
- topk_ids = topk_ids.tolist()
342
- seq_idx = [i // vocab_size for i in topk_ids]
343
- parent_seq_ids = [seq_ids[i] for i in seq_idx]
344
- next_token_ids = [i % vocab_size for i in topk_ids]
345
- elif sampling_params.temperature < _SAMPLING_EPS:
346
- # Greedy sampling.
347
- assert len(seq_ids) == 1
348
- next_token_id = torch.argmax(probs, dim=-1)
349
- next_token_ids = [int(next_token_id.item())]
350
- parent_seq_ids = seq_ids
351
- else:
352
- # Random sampling.
353
- # Sample 1 token for each sequence in the group.
354
- next_token_ids = torch.multinomial(probs,
355
- num_samples=1,
356
- replacement=True)
357
- next_token_ids = next_token_ids.squeeze(dim=-1).tolist()
358
- parent_seq_ids = seq_ids
359
- return parent_seq_ids, next_token_ids
400
+) -> List[Tuple[List[int], List[int]]]:
401
+ # We sample 2 * beam_width candidates to make sure that with high
402
+ # probability we can get `beam_width` candidates in addition to
403
+ # the finished sequences for the next iteration. See
404
+ # https://github.com/tensorflow/tensor2tensor/blob/bafdc1b67730430d38d6ab802cbd51f9d053ba2e/tensor2tensor/utils/beam_search.py#L557-L563
405
+ # for details. See also HF reference:
406
+ # https://github.com/huggingface/transformers/blob/a4dd53d88e4852f023332d284ff07a01afcd5681/src/transformers/generation/utils.py#L3063-L3065
407
+ #
408
+ # Note: Beam search is not vectorized, so its speed can be slower than
409
+ # other sampling methods.
410
+ sample_idx = 0
411
+ results = []
412
+ for seq_group, is_prompt in zip(selected_seq_groups, is_prompts):
413
+ seq_ids, sampling_params = seq_group
414
+ num_parent_seqs = len(seq_ids)
415
+ beam_width = sampling_params.best_of
416
+ seq_group_logprobs = logprobs[sample_idx:sample_idx + num_parent_seqs]
417
+ if is_prompt:
418
+ # Prompt phase.
419
+ assert num_parent_seqs == 1, (
420
+ "Prompt input should have only one seq.")
421
+ parent_ids = [0] * (2 * beam_width)
422
+ _, next_token_ids = torch.topk(seq_group_logprobs[0],
423
+ 2 * beam_width)
424
+ next_token_ids = next_token_ids.tolist()
425
+ else:
426
+ # Generation phase.
427
+ cumulative_logprobs = [
428
+ seq_data[seq_id].cumulative_logprob for seq_id in seq_ids
429
+ ]
430
+ cumulative_logprobs = torch.tensor(
431
+ cumulative_logprobs,
432
+ dtype=torch.float,
433
+ device=seq_group_logprobs.device)
434
+ seq_group_logprobs = (seq_group_logprobs +
435
+ cumulative_logprobs.unsqueeze(dim=1))
436
+ _, topk_ids = torch.topk(seq_group_logprobs.flatten(),
437
+ 2 * beam_width)
438
+ topk_ids = topk_ids.tolist()
439
+ vocab_size = seq_group_logprobs.size(-1)
440
+ parent_ids = [i // vocab_size for i in topk_ids]
441
+ next_token_ids = [i % vocab_size for i in topk_ids]
442
+ results.append((next_token_ids, parent_ids))
443
+ sample_idx += num_parent_seqs
444
+ assert sample_idx == logprobs.size(0)
445
+ return results
446
447
448
def _sample(
@@ -364,65 +450,80 @@ def _sample(
450
logprobs: torch.Tensor,
451
input_metadata: InputMetadata,
452
) -> SamplerOutput:
367
- seq_outputs: SamplerOutput = []
368
-
369
- # TODO(woosuk): Optimize.
370
- idx = 0
453
+ categorized_seq_group_ids = {t: [] for t in SamplingType}
454
+ category_num_tokens = {t: 0 for t in SamplingType}
455
for i, seq_group in enumerate(input_metadata.seq_groups):
372
- seq_group_outputs: List[SequenceOutputs] = []
456
seq_ids, sampling_params = seq_group
374
- if i < input_metadata.num_prompts:
375
- # Generate the next tokens for a prompt input.
376
- assert len(seq_ids) == 1, "Prompt input should have only one seq."
377
- parent_seq_id = seq_ids[0]
378
- prob = probs[idx]
379
- logprob = logprobs[idx]
380
- idx += 1
381
-
382
- # Sample the next tokens.
383
- next_token_ids = _sample_from_prompt(prob, sampling_params)
384
- # Get top-k log probabilities for the next tokens.
385
- next_logprobs = _get_topk_logprobs(logprob,
386
- sampling_params.logprobs)
387
-
388
- # Build the output.
389
- for next_token_id in next_token_ids:
390
- output_logprobs = next_logprobs.copy()
391
- output_logprobs[next_token_id] = logprob[next_token_id].item()
392
- seq_group_outputs.append(
393
- SequenceOutputs(parent_seq_id, next_token_id,
394
- output_logprobs))
457
+ sampling_type = sampling_params.sampling_type
458
+ categorized_seq_group_ids[sampling_type].append(i)
459
+ num_seqs = len(seq_ids)
460
+ category_num_tokens[sampling_type] += num_seqs
461
+
462
+ seq_outputs_dict: Dict[int, List[SequenceOutputs]] = {}
463
+ category_start_idx = 0
464
+ for sampling_type in SamplingType:
465
+ seq_group_ids = categorized_seq_group_ids[sampling_type]
466
+ seq_groups = [input_metadata.seq_groups[i] for i in seq_group_ids]
467
+ is_prompts = [i < input_metadata.num_prompts for i in seq_group_ids]
468
+ num_tokens = category_num_tokens[sampling_type]
469
+ if num_tokens == 0:
470
+ continue
471
+ category_logprobs = logprobs[category_start_idx:category_start_idx +
472
+ num_tokens]
473
+ category_probs = probs[category_start_idx:category_start_idx +
474
+ num_tokens]
475
+ if sampling_type == SamplingType.GREEDY:
476
+ sample_results = _greedy_sample(seq_groups, category_logprobs)
477
+ elif sampling_type == SamplingType.RANDOM:
478
+ sample_results = _random_sample(seq_groups, is_prompts,
479
+ category_probs)
480
+ elif sampling_type == SamplingType.BEAM:
481
+ sample_results = _beam_search_sample(seq_groups, is_prompts,
482
+ input_metadata.seq_data,
483
+ category_logprobs)
484
else:
396
- # Generate the next tokens for generation tokens.
485
+ raise ValueError(f"Unsupported sampling type: {sampling_type}")
486
+
487
+ # Batched query for logprobs of selected token
488
+ batched_logprobs_query_seq_indices: List[int] = []
489
+ batched_logprobs_query_token_indices: List[int] = []
490
+ sample_idx = 0
491
+ for seq_group_id, seq_group, sample_result in zip(
492
+ seq_group_ids, seq_groups, sample_results):
493
+ seq_ids, sampling_params = seq_group
494
+ next_token_ids, parent_ids = sample_result
495
num_parent_seqs = len(seq_ids)
398
- prob = probs[idx:idx + num_parent_seqs]
399
- logprob = logprobs[idx:idx + num_parent_seqs]
400
- idx += num_parent_seqs
401
-
402
- # Sample the next tokens.
403
- seq_logprobs = [
404
- input_metadata.seq_data[seq_id].cumulative_logprob
405
- for seq_id in seq_ids
406
- ]
407
- parent_seq_ids, next_token_ids = _sample_from_generation_tokens(
408
- seq_ids, prob, logprob, seq_logprobs, sampling_params)
409
-
410
- # Get top-k log probabilities for the next tokens.
411
- next_logprobs: Dict[int, Dict[int, float]] = {}
412
- for j, seq_id in enumerate(seq_ids):
413
- next_logprobs[seq_id] = _get_topk_logprobs(
414
- logprob[j], sampling_params.logprobs)
415
-
416
- # Build the output.
417
- for parent_seq_id, next_token_id in zip(parent_seq_ids,
418
- next_token_ids):
419
- j = seq_ids.index(parent_seq_id)
420
- output_logprobs = next_logprobs[parent_seq_id].copy()
421
- output_logprobs[next_token_id] = logprob[j,
422
- next_token_id].item()
423
- seq_group_outputs.append(
424
- SequenceOutputs(parent_seq_id, next_token_id,
425
- output_logprobs))
426
- seq_outputs.append(seq_group_outputs)
427
-
428
- return seq_outputs
496
+ batched_logprobs_query_seq_indices.extend(
497
+ [sample_idx + parent_id for parent_id in parent_ids])
498
+ batched_logprobs_query_token_indices.extend(next_token_ids)
499
+ sample_idx += num_parent_seqs
500
+ assert sample_idx == num_tokens
501
+ batched_logprobs_query_result = category_logprobs[[
502
+ batched_logprobs_query_seq_indices,
503
+ batched_logprobs_query_token_indices
504
+ ]].tolist()
505
+
506
+ # Build the sequence outputs.
507
+ sample_idx = 0
508
+ result_idx = 0
509
+ for seq_group_id, seq_group, sample_result in zip(
510
+ seq_group_ids, seq_groups, sample_results):
511
+ seq_ids, sampling_params = seq_group
512
+ next_token_ids, parent_ids = sample_result
513
+ num_results = len(next_token_ids)
514
+ num_parent_seqs = len(seq_ids)
515
+ parent_logprobs = category_logprobs[sample_idx:sample_idx +
516
+ num_parent_seqs]
517
+ selected_token_logprobs = batched_logprobs_query_result[
518
+ result_idx:result_idx + num_results]
519
+ seq_output = _build_sequence_outputs(parent_ids, next_token_ids,
520
+ selected_token_logprobs,
521
+ seq_ids, parent_logprobs,
522
+ sampling_params.logprobs)
523
+ seq_outputs_dict[seq_group_id] = seq_output
524
+ sample_idx += num_parent_seqs
525
+ result_idx += num_results
526
+ assert sample_idx == num_tokens
527
+ category_start_idx += num_tokens
528
+
529
+ return [seq_outputs_dict[i] for i in range(len(input_metadata.seq_groups))]
vllm/sampling_params.py
+16
@@ -1,9 +1,17 @@
1
"""Sampling parameters for text generation."""
2
+from enum import IntEnum
3
+from functools import cached_property
4
from typing import List, Optional, Union
5
6
_SAMPLING_EPS = 1e-5
7
8
9
+class SamplingType(IntEnum):
10
+ GREEDY = 0
11
+ RANDOM = 1
12
+ BEAM = 2
13
+
14
+
15
class SamplingParams:
16
"""Sampling parameters for text generation.
17
@@ -166,6 +174,14 @@ class SamplingParams:
174
if self.top_k != -1:
175
raise ValueError("top_k must be -1 when using greedy sampling.")
176
177
+ @cached_property
178
+ def sampling_type(self) -> SamplingType:
179
+ if self.use_beam_search:
180
+ return SamplingType.BEAM
181
+ if self.temperature < _SAMPLING_EPS:
182
+ return SamplingType.GREEDY
183
+ return SamplingType.RANDOM
184
+
185
def __repr__(self) -> str:
186
return (f"SamplingParams(n={self.n}, "
187
f"best_of={self.best_of}, "