A high-throughput and memory-efficient inference and serving engine for LLMs
Mirrored from https://github.com/tduncan-lum/Nvidia-Optimization-Stack---VLLM.git
Synced Nov 17, 05:30 UTC
Make sampler less blocking (#1889)
Antoni Baum committed
Dec 17, 2023 at 07:03 UTC
a7347d9a6d2391734d838ab6a4f3a702e348d9fa
2 files changed
+310
-198
vllm/model_executor/layers/sampler.py
+123
-198
@@ -6,13 +6,11 @@ import torch.nn as nn
6
7
from vllm.model_executor.parallel_utils.communication_op import (
8
tensor_model_parallel_all_gather)
9
-from vllm.model_executor.sampling_metadata import SamplingMetadata
9
+from vllm.model_executor.sampling_metadata import SamplingMetadata, SamplingTensors
10
from vllm.sampling_params import SamplingParams, SamplingType
11
from vllm.sequence import (PromptLogprobs, SampleLogprobs, SamplerOutput,
12
SequenceData, SequenceGroupOutput, SequenceOutput)
13
14
-_SAMPLING_EPS = 1e-5
15
-
14
15
class Sampler(nn.Module):
16
"""Samples the next tokens from the model's outputs.
@@ -32,6 +30,7 @@ class Sampler(nn.Module):
30
def __init__(self, vocab_size: int) -> None:
31
super().__init__()
32
self.vocab_size = vocab_size
33
+ self._copy_stream: torch.cuda.Stream = torch.cuda.Stream()
34
35
def forward(
36
self,
@@ -47,40 +46,38 @@ class Sampler(nn.Module):
46
logits = _get_logits(hidden_states, embedding, embedding_bias,
47
self.vocab_size)
48
49
+ _, vocab_size = logits.shape
50
+
51
# Apply logits processors (if any).
52
logits = _apply_logits_processors(logits, sampling_metadata)
53
+
54
+ # Prepare sampling tensors in another stream to overlap
55
+ # CPU<->GPU data transfer with GPU computation in forward pass.
56
+ with torch.cuda.stream(self._copy_stream):
57
+ (sampling_tensors, do_penalties, do_top_p_top_k,
58
+ do_min_p) = SamplingTensors.from_sampling_metadata(
59
+ sampling_metadata, vocab_size, logits.device, logits.dtype)
60
+
61
+ torch.cuda.current_stream().wait_stream(self._copy_stream)
62
+
63
# Apply presence and frequency penalties.
53
- presence_penalties, frequency_penalties, repetition_penalties = (
54
- _get_penalties(sampling_metadata))
55
- assert len(presence_penalties) == logits.shape[0]
56
- assert len(frequency_penalties) == logits.shape[0]
57
- assert len(repetition_penalties) == logits.shape[0]
58
- logits = _apply_penalties(logits, sampling_metadata,
59
- presence_penalties, frequency_penalties,
60
- repetition_penalties)
64
+ if do_penalties:
65
+ logits = _apply_penalties(logits, sampling_tensors.prompt_tokens,
66
+ sampling_tensors.output_tokens,
67
+ sampling_tensors.presence_penalties,
68
+ sampling_tensors.frequency_penalties,
69
+ sampling_tensors.repetition_penalties)
70
71
# Apply temperature scaling.
63
- temperatures = _get_temperatures(sampling_metadata)
64
- assert len(temperatures) == logits.shape[0]
65
- if any(t != 1.0 for t in temperatures):
66
- t = torch.tensor(temperatures,
67
- dtype=logits.dtype,
68
- device=logits.device)
69
- # Use in-place division to avoid creating a new tensor.
70
- logits.div_(t.unsqueeze(dim=1))
71
-
72
- # Apply top-p and top-k truncation.
73
- top_ps, top_ks, min_ps = _get_top_p_top_k_min_p(
74
- sampling_metadata, self.vocab_size)
75
- assert len(top_ps) == len(top_ks) == logits.shape[0]
76
- do_top_p = any(p < 1.0 - _SAMPLING_EPS for p in top_ps)
77
- do_top_k = any(k != self.vocab_size for k in top_ks)
78
- if do_top_p or do_top_k:
79
- logits = _apply_top_p_top_k(logits, top_ps, top_ks)
80
-
81
- do_min_p = any(mp > _SAMPLING_EPS for mp in min_ps)
72
+ # Use in-place division to avoid creating a new tensor.
73
+ logits.div_(sampling_tensors.temperatures.unsqueeze_(dim=1))
74
+
75
+ if do_top_p_top_k:
76
+ logits = _apply_top_p_top_k(logits, sampling_tensors.top_ps,
77
+ sampling_tensors.top_ks)
78
+
79
if do_min_p:
83
- logits = _apply_min_p(logits, min_ps)
80
+ logits = _apply_min_p(logits, sampling_tensors.min_ps)
81
82
# We use float32 for probabilities and log probabilities.
83
# Compute the probabilities.
@@ -120,32 +117,6 @@ def _prune_hidden_states(
117
sampling_metadata.selected_token_indices)
118
119
123
-def _get_penalties(
124
- sampling_metadata: SamplingMetadata
125
-) -> Tuple[List[float], List[float], List[float]]:
126
- # Collect the presence and frequency penalties.
127
- presence_penalties: List[float] = []
128
- frequency_penalties: List[float] = []
129
- repetition_penalties: List[float] = []
130
- for i, seq_group in enumerate(sampling_metadata.seq_groups):
131
- seq_ids, sampling_params = seq_group
132
- p = sampling_params.presence_penalty
133
- f = sampling_params.frequency_penalty
134
- r = sampling_params.repetition_penalty
135
- if (i < sampling_metadata.num_prompts
136
- and sampling_params.prompt_logprobs is not None):
137
- # NOTE: We do not apply presence and frequency penalties for the
138
- # prompt token positions where we don't sample new tokens.
139
- prompt_len = sampling_metadata.prompt_lens[i]
140
- presence_penalties += [0] * (prompt_len - 1)
141
- frequency_penalties += [0] * (prompt_len - 1)
142
- repetition_penalties += [1] * (prompt_len - 1)
143
- presence_penalties += [p] * len(seq_ids)
144
- frequency_penalties += [f] * len(seq_ids)
145
- repetition_penalties += [r] * len(seq_ids)
146
- return presence_penalties, frequency_penalties, repetition_penalties
147
-
148
-
120
def _get_prompt_and_output_tokens(
121
sampling_metadata: SamplingMetadata,
122
) -> Tuple[List[List[int]], List[List[int]]]:
@@ -168,25 +139,16 @@ def _get_prompt_and_output_tokens(
139
140
141
def _get_bin_counts_and_mask(
171
- logits: torch.Tensor,
172
- tokens: List[List[int]],
142
+ tokens: torch.Tensor,
143
vocab_size: int,
144
num_seqs: int,
145
) -> Tuple[torch.Tensor, torch.Tensor]:
176
- max_len = max(len(tokens) for tokens in tokens)
177
- padded_tokens = [
178
- tokens + [vocab_size] * (max_len - len(tokens)) for tokens in tokens
179
- ]
180
- tokens_tensor = torch.tensor(padded_tokens,
181
- dtype=torch.long,
182
- device=logits.device)
183
-
146
# Compute the bin counts for the tokens.
147
# vocab_size + 1 for padding.
148
bin_counts = torch.zeros((num_seqs, vocab_size + 1),
149
dtype=torch.long,
188
- device=logits.device)
189
- bin_counts.scatter_add_(1, tokens_tensor, torch.ones_like(tokens_tensor))
150
+ device=tokens.device)
151
+ bin_counts.scatter_add_(1, tokens, torch.ones_like(tokens))
152
bin_counts = bin_counts[:, :vocab_size]
153
mask = bin_counts > 0
154
@@ -217,45 +179,16 @@ def _apply_logits_processors(
179
return logits
180
181
220
-def _apply_penalties(
221
- logits: torch.Tensor,
222
- sampling_metadata: SamplingMetadata,
223
- presence_penalties: List[float],
224
- frequency_penalties: List[float],
225
- repetition_penalties: List[float],
226
-) -> torch.Tensor:
182
+def _apply_penalties(logits: torch.Tensor, prompt_tokens_tensor: torch.Tensor,
183
+ output_tokens_tensor: torch.Tensor,
184
+ presence_penalties: torch.Tensor,
185
+ frequency_penalties: torch.Tensor,
186
+ repetition_penalties: torch.Tensor) -> torch.Tensor:
187
num_seqs, vocab_size = logits.shape
228
- for i in range(num_seqs):
229
- p = presence_penalties[i]
230
- f = frequency_penalties[i]
231
- r = repetition_penalties[i]
232
- if abs(p) < _SAMPLING_EPS and abs(f) < _SAMPLING_EPS and abs(
233
- r - 1.0) < _SAMPLING_EPS:
234
- continue
235
- break
236
- else:
237
- # Return early if all sequences have zero penalties.
238
- return logits
239
-
240
- prompt_tokens, output_tokens = (
241
- _get_prompt_and_output_tokens(sampling_metadata))
242
- assert len(prompt_tokens) == logits.shape[0]
243
- assert len(output_tokens) == logits.shape[0]
244
-
245
- prompt_bin_counts, prompt_mask = _get_bin_counts_and_mask(
246
- logits, prompt_tokens, vocab_size, num_seqs)
188
+ _, prompt_mask = _get_bin_counts_and_mask(prompt_tokens_tensor, vocab_size,
189
+ num_seqs)
190
output_bin_counts, output_mask = _get_bin_counts_and_mask(
248
- logits, output_tokens, vocab_size, num_seqs)
249
-
250
- repetition_penalties = torch.tensor(repetition_penalties,
251
- dtype=logits.dtype,
252
- device=logits.device)
253
- frequency_penalties = torch.tensor(frequency_penalties,
254
- dtype=logits.dtype,
255
- device=logits.device)
256
- presence_penalties = torch.tensor(presence_penalties,
257
- dtype=logits.dtype,
258
- device=logits.device)
191
+ output_tokens_tensor, vocab_size, num_seqs)
192
193
repetition_penalties = repetition_penalties[:, None].repeat(1, vocab_size)
194
repetition_penalties[~(prompt_mask | output_mask)] = 1.0
@@ -264,109 +197,65 @@ def _apply_penalties(
197
198
# We follow the definition in OpenAI API.
199
# Refer to https://platform.openai.com/docs/api-reference/parameter-details
267
- logits -= frequency_penalties.unsqueeze(dim=1) * output_bin_counts
268
- logits -= presence_penalties.unsqueeze(dim=1) * output_mask
200
+ logits -= frequency_penalties.unsqueeze_(dim=1) * output_bin_counts
201
+ logits -= presence_penalties.unsqueeze_(dim=1) * output_mask
202
return logits
203
204
272
-def _get_temperatures(sampling_metadata: SamplingMetadata) -> List[float]:
273
- # Collect the temperatures for the logits.
274
- temperatures: List[float] = []
275
- for i, seq_group in enumerate(sampling_metadata.seq_groups):
276
- seq_ids, sampling_params = seq_group
277
- temperature = sampling_params.temperature
278
- if temperature < _SAMPLING_EPS:
279
- # NOTE: Zero temperature means deterministic sampling
280
- # (i.e., greedy sampling or beam search).
281
- # Set the temperature to 1 to avoid division by zero.
282
- temperature = 1.0
283
- if (i < sampling_metadata.num_prompts
284
- and sampling_params.prompt_logprobs is not None):
285
- prompt_len = sampling_metadata.prompt_lens[i]
286
- temperatures += [temperature] * (prompt_len - 1)
287
- temperatures += [temperature] * len(seq_ids)
288
- return temperatures
289
-
290
-
291
-def _get_top_p_top_k_min_p(
292
- sampling_metadata: SamplingMetadata,
293
- vocab_size: int,
294
-) -> Tuple[List[float], List[int], List[float]]:
295
- top_ps: List[float] = []
296
- top_ks: List[int] = []
297
- min_ps: List[float] = []
298
- for i, seq_group in enumerate(sampling_metadata.seq_groups):
299
- seq_ids, sampling_params = seq_group
300
- top_p = sampling_params.top_p
301
- min_p = sampling_params.min_p
302
- # k should not be greater than the vocab size.
303
- top_k = min(sampling_params.top_k, vocab_size)
304
- # k=-1 means no truncation.
305
- top_k = vocab_size if top_k == -1 else top_k
306
- if (i < sampling_metadata.num_prompts
307
- and sampling_params.prompt_logprobs is not None):
308
- prompt_len = sampling_metadata.prompt_lens[i]
309
- top_ps += [top_p] * (prompt_len - 1)
310
- top_ks += [top_k] * (prompt_len - 1)
311
- min_ps += [min_p] * (prompt_len - 1)
312
- top_ps += [top_p] * len(seq_ids)
313
- top_ks += [top_k] * len(seq_ids)
314
- min_ps += [min_p] * len(seq_ids)
315
- return top_ps, top_ks, min_ps
316
-
317
-
205
def _apply_top_p_top_k(
206
logits: torch.Tensor,
320
- top_ps: List[float],
321
- top_ks: List[int],
207
+ p: torch.Tensor,
208
+ k: torch.Tensor,
209
) -> torch.Tensor:
323
- p = torch.tensor(top_ps, dtype=logits.dtype, device=logits.device)
324
- k = torch.tensor(top_ks, dtype=torch.int, device=logits.device)
210
logits_sort, logits_idx = logits.sort(dim=-1, descending=True)
211
212
# Apply top-p.
213
probs_sort = logits_sort.softmax(dim=-1)
329
- probs_sum = probs_sort.cumsum(dim=-1)
330
- top_p_mask = (probs_sum - probs_sort) > p.unsqueeze(dim=1)
331
- logits_sort[top_p_mask] = -float("inf")
214
+ probs_sum = probs_sort.cumsum(dim=-1).sub_(probs_sort)
215
+ top_p_mask = probs_sum > p.unsqueeze_(dim=1)
216
217
# Apply top-k.
218
# Create a mask for the top-k elements.
219
top_k_mask = torch.arange(logits_idx.shape[-1], device=logits_idx.device)
220
top_k_mask = top_k_mask.expand(logits_idx.shape[0], -1)
337
- top_k_mask = top_k_mask >= k.unsqueeze(dim=1)
338
- logits_sort[top_k_mask] = -float("inf")
221
+ top_k_mask = top_k_mask >= k.unsqueeze_(dim=1)
222
+
223
+ # Final mask.
224
+ mask = (top_p_mask | top_k_mask)
225
+ logits_sort.masked_fill_(mask, -float("inf"))
226
227
# Re-sort the probabilities.
341
- logits = torch.gather(logits_sort,
342
- dim=-1,
343
- index=torch.argsort(logits_idx, dim=-1))
228
+ src = torch.arange(logits_idx.shape[-1],
229
+ device=logits_idx.device).expand_as(logits_idx)
230
+ logits_idx_inv = torch.empty_like(logits_idx).scatter_(dim=-1,
231
+ index=logits_idx,
232
+ src=src)
233
+ logits = torch.gather(logits_sort, dim=-1, index=logits_idx_inv)
234
return logits
235
236
237
def _apply_min_p(
238
logits: torch.Tensor,
349
- min_ps: List[float],
239
+ min_p: torch.Tensor,
240
) -> torch.Tensor:
241
"""
242
Adapted from
243
https://github.com/oobabooga/text-generation-webui/blob/3146124ec01f02c8fb1650a6517cf1b60b537aaf/modules/sampler_hijack.py#L16C17-L16C17
244
"""
355
- min_p = torch.tensor(min_ps, dtype=logits.dtype, device=logits.device)
245
probs = torch.softmax(logits, dim=-1)
246
top_probs, _ = probs.max(dim=-1, keepdim=True)
358
- scaled_min_p = min_p.unsqueeze(dim=1) * top_probs
247
+ scaled_min_p = min_p.unsqueeze_(dim=1) * top_probs
248
tokens_to_remove = probs < scaled_min_p
360
- logits = logits.masked_fill(tokens_to_remove, -float("inf"))
249
+ logits = logits.masked_fill_(tokens_to_remove, -float("inf"))
250
251
return logits
252
253
254
def _greedy_sample(
255
selected_seq_groups: List[Tuple[List[int], SamplingParams]],
367
- logprobs: torch.Tensor,
256
+ samples: torch.Tensor,
257
) -> List[Tuple[List[int], List[int]]]:
369
- samples = torch.argmax(logprobs, dim=-1).cpu()
258
+ samples = samples.tolist()
259
sample_idx = 0
260
results = []
261
for seq_group in selected_seq_groups:
@@ -375,27 +264,19 @@ def _greedy_sample(
264
assert num_parent_seqs == 1, (
265
"Greedy sampling should have only one seq.")
266
parent_ids = list(range(num_parent_seqs))
378
- next_token_ids = [samples[sample_idx].item()]
267
+ next_token_ids = [samples[sample_idx]]
268
results.append((next_token_ids, parent_ids))
269
sample_idx += num_parent_seqs
381
- assert sample_idx == logprobs.size(0)
270
return results
271
272
273
def _random_sample(
274
selected_seq_groups: List[Tuple[List[int], SamplingParams]],
275
is_prompts: List[bool],
388
- probs: torch.Tensor,
276
+ random_samples: torch.Tensor,
277
) -> List[Tuple[List[int], List[int]]]:
278
# Find the maximum best_of value of the prompt phase requests.
391
- max_best_of = 1
392
- for seq_group, is_prompt in zip(selected_seq_groups, is_prompts):
393
- if is_prompt:
394
- seq_ids, sampling_params = seq_group
395
- max_best_of = max(max_best_of, sampling_params.best_of)
396
- random_samples = torch.multinomial(probs,
397
- num_samples=max_best_of,
398
- replacement=True).cpu()
279
+ random_samples = random_samples.cpu()
280
sample_idx = 0
281
results = []
282
for seq_group, is_prompt in zip(selected_seq_groups, is_prompts):
@@ -403,8 +284,6 @@ def _random_sample(
284
num_parent_seqs = len(seq_ids)
285
if is_prompt:
286
# Prompt phase.
406
- assert num_parent_seqs == 1, (
407
- "Prompt input should have only one seq.")
287
parent_ids = [0] * sampling_params.best_of
288
next_token_ids = random_samples[
289
sample_idx, :sampling_params.best_of].tolist()
@@ -415,7 +294,6 @@ def _random_sample(
294
num_parent_seqs, 0].tolist()
295
results.append((next_token_ids, parent_ids))
296
sample_idx += num_parent_seqs
418
- assert sample_idx == probs.size(0)
297
return results
298
299
@@ -472,6 +350,28 @@ def _beam_search_sample(
350
return results
351
352
353
+# torch.multinomial forces a GPU<->CPU sync.
354
+# Therefore, we use an optimized implementation instead.
355
+# Note that we always sample with replacement.
356
+# probs will be modified in place, but this is fine, as we pass
357
+# in a copy already.
358
+def _multinomial(
359
+ probs: torch.Tensor,
360
+ num_samples: int,
361
+):
362
+ if num_samples > 1:
363
+ # This is equivalent to torch.repeat_interleaved (which also
364
+ # forces a GPU<->CPU sync).
365
+ # This allows us to do sampling with replacement by creating
366
+ # num_samples copies of each row in the tensor, and then
367
+ # batch sampling the resulting tensor.
368
+ probs = probs[:, None, :].expand(probs.shape[0], num_samples,
369
+ probs.shape[1]).contiguous().view(
370
+ -1, probs.shape[1])
371
+ q = torch.empty_like(probs).exponential_(1)
372
+ return probs.div_(q).argmax(dim=1).view(-1, num_samples)
373
+
374
+
375
def _sample(
376
probs: torch.Tensor,
377
logprobs: torch.Tensor,
@@ -485,28 +385,51 @@ def _sample(
385
categorized_seq_group_ids[sampling_type].append(i)
386
387
sample_results_dict: Dict[int, Tuple[List[int], List[int]]] = {}
388
+ sample_metadata = {}
389
+
390
+ # Counterintiutively, having two loops here is actually faster.
391
+ # The first loop can run without waiting on GPU<->CPU sync.
392
for sampling_type in SamplingType:
489
- seq_group_ids = categorized_seq_group_ids[sampling_type]
490
- seq_groups = [sampling_metadata.seq_groups[i] for i in seq_group_ids]
491
- is_prompts = [i < sampling_metadata.num_prompts for i in seq_group_ids]
393
sample_indices = categorized_sample_indices[sampling_type]
394
num_tokens = len(sample_indices)
395
if num_tokens == 0:
396
continue
397
+ seq_group_ids = categorized_seq_group_ids[sampling_type]
398
+ seq_groups = [sampling_metadata.seq_groups[i] for i in seq_group_ids]
399
+ is_prompts = [i < sampling_metadata.num_prompts for i in seq_group_ids]
400
+ sample_metadata[sampling_type] = (seq_group_ids, seq_groups,
401
+ is_prompts, sample_indices)
402
if sampling_type == SamplingType.GREEDY:
497
- category_logprobs = logprobs[sample_indices]
498
- sample_results = _greedy_sample(seq_groups, category_logprobs)
403
+ greedy_samples = torch.argmax(logprobs[sample_indices], dim=-1)
404
+ elif sampling_type == SamplingType.RANDOM:
405
+ max_best_of = 1
406
+ for seq_group, is_prompt in zip(seq_groups, is_prompts):
407
+ if is_prompt:
408
+ _, sampling_params = seq_group
409
+ max_best_of = max(max_best_of, sampling_params.best_of)
410
+ multinomial_samples = _multinomial(probs[sample_indices],
411
+ max_best_of)
412
+ elif sampling_type == SamplingType.BEAM:
413
+ beam_search_logprobs = logprobs[sample_indices]
414
+ else:
415
+ raise ValueError(f"Unsupported sampling type: {sampling_type}")
416
+
417
+ # GPU<->CPU sync happens in the loop below.
418
+
419
+ for sampling_type in SamplingType:
420
+ if sampling_type not in sample_metadata:
421
+ continue
422
+ seq_group_ids, seq_groups, is_prompts, sample_indices = sample_metadata[
423
+ sampling_type]
424
+ if sampling_type == SamplingType.GREEDY:
425
+ sample_results = _greedy_sample(seq_groups, greedy_samples)
426
elif sampling_type == SamplingType.RANDOM:
500
- category_probs = probs[sample_indices]
427
sample_results = _random_sample(seq_groups, is_prompts,
502
- category_probs)
428
+ multinomial_samples)
429
elif sampling_type == SamplingType.BEAM:
504
- category_logprobs = logprobs[sample_indices]
430
sample_results = _beam_search_sample(seq_groups, is_prompts,
431
sampling_metadata.seq_data,
507
- category_logprobs)
508
- else:
509
- raise ValueError(f"Unsupported sampling type: {sampling_type}")
432
+ beam_search_logprobs)
433
sample_results_dict.update(zip(seq_group_ids, sample_results))
434
435
sample_results = [
@@ -557,7 +480,7 @@ def _get_logprobs(
480
batched_logprobs_query_result = logprobs[[
481
batched_logprobs_query_seq_indices,
482
batched_logprobs_query_token_indices
560
- ]].cpu()
483
+ ]]
484
485
# Batched query for logprobs of topk tokens
486
if largest_num_logprobs > 0:
@@ -569,6 +492,8 @@ def _get_logprobs(
492
else:
493
top_logprobs, top_token_ids = None, None
494
495
+ batched_logprobs_query_result = batched_logprobs_query_result.cpu()
496
+
497
# Gather results
498
result_prompt_logprobs: List[Optional[PromptLogprobs]] = []
499
result_sample_logprobs: List[SampleLogprobs] = []
vllm/model_executor/sampling_metadata.py
+187
@@ -1,9 +1,13 @@
1
+from dataclasses import dataclass
2
from typing import Dict, List, Tuple
3
4
import torch
5
6
from vllm.sampling_params import SamplingParams, SamplingType
7
from vllm.sequence import SequenceData
8
+from vllm.utils import in_wsl
9
+
10
+_SAMPLING_EPS = 1e-5
11
12
13
class SamplingMetadata:
@@ -41,3 +45,186 @@ class SamplingMetadata:
45
f"prompt_lens={self.prompt_lens}, "
46
f"selected_token_indices={self.selected_token_indices}, "
47
f"categorized_sample_indices={self.categorized_sample_indices})")
48
+
49
+
50
+@dataclass
51
+class SamplingTensors:
52
+ """Tensors for sampling."""
53
+
54
+ temperatures: torch.Tensor
55
+ top_ps: torch.Tensor
56
+ top_ks: torch.Tensor
57
+ min_ps: torch.Tensor
58
+ presence_penalties: torch.Tensor
59
+ frequency_penalties: torch.Tensor
60
+ repetition_penalties: torch.Tensor
61
+ prompt_tokens: torch.Tensor
62
+ output_tokens: torch.Tensor
63
+
64
+ @classmethod
65
+ def from_sampling_metadata(
66
+ cls, sampling_metadata: "SamplingMetadata", vocab_size: int,
67
+ device: torch.device,
68
+ dtype: torch.dtype) -> Tuple["SamplingTensors", bool, bool, bool]:
69
+ prompt_tokens: List[List[int]] = []
70
+ output_tokens: List[List[int]] = []
71
+ top_ks: List[int] = []
72
+ temperatures: List[float] = []
73
+ top_ps: List[float] = []
74
+ min_ps: List[float] = []
75
+ presence_penalties: List[float] = []
76
+ frequency_penalties: List[float] = []
77
+ repetition_penalties: List[float] = []
78
+ do_penalties = False
79
+ do_top_p_top_k = False
80
+ do_min_p = False
81
+ for i, seq_group in enumerate(sampling_metadata.seq_groups):
82
+ seq_ids, sampling_params = seq_group
83
+ temperature = sampling_params.temperature
84
+ p = sampling_params.presence_penalty
85
+ f = sampling_params.frequency_penalty
86
+ r = sampling_params.repetition_penalty
87
+ top_p = sampling_params.top_p
88
+ min_p = sampling_params.min_p
89
+ # k should not be greater than the vocab size.
90
+ top_k = min(sampling_params.top_k, vocab_size)
91
+ top_k = vocab_size if top_k == -1 else top_k
92
+ if temperature < _SAMPLING_EPS:
93
+ # NOTE: Zero temperature means deterministic sampling
94
+ # (i.e., greedy sampling or beam search).
95
+ # Set the temperature to 1 to avoid division by zero.
96
+ temperature = 1.0
97
+ if not do_top_p_top_k and (top_p < 1.0 - _SAMPLING_EPS
98
+ or top_k != vocab_size):
99
+ do_top_p_top_k = True
100
+ if not do_min_p and min_p > _SAMPLING_EPS:
101
+ do_min_p = True
102
+ if not do_penalties and (abs(p) >= _SAMPLING_EPS
103
+ or abs(f) >= _SAMPLING_EPS
104
+ or abs(r - 1.0) >= _SAMPLING_EPS):
105
+ do_penalties = True
106
+ if (i < sampling_metadata.num_prompts
107
+ and sampling_params.prompt_logprobs is not None):
108
+ # For tokens in the prompt that we only need to get their logprobs
109
+ prompt_len = sampling_metadata.prompt_lens[i]
110
+ temperatures += [temperature] * (prompt_len - 1)
111
+ top_ps += [top_p] * (prompt_len - 1)
112
+ top_ks += [top_k] * (prompt_len - 1)
113
+ min_ps += [min_p] * (prompt_len - 1)
114
+ presence_penalties += [0] * (prompt_len - 1)
115
+ frequency_penalties += [0] * (prompt_len - 1)
116
+ repetition_penalties += [1] * (prompt_len - 1)
117
+ prompt_tokens.extend([] for _ in range(prompt_len - 1))
118
+ output_tokens.extend([] for _ in range(prompt_len - 1))
119
+ for seq_id in seq_ids:
120
+ seq_data = sampling_metadata.seq_data[seq_id]
121
+ prompt_tokens.append(seq_data.prompt_token_ids)
122
+ output_tokens.append(seq_data.output_token_ids)
123
+ temperatures += [temperature] * len(seq_ids)
124
+ top_ps += [top_p] * len(seq_ids)
125
+ top_ks += [top_k] * len(seq_ids)
126
+ min_ps += [min_p] * len(seq_ids)
127
+ presence_penalties += [p] * len(seq_ids)
128
+ frequency_penalties += [f] * len(seq_ids)
129
+ repetition_penalties += [r] * len(seq_ids)
130
+
131
+ sampling_tensors = SamplingTensors.from_lists(
132
+ temperatures, top_ps, top_ks, min_ps, presence_penalties,
133
+ frequency_penalties, repetition_penalties, prompt_tokens,
134
+ output_tokens, vocab_size, device, dtype)
135
+ return (sampling_tensors, do_penalties, do_top_p_top_k, do_min_p)
136
+
137
+ @classmethod
138
+ def from_lists(cls, temperatures: List[float], top_ps: List[float],
139
+ top_ks: List[int], min_ps: List[float],
140
+ presence_penalties: List[float],
141
+ frequency_penalties: List[float],
142
+ repetition_penalties: List[float],
143
+ prompt_tokens: List[List[int]],
144
+ output_tokens: List[List[int]], vocab_size: int,
145
+ device: torch.device,
146
+ dtype: torch.dtype) -> "SamplingTensors":
147
+ # Note that the performance will be very bad without
148
+ # pinned memory.
149
+ pin_memory = not in_wsl()
150
+ prompt_max_len = max(len(tokens) for tokens in prompt_tokens)
151
+ prompt_padded_tokens = [
152
+ tokens + [vocab_size] * (prompt_max_len - len(tokens))
153
+ for tokens in prompt_tokens
154
+ ]
155
+ output_max_len = max(len(tokens) for tokens in output_tokens)
156
+ output_padded_tokens = [
157
+ tokens + [vocab_size] * (output_max_len - len(tokens))
158
+ for tokens in output_tokens
159
+ ]
160
+
161
+ temperatures_t = torch.tensor(
162
+ temperatures,
163
+ device="cpu",
164
+ dtype=dtype,
165
+ pin_memory=pin_memory,
166
+ )
167
+ top_ps_t = torch.tensor(
168
+ top_ps,
169
+ device="cpu",
170
+ dtype=dtype,
171
+ pin_memory=pin_memory,
172
+ )
173
+ min_ps_t = torch.tensor(
174
+ min_ps,
175
+ device="cpu",
176
+ dtype=dtype,
177
+ pin_memory=pin_memory,
178
+ )
179
+ presence_penalties_t = torch.tensor(
180
+ presence_penalties,
181
+ device="cpu",
182
+ dtype=dtype,
183
+ pin_memory=pin_memory,
184
+ )
185
+ frequency_penalties_t = torch.tensor(
186
+ frequency_penalties,
187
+ device="cpu",
188
+ dtype=dtype,
189
+ pin_memory=pin_memory,
190
+ )
191
+ repetition_penalties_t = torch.tensor(
192
+ repetition_penalties,
193
+ device="cpu",
194
+ dtype=dtype,
195
+ pin_memory=pin_memory,
196
+ )
197
+ top_ks_t = torch.tensor(
198
+ top_ks,
199
+ device="cpu",
200
+ dtype=torch.int,
201
+ pin_memory=pin_memory,
202
+ )
203
+ prompt_tensor = torch.tensor(
204
+ prompt_padded_tokens,
205
+ device="cpu",
206
+ dtype=torch.long,
207
+ pin_memory=pin_memory,
208
+ )
209
+ output_tensor = torch.tensor(
210
+ output_padded_tokens,
211
+ device="cpu",
212
+ dtype=torch.long,
213
+ pin_memory=pin_memory,
214
+ )
215
+ # Because the memory is pinned, we can do non-blocking
216
+ # transfer to device.
217
+ return cls(
218
+ temperatures=temperatures_t.to(device=device, non_blocking=True),
219
+ top_ps=top_ps_t.to(device=device, non_blocking=True),
220
+ top_ks=top_ks_t.to(device=device, non_blocking=True),
221
+ min_ps=min_ps_t.to(device=device, non_blocking=True),
222
+ presence_penalties=presence_penalties_t.to(device=device,
223
+ non_blocking=True),
224
+ frequency_penalties=frequency_penalties_t.to(device=device,
225
+ non_blocking=True),
226
+ repetition_penalties=repetition_penalties_t.to(device=device,
227
+ non_blocking=True),
228
+ prompt_tokens=prompt_tensor.to(device=device, non_blocking=True),
229
+ output_tokens=output_tensor.to(device=device, non_blocking=True),
230
+ )