A high-throughput and memory-efficient inference and serving engine for LLMs
Mirrored from https://github.com/tduncan-lum/Nvidia-Optimization-Stack---VLLM.git
Synced Nov 17, 08:00 UTC
Align with huggingface Top K sampling (#753)
Abraham-Xu committed
Aug 16, 2023 at 07:44 UTC
d1744376ae9fdbfa6a2dc763e1c67309e138fa3d
1 file changed
+23
-24
vllm/model_executor/layers/sampler.py
+23
-24
@@ -71,19 +71,19 @@ class Sampler(nn.Module):
71
# Use in-place division to avoid creating a new tensor.
72
logits.div_(t.unsqueeze(dim=1))
73
74
- # We use float32 for probabilities and log probabilities.
75
- # Compute the probabilities.
76
- probs = torch.softmax(logits, dim=-1, dtype=torch.float)
77
- # Compute the log probabilities (before applying top-p and top-k).
78
- logprobs = torch.log(probs)
79
-
74
# Apply top-p and top-k truncation.
75
top_ps, top_ks = _get_top_p_top_k(input_metadata, self.vocab_size)
82
- assert len(top_ps) == len(top_ks) == probs.shape[0]
76
+ assert len(top_ps) == len(top_ks) == logits.shape[0]
77
do_top_p = any(p < 1.0 - _SAMPLING_EPS for p in top_ps)
78
do_top_k = any(k != self.vocab_size for k in top_ks)
79
if do_top_p or do_top_k:
86
- probs = _apply_top_p_top_k(probs, top_ps, top_ks)
80
+ logits = _apply_top_p_top_k(logits, top_ps, top_ks)
81
+
82
+ # We use float32 for probabilities and log probabilities.
83
+ # Compute the probabilities.
84
+ probs = torch.softmax(logits, dim=-1, dtype=torch.float)
85
+ # Compute the log probabilities (before applying top-p and top-k).
86
+ logprobs = torch.log(probs)
87
88
# Sample the next tokens.
89
return _sample(probs, logprobs, input_metadata)
@@ -235,31 +235,32 @@ def _get_top_p_top_k(
235
236
237
def _apply_top_p_top_k(
238
- probs: torch.Tensor,
238
+ logits: torch.Tensor,
239
top_ps: List[float],
240
top_ks: List[int],
241
) -> torch.Tensor:
242
- p = torch.tensor(top_ps, dtype=probs.dtype, device=probs.device)
243
- k = torch.tensor(top_ks, dtype=torch.int, device=probs.device)
244
- probs_sort, probs_idx = probs.sort(dim=-1, descending=True)
242
+ p = torch.tensor(top_ps, dtype=logits.dtype, device=logits.device)
243
+ k = torch.tensor(top_ks, dtype=torch.int, device=logits.device)
244
+ logits_sort, logits_idx = logits.sort(dim=-1, descending=True)
245
246
# Apply top-p.
247
- probs_sum = torch.cumsum(probs_sort, dim=-1)
247
+ probs_sort = logits_sort.softmax(dim=-1)
248
+ probs_sum = probs_sort.cumsum(dim=-1)
249
top_p_mask = (probs_sum - probs_sort) > p.unsqueeze(dim=1)
249
- probs_sort[top_p_mask] = 0.0
250
+ logits_sort[top_p_mask] = -float("inf")
251
252
# Apply top-k.
253
# Create a mask for the top-k elements.
253
- top_k_mask = torch.arange(probs_idx.shape[-1], device=probs_idx.device)
254
- top_k_mask = top_k_mask.expand(probs_idx.shape[0], -1)
254
+ top_k_mask = torch.arange(logits_idx.shape[-1], device=logits_idx.device)
255
+ top_k_mask = top_k_mask.expand(logits_idx.shape[0], -1)
256
top_k_mask = top_k_mask >= k.unsqueeze(dim=1)
256
- probs_sort[top_k_mask] = 0.0
257
+ logits_sort[top_k_mask] = -float("inf")
258
259
# Re-sort the probabilities.
259
- probs = torch.gather(probs_sort,
260
- dim=-1,
261
- index=torch.argsort(probs_idx, dim=-1))
262
- return probs
260
+ logits = torch.gather(logits_sort,
261
+ dim=-1,
262
+ index=torch.argsort(logits_idx, dim=-1))
263
+ return logits
264
265
266
def _get_topk_logprobs(
@@ -301,9 +302,7 @@ def _sample_from_prompt(
302
# Random sampling.
303
# Sample `best_of` tokens for the prompt.
304
num_seqs = sampling_params.best_of
304
- next_token_ids = torch.multinomial(prob,
305
- num_samples=num_seqs,
306
- replacement=True)
305
+ next_token_ids = torch.multinomial(prob, num_samples=num_seqs)
306
next_token_ids = next_token_ids.tolist()
307
return next_token_ids
308