@tduncan / Nvidia-Optimization-Stack-VLLM Mirror · read-only / commits / d1744376ae
Fork 0

A high-throughput and memory-efficient inference and serving engine for LLMs

Mirrored from https://github.com/tduncan-lum/Nvidia-Optimization-Stack---VLLM.git

Synced Nov 17, 08:00 UTC

Align with huggingface Top K sampling (#753)

Abraham-Xu committed Aug 16, 2023 at 07:44 UTC d1744376ae9fdbfa6a2dc763e1c67309e138fa3d
1 file changed +23 -24
vllm/model_executor/layers/sampler.py
+23 -24
@@ -71,19 +71,19 @@ class Sampler(nn.Module):
71 # Use in-place division to avoid creating a new tensor.
72 logits.div_(t.unsqueeze(dim=1))
73
74 - # We use float32 for probabilities and log probabilities.
75 - # Compute the probabilities.
76 - probs = torch.softmax(logits, dim=-1, dtype=torch.float)
77 - # Compute the log probabilities (before applying top-p and top-k).
78 - logprobs = torch.log(probs)
79 -
74 # Apply top-p and top-k truncation.
75 top_ps, top_ks = _get_top_p_top_k(input_metadata, self.vocab_size)
82 - assert len(top_ps) == len(top_ks) == probs.shape[0]
76 + assert len(top_ps) == len(top_ks) == logits.shape[0]
77 do_top_p = any(p < 1.0 - _SAMPLING_EPS for p in top_ps)
78 do_top_k = any(k != self.vocab_size for k in top_ks)
79 if do_top_p or do_top_k:
86 - probs = _apply_top_p_top_k(probs, top_ps, top_ks)
80 + logits = _apply_top_p_top_k(logits, top_ps, top_ks)
81 +
82 + # We use float32 for probabilities and log probabilities.
83 + # Compute the probabilities.
84 + probs = torch.softmax(logits, dim=-1, dtype=torch.float)
85 + # Compute the log probabilities (before applying top-p and top-k).
86 + logprobs = torch.log(probs)
87
88 # Sample the next tokens.
89 return _sample(probs, logprobs, input_metadata)
@@ -235,31 +235,32 @@ def _get_top_p_top_k(
235
236
237 def _apply_top_p_top_k(
238 - probs: torch.Tensor,
238 + logits: torch.Tensor,
239 top_ps: List[float],
240 top_ks: List[int],
241 ) -> torch.Tensor:
242 - p = torch.tensor(top_ps, dtype=probs.dtype, device=probs.device)
243 - k = torch.tensor(top_ks, dtype=torch.int, device=probs.device)
244 - probs_sort, probs_idx = probs.sort(dim=-1, descending=True)
242 + p = torch.tensor(top_ps, dtype=logits.dtype, device=logits.device)
243 + k = torch.tensor(top_ks, dtype=torch.int, device=logits.device)
244 + logits_sort, logits_idx = logits.sort(dim=-1, descending=True)
245
246 # Apply top-p.
247 - probs_sum = torch.cumsum(probs_sort, dim=-1)
247 + probs_sort = logits_sort.softmax(dim=-1)
248 + probs_sum = probs_sort.cumsum(dim=-1)
249 top_p_mask = (probs_sum - probs_sort) > p.unsqueeze(dim=1)
249 - probs_sort[top_p_mask] = 0.0
250 + logits_sort[top_p_mask] = -float("inf")
251
252 # Apply top-k.
253 # Create a mask for the top-k elements.
253 - top_k_mask = torch.arange(probs_idx.shape[-1], device=probs_idx.device)
254 - top_k_mask = top_k_mask.expand(probs_idx.shape[0], -1)
254 + top_k_mask = torch.arange(logits_idx.shape[-1], device=logits_idx.device)
255 + top_k_mask = top_k_mask.expand(logits_idx.shape[0], -1)
256 top_k_mask = top_k_mask >= k.unsqueeze(dim=1)
256 - probs_sort[top_k_mask] = 0.0
257 + logits_sort[top_k_mask] = -float("inf")
258
259 # Re-sort the probabilities.
259 - probs = torch.gather(probs_sort,
260 - dim=-1,
261 - index=torch.argsort(probs_idx, dim=-1))
262 - return probs
260 + logits = torch.gather(logits_sort,
261 + dim=-1,
262 + index=torch.argsort(logits_idx, dim=-1))
263 + return logits
264
265
266 def _get_topk_logprobs(
@@ -301,9 +302,7 @@ def _sample_from_prompt(
302 # Random sampling.
303 # Sample `best_of` tokens for the prompt.
304 num_seqs = sampling_params.best_of
304 - next_token_ids = torch.multinomial(prob,
305 - num_samples=num_seqs,
306 - replacement=True)
305 + next_token_ids = torch.multinomial(prob, num_samples=num_seqs)
306 next_token_ids = next_token_ids.tolist()
307 return next_token_ids
308