A high-throughput and memory-efficient inference and serving engine for LLMs
Mirrored from https://github.com/tduncan-lum/Nvidia-Optimization-Stack---VLLM.git
Synced Nov 16, 15:45 UTC
[Core] simplify logits resort in _apply_top_k_top_p (#8619)
盏一 committed
Sep 20, 2024 at 02:28 UTC
e42c634acbd1b86b5becca51e8b8108a32a438d5
1 file changed
+3
-6
vllm/model_executor/layers/sampler.py
+3
-6
@@ -433,12 +433,9 @@ def _apply_top_k_top_p(
433
logits_sort.masked_fill_(top_p_mask, -float("inf"))
434
435
# Re-sort the probabilities.
436
- src = torch.arange(logits_idx.shape[-1],
437
- device=logits_idx.device).expand_as(logits_idx)
438
- logits_idx_inv = torch.empty_like(logits_idx).scatter_(dim=-1,
439
- index=logits_idx,
440
- src=src)
441
- logits = torch.gather(logits_sort, dim=-1, index=logits_idx_inv)
436
+ logits = torch.empty_like(logits_sort).scatter_(dim=-1,
437
+ index=logits_idx,
438
+ src=logits_sort)
439
return logits
440
441