@tduncan / Nvidia-Optimization-Stack-VLLM Mirror · read-only / commits / e42c634acb
Fork 0

A high-throughput and memory-efficient inference and serving engine for LLMs

Mirrored from https://github.com/tduncan-lum/Nvidia-Optimization-Stack---VLLM.git

Synced Nov 16, 15:45 UTC

[Core] simplify logits resort in _apply_top_k_top_p (#8619)

盏一 committed Sep 20, 2024 at 02:28 UTC e42c634acbd1b86b5becca51e8b8108a32a438d5
1 file changed +3 -6
vllm/model_executor/layers/sampler.py
+3 -6
@@ -433,12 +433,9 @@ def _apply_top_k_top_p(
433 logits_sort.masked_fill_(top_p_mask, -float("inf"))
434
435 # Re-sort the probabilities.
436 - src = torch.arange(logits_idx.shape[-1],
437 - device=logits_idx.device).expand_as(logits_idx)
438 - logits_idx_inv = torch.empty_like(logits_idx).scatter_(dim=-1,
439 - index=logits_idx,
440 - src=src)
441 - logits = torch.gather(logits_sort, dim=-1, index=logits_idx_inv)
436 + logits = torch.empty_like(logits_sort).scatter_(dim=-1,
437 + index=logits_idx,
438 + src=logits_sort)
439 return logits
440
441