@tduncan / Nvidia-Optimization-Stack-VLLM Mirror · read-only / commits / a7be4d0072
Fork 0

A high-throughput and memory-efficient inference and serving engine for LLMs

Mirrored from https://github.com/tduncan-lum/Nvidia-Optimization-Stack---VLLM.git

Synced Nov 17, 01:45 UTC

[CORE] Improvement in ranks code (#4718)

Swapnil Parekh committed May 12, 2024 at 20:47 UTC a7be4d00725db5ae4f738f70c3a89fd9dedaf7ec
1 file changed +3 -1
vllm/model_executor/layers/sampler.py
+3 -1
@@ -681,7 +681,9 @@ def _get_ranks(x: torch.Tensor, indices: torch.Tensor) -> torch.Tensor:
681 """
682 vals = x[torch.arange(0, len(x), device=x.device, dtype=indices.dtype),
683 indices]
684 - return (x > vals[:, None]).long().sum(1).add_(1)
684 + result = (x > vals[:, None])
685 + del vals
686 + return result.sum(1).add_(1)
687
688
689 def _get_logprobs(