A high-throughput and memory-efficient inference and serving engine for LLMs
Mirrored from https://github.com/tduncan-lum/Nvidia-Optimization-Stack---VLLM.git
Synced Nov 17, 01:45 UTC
[CORE] Improvement in ranks code (#4718)
Swapnil Parekh committed
May 12, 2024 at 20:47 UTC
a7be4d00725db5ae4f738f70c3a89fd9dedaf7ec
1 file changed
+3
-1
vllm/model_executor/layers/sampler.py
+3
-1
@@ -681,7 +681,9 @@ def _get_ranks(x: torch.Tensor, indices: torch.Tensor) -> torch.Tensor:
681
"""
682
vals = x[torch.arange(0, len(x), device=x.device, dtype=indices.dtype),
683
indices]
684
- return (x > vals[:, None]).long().sum(1).add_(1)
684
+ result = (x > vals[:, None])
685
+ del vals
686
+ return result.sum(1).add_(1)
687
688
689
def _get_logprobs(