@tduncan / Nvidia-Optimization-Stack-VLLM Mirror · read-only / commits / 3a243095e5
Fork 0

A high-throughput and memory-efficient inference and serving engine for LLMs

Mirrored from https://github.com/tduncan-lum/Nvidia-Optimization-Stack---VLLM.git

Synced Nov 17, 04:15 UTC

Optimize `_get_ranks` in Sampler (#3623)

Antoni Baum committed Mar 25, 2024 at 16:03 UTC 3a243095e5e7b655b63ab08fbd5936cb40850415
1 file changed +17 -10
vllm/model_executor/layers/sampler.py
+17 -10
@@ -506,22 +506,23 @@ def _sample(
506 # sampling_tensors)
507
508
509 -def _get_ranks(x: torch.Tensor, indices: List[int]) -> torch.Tensor:
509 +def _get_ranks(x: torch.Tensor, indices: torch.Tensor) -> torch.Tensor:
510 """
511 This function calculates the ranks of the chosen tokens in a logprob tensor.
512
513 Args:
514 x (torch.Tensor): 2D logprob tensor of shape (N, M)
515 where N is the no. of tokens and M is the vocab dim.
516 - indices (List[int]): List of chosen token indices.
516 + indices (torch.Tensor): List of chosen token indices.
517
518 Returns:
519 torch.Tensor: 1D tensor of shape (N,) where N is the no. of tokens.
520 Each element in the returned tensor represents the rank
521 of the chosen token in the input logprob tensor.
522 """
523 - vals = x[range(len(x)), indices]
524 - return (x > vals[:, None]).long().sum(1) + 1
523 + vals = x[torch.arange(0, len(x), device=x.device, dtype=indices.dtype),
524 + indices]
525 + return (x > vals[:, None]).long().sum(1).add_(1)
526
527
528 def _get_logprobs(
@@ -561,12 +562,21 @@ def _get_logprobs(
562 sample_idx += num_parent_seqs
563 assert sample_idx == logprobs.size(0)
564
565 + batched_logprobs_query_seq_indices_gpu = torch.tensor(
566 + batched_logprobs_query_seq_indices, device=logprobs.device)
567 + batched_logprobs_query_token_indices_gpu = torch.tensor(
568 + batched_logprobs_query_token_indices, device=logprobs.device)
569 +
570 # Batched query for logprobs of selected token
571 batched_logprobs_query_result = logprobs[[
566 - batched_logprobs_query_seq_indices,
567 - batched_logprobs_query_token_indices
572 + batched_logprobs_query_seq_indices_gpu,
573 + batched_logprobs_query_token_indices_gpu
574 ]]
575
576 + batched_ranks_query_result = _get_ranks(
577 + logprobs[batched_logprobs_query_seq_indices_gpu],
578 + batched_logprobs_query_token_indices_gpu)
579 +
580 # Batched query for logprobs of topk tokens
581 if largest_num_logprobs > 0:
582 top_logprobs, top_token_ids = torch.topk(logprobs,
@@ -578,10 +588,7 @@ def _get_logprobs(
588 top_logprobs, top_token_ids = None, None
589
590 batched_logprobs_query_result = batched_logprobs_query_result.cpu()
581 -
582 - batched_ranks_query_result = _get_ranks(
583 - logprobs[batched_logprobs_query_seq_indices],
584 - batched_logprobs_query_token_indices)
591 + batched_ranks_query_result = batched_ranks_query_result.cpu()
592
593 # Gather results
594 result_prompt_logprobs: List[Optional[PromptLogprobs]] = []