@tduncan / Nvidia-Optimization-Stack-VLLM Mirror · read-only / commits / d7740ea4dc
Fork 0

A high-throughput and memory-efficient inference and serving engine for LLMs

Mirrored from https://github.com/tduncan-lum/Nvidia-Optimization-Stack---VLLM.git

Synced Nov 17, 01:45 UTC

[Core] Optimize sampler get_logprobs (#4594)

SangBin Cho committed May 9, 2024 at 00:42 UTC d7740ea4dcee4ab75d7d6eef723f33cae957b288
1 file changed +68 -49
vllm/model_executor/layers/sampler.py
+68 -49
@@ -782,13 +782,14 @@ def _get_logprobs(
782 top_logprobs, top_token_ids = torch.topk(logprobs,
783 largest_num_logprobs,
784 dim=-1)
785 - top_logprobs = top_logprobs.cpu()
786 - top_token_ids = top_token_ids.cpu()
785 else:
786 top_logprobs, top_token_ids = None, None
787
790 - selected_logprobs = selected_logprobs.cpu()
791 - ranks = ranks.cpu()
788 + selected_logprobs = selected_logprobs.to('cpu')
789 + ranks = ranks.to('cpu')
790 + if top_logprobs is not None and top_token_ids is not None:
791 + top_logprobs = top_logprobs.to('cpu')
792 + top_token_ids = top_token_ids.to('cpu')
793
794 # Find prompt/sample logprobs.
795 prompt_logprobs_per_seq_group: List[Optional[PromptLogprobs]] = []
@@ -828,37 +829,48 @@ def _get_prompt_logprob_if_needed(
829
830 # Find prompt logprobs
831 prompt_logprobs: Optional[PromptLogprobs] = None
831 - if (is_prompt and sampling_params.prompt_logprobs is not None):
832 + if is_prompt and sampling_params.prompt_logprobs is not None:
833 prompt_logprobs = []
834 num_logprobs = sampling_params.prompt_logprobs
835 next_prompt_tokens = _get_next_prompt_tokens(seq_group)
835 - for token_id in next_prompt_tokens:
836 + # Pre-select indexes and create a list. It is faster than calling .item
837 + # repetitively.
838 + selected_logprob_items = selected_logprobs[
839 + selected_logprobs_idx:selected_logprobs_idx +
840 + len(next_prompt_tokens)].tolist()
841 + rank_items = ranks[selected_logprobs_idx:selected_logprobs_idx +
842 + len(next_prompt_tokens)].tolist()
843 +
844 + for idx, token_id in enumerate(next_prompt_tokens):
845 # Calculate the prompt logprob of the real prompt tokens.
837 - # Use tuple here for performance (to use to_list()).
846 # {token_id: (logprob, rank_from_vocab)}
847 prompt_logprobs_dict: Dict[int, Tuple[float, int]] = {
840 - token_id: (selected_logprobs[selected_logprobs_idx].item(),
841 - ranks[selected_logprobs_idx].item())
848 + token_id: (selected_logprob_items[idx], rank_items[idx])
849 }
850
851 # Add top K prompt logprobs along with its rank.
852 if num_logprobs > 0:
846 - prompt_logprobs_dict.update(
847 - zip(
848 - top_token_ids[top_logprob_idx, :num_logprobs].tolist(),
849 - zip(
850 - top_logprobs[
851 - top_logprob_idx, :num_logprobs].tolist(),
852 - # This is ranks. Since top_logprob is sorted,
853 - # we can just use a range here.
854 - range(1, num_logprobs + 1))))
853 + top_ids = top_token_ids[
854 + top_logprob_idx, :num_logprobs].tolist()
855 + top_probs = top_logprobs[
856 + top_logprob_idx, :num_logprobs].tolist()
857 + # Top K is already sorted by rank, so we can use 1 ~
858 + # num_logprobs + 1 for rank.
859 + top_ranks = range(1, num_logprobs + 1)
860 + prompt_logprobs_dict.update({
861 + top_id: (top_prob, rank)
862 + for top_id, top_prob, rank in zip(top_ids, top_probs,
863 + top_ranks)
864 + })
865 prompt_logprobs.append({
866 token_id: Logprob(*logprob_and_rank)
867 for token_id, logprob_and_rank in prompt_logprobs_dict.items()
868 })
869 # + 1 to go to the next prompt token.
870 top_logprob_idx += 1
861 - selected_logprobs_idx += 1
871 +
872 + # + len(next_prompt_tokens) to go to the next prompt.
873 + selected_logprobs_idx += len(next_prompt_tokens)
874 return prompt_logprobs, top_logprob_idx, selected_logprobs_idx
875
876
@@ -874,47 +886,54 @@ def _get_sampled_logprob_if_needed(
886 ):
887 """Compute the sample logprob if needed."""
888 seq_ids = seq_group.seq_ids
877 - num_logprobs = seq_group.sampling_params.logprobs
878 - if num_logprobs is None:
879 - num_logprobs = 0
889 + num_logprobs = seq_group.sampling_params.logprobs or 0
890 sampled_logprobs: SampleLogprobs = []
891 next_token_ids, parent_seq_ids = sample_result
892
893 if seq_group.do_sample:
894 assert len(next_token_ids) > 0
885 - for (next_token_id, parent_id) in zip(next_token_ids, parent_seq_ids):
886 - # Calculate the sample logprob of the real sampled tokens.
887 - # Use tuple here for performance (to use to_list()).
888 - # token_id: (logprob, rank_from_vocab)
889 - sampled_logprobs_dict: Dict[int, Tuple[float, int]] = {
890 - next_token_id:
891 - (selected_logprobs[selected_logprobs_idx].item(),
892 - ranks[selected_logprobs_idx].item())
895 + # Pre-select items from tensor. tolist() is faster than repetitive
896 + # `.item()` calls.
897 + selected_logprob_items = selected_logprobs[
898 + selected_logprobs_idx:selected_logprobs_idx +
899 + len(next_token_ids)].tolist()
900 + rank_items = ranks[selected_logprobs_idx:selected_logprobs_idx +
901 + len(next_token_ids)].tolist()
902 + for idx, (next_token_id,
903 + parent_id) in enumerate(zip(next_token_ids, parent_seq_ids)):
904 + # Get the logprob of a sampled token.
905 + sampled_logprobs_dict = {
906 + next_token_id: (selected_logprob_items[idx], rank_items[idx])
907 }
894 - # +1 to go to the next sampled token. Note that
895 - # selected_logprobs can contain duplicates unlike top_logprobs
896 - # when beam search is enabled.
897 - selected_logprobs_idx += 1
898 -
899 - # Second, add top K logprobs along with its rank.
900 - if num_logprobs >= 0:
901 - sampled_logprobs_dict.update(
902 - zip(
903 - top_token_ids[top_logprob_idx +
904 - parent_id, :num_logprobs].tolist(),
905 - zip(
906 - top_logprobs[top_logprob_idx +
907 - parent_id, :num_logprobs].tolist(),
908 - # This is rank. Since top_logprob is sorted, we
909 - # can just use a range here.
910 - range(1, num_logprobs + 1))))
908 + # Get top K logprobs.
909 + if num_logprobs > 0:
910 + top_ids = top_token_ids[top_logprob_idx +
911 + parent_id, :num_logprobs].tolist()
912 + top_probs = top_logprobs[top_logprob_idx +
913 + parent_id, :num_logprobs].tolist()
914 + # Top K is already sorted by rank, so we can use 1 ~
915 + # num_logprobs + 1 for rank.
916 + top_ranks = range(1, num_logprobs + 1)
917 + sampled_logprobs_dict.update({
918 + top_id: (top_prob, rank)
919 + for top_id, top_prob, rank in zip(top_ids, top_probs,
920 + top_ranks)
921 + })
922 +
923 sampled_logprobs.append({
924 token_id: Logprob(*logprob_and_rank)
925 for token_id, logprob_and_rank in
926 sampled_logprobs_dict.items()
927 })
916 - # There are len(seq_ids) number of sampled tokens for the current
917 - # sequence group in top_logprobs. Jump to the next seq_group.
928 +
929 + # NOTE: This part of code is not intuitive. `selected_logprobs` include
930 + # logprobs for the current step, which has len(next_token_ids) tokens
931 + # per sequence group. `logprobs` includes logprobs from the previous
932 + # steps, which has len(seq_ids) tokens per sequence group.
933 +
934 + # Iterate to the next sequence group in a batch.
935 + selected_logprobs_idx += len(next_token_ids)
936 + # Iterate to the next sequence group in a batch.
937 top_logprob_idx += len(seq_ids)
938 return sampled_logprobs, top_logprob_idx, selected_logprobs_idx
939