A high-throughput and memory-efficient inference and serving engine for LLMs
Mirrored from https://github.com/tduncan-lum/Nvidia-Optimization-Stack---VLLM.git
Synced Nov 17, 01:45 UTC
[Core] Optimize sampler get_logprobs (#4594)
SangBin Cho committed
May 9, 2024 at 00:42 UTC
d7740ea4dcee4ab75d7d6eef723f33cae957b288
1 file changed
+68
-49
vllm/model_executor/layers/sampler.py
+68
-49
@@ -782,13 +782,14 @@ def _get_logprobs(
782
top_logprobs, top_token_ids = torch.topk(logprobs,
783
largest_num_logprobs,
784
dim=-1)
785
- top_logprobs = top_logprobs.cpu()
786
- top_token_ids = top_token_ids.cpu()
785
else:
786
top_logprobs, top_token_ids = None, None
787
790
- selected_logprobs = selected_logprobs.cpu()
791
- ranks = ranks.cpu()
788
+ selected_logprobs = selected_logprobs.to('cpu')
789
+ ranks = ranks.to('cpu')
790
+ if top_logprobs is not None and top_token_ids is not None:
791
+ top_logprobs = top_logprobs.to('cpu')
792
+ top_token_ids = top_token_ids.to('cpu')
793
794
# Find prompt/sample logprobs.
795
prompt_logprobs_per_seq_group: List[Optional[PromptLogprobs]] = []
@@ -828,37 +829,48 @@ def _get_prompt_logprob_if_needed(
829
830
# Find prompt logprobs
831
prompt_logprobs: Optional[PromptLogprobs] = None
831
- if (is_prompt and sampling_params.prompt_logprobs is not None):
832
+ if is_prompt and sampling_params.prompt_logprobs is not None:
833
prompt_logprobs = []
834
num_logprobs = sampling_params.prompt_logprobs
835
next_prompt_tokens = _get_next_prompt_tokens(seq_group)
835
- for token_id in next_prompt_tokens:
836
+ # Pre-select indexes and create a list. It is faster than calling .item
837
+ # repetitively.
838
+ selected_logprob_items = selected_logprobs[
839
+ selected_logprobs_idx:selected_logprobs_idx +
840
+ len(next_prompt_tokens)].tolist()
841
+ rank_items = ranks[selected_logprobs_idx:selected_logprobs_idx +
842
+ len(next_prompt_tokens)].tolist()
843
+
844
+ for idx, token_id in enumerate(next_prompt_tokens):
845
# Calculate the prompt logprob of the real prompt tokens.
837
- # Use tuple here for performance (to use to_list()).
846
# {token_id: (logprob, rank_from_vocab)}
847
prompt_logprobs_dict: Dict[int, Tuple[float, int]] = {
840
- token_id: (selected_logprobs[selected_logprobs_idx].item(),
841
- ranks[selected_logprobs_idx].item())
848
+ token_id: (selected_logprob_items[idx], rank_items[idx])
849
}
850
851
# Add top K prompt logprobs along with its rank.
852
if num_logprobs > 0:
846
- prompt_logprobs_dict.update(
847
- zip(
848
- top_token_ids[top_logprob_idx, :num_logprobs].tolist(),
849
- zip(
850
- top_logprobs[
851
- top_logprob_idx, :num_logprobs].tolist(),
852
- # This is ranks. Since top_logprob is sorted,
853
- # we can just use a range here.
854
- range(1, num_logprobs + 1))))
853
+ top_ids = top_token_ids[
854
+ top_logprob_idx, :num_logprobs].tolist()
855
+ top_probs = top_logprobs[
856
+ top_logprob_idx, :num_logprobs].tolist()
857
+ # Top K is already sorted by rank, so we can use 1 ~
858
+ # num_logprobs + 1 for rank.
859
+ top_ranks = range(1, num_logprobs + 1)
860
+ prompt_logprobs_dict.update({
861
+ top_id: (top_prob, rank)
862
+ for top_id, top_prob, rank in zip(top_ids, top_probs,
863
+ top_ranks)
864
+ })
865
prompt_logprobs.append({
866
token_id: Logprob(*logprob_and_rank)
867
for token_id, logprob_and_rank in prompt_logprobs_dict.items()
868
})
869
# + 1 to go to the next prompt token.
870
top_logprob_idx += 1
861
- selected_logprobs_idx += 1
871
+
872
+ # + len(next_prompt_tokens) to go to the next prompt.
873
+ selected_logprobs_idx += len(next_prompt_tokens)
874
return prompt_logprobs, top_logprob_idx, selected_logprobs_idx
875
876
@@ -874,47 +886,54 @@ def _get_sampled_logprob_if_needed(
886
):
887
"""Compute the sample logprob if needed."""
888
seq_ids = seq_group.seq_ids
877
- num_logprobs = seq_group.sampling_params.logprobs
878
- if num_logprobs is None:
879
- num_logprobs = 0
889
+ num_logprobs = seq_group.sampling_params.logprobs or 0
890
sampled_logprobs: SampleLogprobs = []
891
next_token_ids, parent_seq_ids = sample_result
892
893
if seq_group.do_sample:
894
assert len(next_token_ids) > 0
885
- for (next_token_id, parent_id) in zip(next_token_ids, parent_seq_ids):
886
- # Calculate the sample logprob of the real sampled tokens.
887
- # Use tuple here for performance (to use to_list()).
888
- # token_id: (logprob, rank_from_vocab)
889
- sampled_logprobs_dict: Dict[int, Tuple[float, int]] = {
890
- next_token_id:
891
- (selected_logprobs[selected_logprobs_idx].item(),
892
- ranks[selected_logprobs_idx].item())
895
+ # Pre-select items from tensor. tolist() is faster than repetitive
896
+ # `.item()` calls.
897
+ selected_logprob_items = selected_logprobs[
898
+ selected_logprobs_idx:selected_logprobs_idx +
899
+ len(next_token_ids)].tolist()
900
+ rank_items = ranks[selected_logprobs_idx:selected_logprobs_idx +
901
+ len(next_token_ids)].tolist()
902
+ for idx, (next_token_id,
903
+ parent_id) in enumerate(zip(next_token_ids, parent_seq_ids)):
904
+ # Get the logprob of a sampled token.
905
+ sampled_logprobs_dict = {
906
+ next_token_id: (selected_logprob_items[idx], rank_items[idx])
907
}
894
- # +1 to go to the next sampled token. Note that
895
- # selected_logprobs can contain duplicates unlike top_logprobs
896
- # when beam search is enabled.
897
- selected_logprobs_idx += 1
898
-
899
- # Second, add top K logprobs along with its rank.
900
- if num_logprobs >= 0:
901
- sampled_logprobs_dict.update(
902
- zip(
903
- top_token_ids[top_logprob_idx +
904
- parent_id, :num_logprobs].tolist(),
905
- zip(
906
- top_logprobs[top_logprob_idx +
907
- parent_id, :num_logprobs].tolist(),
908
- # This is rank. Since top_logprob is sorted, we
909
- # can just use a range here.
910
- range(1, num_logprobs + 1))))
908
+ # Get top K logprobs.
909
+ if num_logprobs > 0:
910
+ top_ids = top_token_ids[top_logprob_idx +
911
+ parent_id, :num_logprobs].tolist()
912
+ top_probs = top_logprobs[top_logprob_idx +
913
+ parent_id, :num_logprobs].tolist()
914
+ # Top K is already sorted by rank, so we can use 1 ~
915
+ # num_logprobs + 1 for rank.
916
+ top_ranks = range(1, num_logprobs + 1)
917
+ sampled_logprobs_dict.update({
918
+ top_id: (top_prob, rank)
919
+ for top_id, top_prob, rank in zip(top_ids, top_probs,
920
+ top_ranks)
921
+ })
922
+
923
sampled_logprobs.append({
924
token_id: Logprob(*logprob_and_rank)
925
for token_id, logprob_and_rank in
926
sampled_logprobs_dict.items()
927
})
916
- # There are len(seq_ids) number of sampled tokens for the current
917
- # sequence group in top_logprobs. Jump to the next seq_group.
928
+
929
+ # NOTE: This part of code is not intuitive. `selected_logprobs` include
930
+ # logprobs for the current step, which has len(next_token_ids) tokens
931
+ # per sequence group. `logprobs` includes logprobs from the previous
932
+ # steps, which has len(seq_ids) tokens per sequence group.
933
+
934
+ # Iterate to the next sequence group in a batch.
935
+ selected_logprobs_idx += len(next_token_ids)
936
+ # Iterate to the next sequence group in a batch.
937
top_logprob_idx += len(seq_ids)
938
return sampled_logprobs, top_logprob_idx, selected_logprobs_idx
939