@tduncan / Nvidia-Optimization-Stack-VLLM Mirror · read-only / commits / db9e5708a9
Fork 0

A high-throughput and memory-efficient inference and serving engine for LLMs

Mirrored from https://github.com/tduncan-lum/Nvidia-Optimization-Stack---VLLM.git

Synced Nov 16, 19:30 UTC

[Core] Reduce unnecessary compute when logprobs=None (#6532)

Peng Guanwen committed Jul 30, 2024 at 00:47 UTC db9e5708a98b7209cf4465a0391139cf8fca7674
4 files changed +135 -80
tests/samplers/test_logprobs.py
+37 -2
@@ -14,7 +14,7 @@ MODELS = ["facebook/opt-125m"]
14 @pytest.mark.parametrize("dtype",
15 ["float"]) # needed for comparing logprobs with HF
16 @pytest.mark.parametrize("chunked_prefill_token_size", [1, 4, 16, -1])
17 -@pytest.mark.parametrize("num_top_logprobs", [6]) # 32000 == vocab_size
17 +@pytest.mark.parametrize("num_top_logprobs", [0, 6]) # 32000 == vocab_size
18 @pytest.mark.parametrize("detokenize", [True, False])
19 def test_get_prompt_logprobs(
20 hf_runner,
@@ -63,7 +63,10 @@ def test_get_prompt_logprobs(
63 assert result.outputs[0].logprobs is not None
64 assert len(result.outputs[0].logprobs) == max_tokens
65 for logprobs in result.outputs[0].logprobs:
66 - assert len(logprobs) == num_top_logprobs
66 + # If the output token is not included in the top X
67 + # logprob, it can return 1 more data
68 + assert (len(logprobs) == num_top_logprobs
69 + or len(logprobs) == num_top_logprobs + 1)
70 output_text = result.outputs[0].text
71 output_string_from_most_likely_tokens_lst: List[str] = []
72 for top_logprobs in result.outputs[0].logprobs:
@@ -135,3 +138,35 @@ def test_max_logprobs():
138 bad_sampling_params = SamplingParams(logprobs=2)
139 with pytest.raises(ValueError):
140 runner.generate(["Hello world"], sampling_params=bad_sampling_params)
141 +
142 +
143 +@pytest.mark.parametrize("model", MODELS)
144 +@pytest.mark.parametrize("chunked_prefill_token_size", [1, 4, 16, -1])
145 +@pytest.mark.parametrize("detokenize", [True, False])
146 +def test_none_logprobs(vllm_runner, model, chunked_prefill_token_size: int,
147 + detokenize: bool, example_prompts):
148 + max_num_seqs = 256
149 + enable_chunked_prefill = False
150 + max_num_batched_tokens = None
151 + if chunked_prefill_token_size != -1:
152 + enable_chunked_prefill = True
153 + max_num_seqs = min(chunked_prefill_token_size, max_num_seqs)
154 + max_num_batched_tokens = chunked_prefill_token_size
155 + max_tokens = 5
156 +
157 + with vllm_runner(
158 + model,
159 + enable_chunked_prefill=enable_chunked_prefill,
160 + max_num_batched_tokens=max_num_batched_tokens,
161 + max_num_seqs=max_num_seqs,
162 + ) as vllm_model:
163 + sampling_params_logprobs_none = SamplingParams(max_tokens=max_tokens,
164 + logprobs=None,
165 + temperature=0.0,
166 + detokenize=detokenize)
167 + results_logprobs_none = vllm_model.model.generate(
168 + example_prompts, sampling_params=sampling_params_logprobs_none)
169 +
170 + for i in range(len(results_logprobs_none)):
171 + assert results_logprobs_none[i].outputs[0].logprobs is None
172 + assert results_logprobs_none[i].outputs[0].cumulative_logprob is None
vllm/model_executor/layers/sampler.py
+81 -63
@@ -1,5 +1,6 @@
1 """A layer that samples the next tokens from the model's outputs."""
2 import itertools
3 +from math import inf
4 from typing import Dict, List, Optional, Tuple
5
6 import torch
@@ -774,8 +775,11 @@ def _get_logprobs(
775 # The next token ids to get the logprob value from.
776 next_token_ids: List[int] = []
777 # The largest requested number of logprobs. We find logprobs as many as the
777 - # largest num logprobs in this API.
778 - largest_num_logprobs = 1
778 + # largest num logprobs in this API. If every logprobs is None, it will be
779 + # set to -1.
780 + largest_num_logprobs = -1
781 + # If beam search is enabled.
782 + use_beam_search = False
783
784 # Select indices to compute logprob from, ranks of token ids, and the top
785 # k token ids from logprobs.
@@ -808,6 +812,8 @@ def _get_logprobs(
812 largest_num_logprobs = max(largest_num_logprobs,
813 sampling_params.logprobs)
814
815 + use_beam_search = use_beam_search or sampling_params.use_beam_search
816 +
817 assert len(next_token_ids) == len(query_indices)
818
819 if len(query_indices) == 0:
@@ -815,35 +821,40 @@ def _get_logprobs(
821 empty_prompt_logprob: Optional[PromptLogprobs] = None
822 return [empty_prompt_logprob], [empty_sampled_logprob]
823
818 - query_indices_gpu = torch.tensor(query_indices, device=logprobs.device)
819 - next_token_ids_gpu = torch.tensor(next_token_ids, device=logprobs.device)
820 -
821 - # (num_selected_query_tokens, num_logprobs). Note that query_indices can
822 - # contain duplicates if beam search is enabled.
823 - selected_logprobs = logprobs[[
824 - query_indices_gpu,
825 - next_token_ids_gpu,
826 - ]]
827 - ranks = _get_ranks(
828 - logprobs[query_indices_gpu],
829 - next_token_ids_gpu,
830 - )
831 - assert selected_logprobs.shape[0] == ranks.shape[0]
832 -
833 - # Logprobs of topk tokens for a batch of sequence groups.
834 - # (num_query_tokens_across_batch).
835 - if largest_num_logprobs > 0:
836 - top_logprobs, top_token_ids = torch.topk(logprobs,
837 - largest_num_logprobs,
838 - dim=-1)
839 - else:
840 - top_logprobs, top_token_ids = None, None
824 + selected_logprobs, ranks = None, None
825 + top_logprobs, top_token_ids = None, None
826 +
827 + # If largest_num_logprobs == -1, i.e. no logprobs are requested, we can
828 + # skip the whole logprob calculation.
829 + if largest_num_logprobs >= 0 or use_beam_search:
830 + query_indices_gpu = torch.tensor(query_indices, device=logprobs.device)
831 + next_token_ids_gpu = torch.tensor(next_token_ids,
832 + device=logprobs.device)
833 +
834 + # (num_selected_query_tokens, num_logprobs). Note that query_indices can
835 + # contain duplicates if beam search is enabled.
836 + selected_logprobs = logprobs[[
837 + query_indices_gpu,
838 + next_token_ids_gpu,
839 + ]]
840 + ranks = _get_ranks(
841 + logprobs[query_indices_gpu],
842 + next_token_ids_gpu,
843 + )
844 + assert selected_logprobs.shape[0] == ranks.shape[0]
845 +
846 + # We need to compute top k only if there exists logprobs > 0.
847 + if largest_num_logprobs > 0:
848 + # Logprobs of topk tokens for a batch of sequence groups.
849 + # (num_query_tokens_across_batch).
850 + top_logprobs, top_token_ids = torch.topk(logprobs,
851 + largest_num_logprobs,
852 + dim=-1)
853 + top_logprobs = top_logprobs.to('cpu')
854 + top_token_ids = top_token_ids.to('cpu')
855
842 - selected_logprobs = selected_logprobs.to('cpu')
843 - ranks = ranks.to('cpu')
844 - if top_logprobs is not None and top_token_ids is not None:
845 - top_logprobs = top_logprobs.to('cpu')
846 - top_token_ids = top_token_ids.to('cpu')
856 + selected_logprobs = selected_logprobs.to('cpu')
857 + ranks = ranks.to('cpu')
858
859 # Find prompt/sample logprobs.
860 prompt_logprobs_per_seq_group: List[Optional[PromptLogprobs]] = []
@@ -940,46 +951,53 @@ def _get_sampled_logprob_if_needed(
951 ):
952 """Compute the sample logprob if needed."""
953 seq_ids = seq_group.seq_ids
943 - num_logprobs = seq_group.sampling_params.logprobs or 0
954 + num_logprobs = seq_group.sampling_params.logprobs
955 + use_beam_search = seq_group.sampling_params.use_beam_search
956 sampled_logprobs: SampleLogprobs = []
957 next_token_ids, parent_seq_ids = sample_result
958
959 if seq_group.do_sample:
960 assert len(next_token_ids) > 0
949 - # Pre-select items from tensor. tolist() is faster than repetitive
950 - # `.item()` calls.
951 - selected_logprob_items = selected_logprobs[
952 - selected_logprobs_idx:selected_logprobs_idx +
953 - len(next_token_ids)].tolist()
954 - rank_items = ranks[selected_logprobs_idx:selected_logprobs_idx +
955 - len(next_token_ids)].tolist()
956 - for idx, (next_token_id,
957 - parent_id) in enumerate(zip(next_token_ids, parent_seq_ids)):
958 - # Get the logprob of a sampled token.
959 - sampled_logprobs_dict = {
960 - next_token_id: (selected_logprob_items[idx], rank_items[idx])
961 - }
962 - # Get top K logprobs.
963 - if num_logprobs > 0:
964 - top_ids = top_token_ids[top_logprob_idx +
965 - parent_id, :num_logprobs].tolist()
966 - top_probs = top_logprobs[top_logprob_idx +
967 - parent_id, :num_logprobs].tolist()
968 - # Top K is already sorted by rank, so we can use 1 ~
969 - # num_logprobs + 1 for rank.
970 - top_ranks = range(1, num_logprobs + 1)
971 - sampled_logprobs_dict.update({
972 - top_id: (top_prob, rank)
973 - for top_id, top_prob, rank in zip(top_ids, top_probs,
974 - top_ranks)
961 + if num_logprobs is None and not use_beam_search:
962 + for next_token_id in next_token_ids:
963 + # Use a dummy logprob
964 + sampled_logprobs.append({next_token_id: Logprob(inf)})
965 + else:
966 + # Pre-select items from tensor. tolist() is faster than repetitive
967 + # `.item()` calls.
968 + selected_logprob_items = selected_logprobs[
969 + selected_logprobs_idx:selected_logprobs_idx +
970 + len(next_token_ids)].tolist()
971 + rank_items = ranks[selected_logprobs_idx:selected_logprobs_idx +
972 + len(next_token_ids)].tolist()
973 + for idx, (next_token_id, parent_id) in enumerate(
974 + zip(next_token_ids, parent_seq_ids)):
975 + # Get the logprob of a sampled token.
976 + sampled_logprobs_dict = {
977 + next_token_id:
978 + (selected_logprob_items[idx], rank_items[idx])
979 + }
980 + if num_logprobs is not None and num_logprobs > 0:
981 + # Get top K logprobs.
982 + top_ids = top_token_ids[top_logprob_idx +
983 + parent_id, :num_logprobs].tolist()
984 + top_probs = top_logprobs[
985 + top_logprob_idx + parent_id, :num_logprobs].tolist()
986 + # Top K is already sorted by rank, so we can use 1 ~
987 + # num_logprobs + 1 for rank.
988 + top_ranks = range(1, num_logprobs + 1)
989 + sampled_logprobs_dict.update({
990 + top_id: (top_prob, rank)
991 + for top_id, top_prob, rank in zip(
992 + top_ids, top_probs, top_ranks)
993 + })
994 +
995 + sampled_logprobs.append({
996 + token_id: Logprob(*logprob_and_rank)
997 + for token_id, logprob_and_rank in
998 + sampled_logprobs_dict.items()
999 })
1000
977 - sampled_logprobs.append({
978 - token_id: Logprob(*logprob_and_rank)
979 - for token_id, logprob_and_rank in
980 - sampled_logprobs_dict.items()
981 - })
982 -
1001 # NOTE: This part of code is not intuitive. `selected_logprobs` include
1002 # logprobs for the current step, which has len(next_token_ids) tokens
1003 # per sequence group. `logprobs` includes logprobs from the previous
vllm/outputs.py
+9 -8
@@ -29,7 +29,7 @@ class CompletionOutput:
29 index: int
30 text: str
31 token_ids: Tuple[int, ...]
32 - cumulative_logprob: float
32 + cumulative_logprob: Optional[float]
33 logprobs: Optional[SampleLogprobs]
34 finish_reason: Optional[str] = None
35 stop_reason: Union[int, str, None] = None
@@ -124,13 +124,14 @@ class RequestOutput:
124 include_logprobs = seq_group.sampling_params.logprobs is not None
125 text_buffer_length = seq_group.sampling_params.output_text_buffer_length
126 outputs = [
127 - CompletionOutput(seqs.index(seq),
128 - seq.get_output_text_to_return(text_buffer_length),
129 - seq.get_output_token_ids(),
130 - seq.get_cumulative_logprob(),
131 - seq.output_logprobs if include_logprobs else None,
132 - SequenceStatus.get_finished_reason(seq.status),
133 - seq.stop_reason) for seq in top_n_seqs
127 + CompletionOutput(
128 + seqs.index(seq),
129 + seq.get_output_text_to_return(text_buffer_length),
130 + seq.get_output_token_ids(),
131 + seq.get_cumulative_logprob() if include_logprobs else None,
132 + seq.output_logprobs if include_logprobs else None,
133 + SequenceStatus.get_finished_reason(seq.status),
134 + seq.stop_reason) for seq in top_n_seqs
135 ]
136
137 # Every sequence in the sequence group should have the same prompt.
vllm/sampling_params.py
+8 -7
@@ -92,11 +92,12 @@ class SamplingParams:
92 min_tokens: Minimum number of tokens to generate per output sequence
93 before EOS or stop_token_ids can be generated
94 logprobs: Number of log probabilities to return per output token.
95 - Note that the implementation follows the OpenAI API: The return
96 - result includes the log probabilities on the `logprobs` most likely
97 - tokens, as well the chosen tokens. The API will always return the
98 - log probability of the sampled token, so there may be up to
99 - `logprobs+1` elements in the response.
95 + When set to None, no probability is returned. If set to a non-None
96 + value, the result includes the log probabilities of the specified
97 + number of most likely tokens, as well as the chosen tokens.
98 + Note that the implementation follows the OpenAI API: The API will
99 + always return the log probability of the sampled token, so there
100 + may be up to `logprobs+1` elements in the response.
101 prompt_logprobs: Number of log probabilities to return per prompt token.
102 detokenize: Whether to detokenize the output. Defaults to True.
103 skip_special_tokens: Whether to skip special tokens in the output.
@@ -168,8 +169,8 @@ class SamplingParams:
169 self.ignore_eos = ignore_eos
170 self.max_tokens = max_tokens
171 self.min_tokens = min_tokens
171 - self.logprobs = logprobs
172 - self.prompt_logprobs = prompt_logprobs
172 + self.logprobs = 1 if logprobs is True else logprobs
173 + self.prompt_logprobs = 1 if prompt_logprobs is True else prompt_logprobs
174 # NOTE: This parameter is only exposed at the engine level for now.
175 # It is not exposed in the OpenAI API server, as the OpenAI API does
176 # not support returning only a list of token IDs.