A high-throughput and memory-efficient inference and serving engine for LLMs
Mirrored from https://github.com/tduncan-lum/Nvidia-Optimization-Stack---VLLM.git
Synced Nov 16, 19:30 UTC
[Core] Reduce unnecessary compute when logprobs=None (#6532)
Peng Guanwen committed
Jul 30, 2024 at 00:47 UTC
db9e5708a98b7209cf4465a0391139cf8fca7674
4 files changed
+135
-80
tests/samplers/test_logprobs.py
+37
-2
@@ -14,7 +14,7 @@ MODELS = ["facebook/opt-125m"]
14
@pytest.mark.parametrize("dtype",
15
["float"]) # needed for comparing logprobs with HF
16
@pytest.mark.parametrize("chunked_prefill_token_size", [1, 4, 16, -1])
17
-@pytest.mark.parametrize("num_top_logprobs", [6]) # 32000 == vocab_size
17
+@pytest.mark.parametrize("num_top_logprobs", [0, 6]) # 32000 == vocab_size
18
@pytest.mark.parametrize("detokenize", [True, False])
19
def test_get_prompt_logprobs(
20
hf_runner,
@@ -63,7 +63,10 @@ def test_get_prompt_logprobs(
63
assert result.outputs[0].logprobs is not None
64
assert len(result.outputs[0].logprobs) == max_tokens
65
for logprobs in result.outputs[0].logprobs:
66
- assert len(logprobs) == num_top_logprobs
66
+ # If the output token is not included in the top X
67
+ # logprob, it can return 1 more data
68
+ assert (len(logprobs) == num_top_logprobs
69
+ or len(logprobs) == num_top_logprobs + 1)
70
output_text = result.outputs[0].text
71
output_string_from_most_likely_tokens_lst: List[str] = []
72
for top_logprobs in result.outputs[0].logprobs:
@@ -135,3 +138,35 @@ def test_max_logprobs():
138
bad_sampling_params = SamplingParams(logprobs=2)
139
with pytest.raises(ValueError):
140
runner.generate(["Hello world"], sampling_params=bad_sampling_params)
141
+
142
+
143
+@pytest.mark.parametrize("model", MODELS)
144
+@pytest.mark.parametrize("chunked_prefill_token_size", [1, 4, 16, -1])
145
+@pytest.mark.parametrize("detokenize", [True, False])
146
+def test_none_logprobs(vllm_runner, model, chunked_prefill_token_size: int,
147
+ detokenize: bool, example_prompts):
148
+ max_num_seqs = 256
149
+ enable_chunked_prefill = False
150
+ max_num_batched_tokens = None
151
+ if chunked_prefill_token_size != -1:
152
+ enable_chunked_prefill = True
153
+ max_num_seqs = min(chunked_prefill_token_size, max_num_seqs)
154
+ max_num_batched_tokens = chunked_prefill_token_size
155
+ max_tokens = 5
156
+
157
+ with vllm_runner(
158
+ model,
159
+ enable_chunked_prefill=enable_chunked_prefill,
160
+ max_num_batched_tokens=max_num_batched_tokens,
161
+ max_num_seqs=max_num_seqs,
162
+ ) as vllm_model:
163
+ sampling_params_logprobs_none = SamplingParams(max_tokens=max_tokens,
164
+ logprobs=None,
165
+ temperature=0.0,
166
+ detokenize=detokenize)
167
+ results_logprobs_none = vllm_model.model.generate(
168
+ example_prompts, sampling_params=sampling_params_logprobs_none)
169
+
170
+ for i in range(len(results_logprobs_none)):
171
+ assert results_logprobs_none[i].outputs[0].logprobs is None
172
+ assert results_logprobs_none[i].outputs[0].cumulative_logprob is None
vllm/model_executor/layers/sampler.py
+81
-63
@@ -1,5 +1,6 @@
1
"""A layer that samples the next tokens from the model's outputs."""
2
import itertools
3
+from math import inf
4
from typing import Dict, List, Optional, Tuple
5
6
import torch
@@ -774,8 +775,11 @@ def _get_logprobs(
775
# The next token ids to get the logprob value from.
776
next_token_ids: List[int] = []
777
# The largest requested number of logprobs. We find logprobs as many as the
777
- # largest num logprobs in this API.
778
- largest_num_logprobs = 1
778
+ # largest num logprobs in this API. If every logprobs is None, it will be
779
+ # set to -1.
780
+ largest_num_logprobs = -1
781
+ # If beam search is enabled.
782
+ use_beam_search = False
783
784
# Select indices to compute logprob from, ranks of token ids, and the top
785
# k token ids from logprobs.
@@ -808,6 +812,8 @@ def _get_logprobs(
812
largest_num_logprobs = max(largest_num_logprobs,
813
sampling_params.logprobs)
814
815
+ use_beam_search = use_beam_search or sampling_params.use_beam_search
816
+
817
assert len(next_token_ids) == len(query_indices)
818
819
if len(query_indices) == 0:
@@ -815,35 +821,40 @@ def _get_logprobs(
821
empty_prompt_logprob: Optional[PromptLogprobs] = None
822
return [empty_prompt_logprob], [empty_sampled_logprob]
823
818
- query_indices_gpu = torch.tensor(query_indices, device=logprobs.device)
819
- next_token_ids_gpu = torch.tensor(next_token_ids, device=logprobs.device)
820
-
821
- # (num_selected_query_tokens, num_logprobs). Note that query_indices can
822
- # contain duplicates if beam search is enabled.
823
- selected_logprobs = logprobs[[
824
- query_indices_gpu,
825
- next_token_ids_gpu,
826
- ]]
827
- ranks = _get_ranks(
828
- logprobs[query_indices_gpu],
829
- next_token_ids_gpu,
830
- )
831
- assert selected_logprobs.shape[0] == ranks.shape[0]
832
-
833
- # Logprobs of topk tokens for a batch of sequence groups.
834
- # (num_query_tokens_across_batch).
835
- if largest_num_logprobs > 0:
836
- top_logprobs, top_token_ids = torch.topk(logprobs,
837
- largest_num_logprobs,
838
- dim=-1)
839
- else:
840
- top_logprobs, top_token_ids = None, None
824
+ selected_logprobs, ranks = None, None
825
+ top_logprobs, top_token_ids = None, None
826
+
827
+ # If largest_num_logprobs == -1, i.e. no logprobs are requested, we can
828
+ # skip the whole logprob calculation.
829
+ if largest_num_logprobs >= 0 or use_beam_search:
830
+ query_indices_gpu = torch.tensor(query_indices, device=logprobs.device)
831
+ next_token_ids_gpu = torch.tensor(next_token_ids,
832
+ device=logprobs.device)
833
+
834
+ # (num_selected_query_tokens, num_logprobs). Note that query_indices can
835
+ # contain duplicates if beam search is enabled.
836
+ selected_logprobs = logprobs[[
837
+ query_indices_gpu,
838
+ next_token_ids_gpu,
839
+ ]]
840
+ ranks = _get_ranks(
841
+ logprobs[query_indices_gpu],
842
+ next_token_ids_gpu,
843
+ )
844
+ assert selected_logprobs.shape[0] == ranks.shape[0]
845
+
846
+ # We need to compute top k only if there exists logprobs > 0.
847
+ if largest_num_logprobs > 0:
848
+ # Logprobs of topk tokens for a batch of sequence groups.
849
+ # (num_query_tokens_across_batch).
850
+ top_logprobs, top_token_ids = torch.topk(logprobs,
851
+ largest_num_logprobs,
852
+ dim=-1)
853
+ top_logprobs = top_logprobs.to('cpu')
854
+ top_token_ids = top_token_ids.to('cpu')
855
842
- selected_logprobs = selected_logprobs.to('cpu')
843
- ranks = ranks.to('cpu')
844
- if top_logprobs is not None and top_token_ids is not None:
845
- top_logprobs = top_logprobs.to('cpu')
846
- top_token_ids = top_token_ids.to('cpu')
856
+ selected_logprobs = selected_logprobs.to('cpu')
857
+ ranks = ranks.to('cpu')
858
859
# Find prompt/sample logprobs.
860
prompt_logprobs_per_seq_group: List[Optional[PromptLogprobs]] = []
@@ -940,46 +951,53 @@ def _get_sampled_logprob_if_needed(
951
):
952
"""Compute the sample logprob if needed."""
953
seq_ids = seq_group.seq_ids
943
- num_logprobs = seq_group.sampling_params.logprobs or 0
954
+ num_logprobs = seq_group.sampling_params.logprobs
955
+ use_beam_search = seq_group.sampling_params.use_beam_search
956
sampled_logprobs: SampleLogprobs = []
957
next_token_ids, parent_seq_ids = sample_result
958
959
if seq_group.do_sample:
960
assert len(next_token_ids) > 0
949
- # Pre-select items from tensor. tolist() is faster than repetitive
950
- # `.item()` calls.
951
- selected_logprob_items = selected_logprobs[
952
- selected_logprobs_idx:selected_logprobs_idx +
953
- len(next_token_ids)].tolist()
954
- rank_items = ranks[selected_logprobs_idx:selected_logprobs_idx +
955
- len(next_token_ids)].tolist()
956
- for idx, (next_token_id,
957
- parent_id) in enumerate(zip(next_token_ids, parent_seq_ids)):
958
- # Get the logprob of a sampled token.
959
- sampled_logprobs_dict = {
960
- next_token_id: (selected_logprob_items[idx], rank_items[idx])
961
- }
962
- # Get top K logprobs.
963
- if num_logprobs > 0:
964
- top_ids = top_token_ids[top_logprob_idx +
965
- parent_id, :num_logprobs].tolist()
966
- top_probs = top_logprobs[top_logprob_idx +
967
- parent_id, :num_logprobs].tolist()
968
- # Top K is already sorted by rank, so we can use 1 ~
969
- # num_logprobs + 1 for rank.
970
- top_ranks = range(1, num_logprobs + 1)
971
- sampled_logprobs_dict.update({
972
- top_id: (top_prob, rank)
973
- for top_id, top_prob, rank in zip(top_ids, top_probs,
974
- top_ranks)
961
+ if num_logprobs is None and not use_beam_search:
962
+ for next_token_id in next_token_ids:
963
+ # Use a dummy logprob
964
+ sampled_logprobs.append({next_token_id: Logprob(inf)})
965
+ else:
966
+ # Pre-select items from tensor. tolist() is faster than repetitive
967
+ # `.item()` calls.
968
+ selected_logprob_items = selected_logprobs[
969
+ selected_logprobs_idx:selected_logprobs_idx +
970
+ len(next_token_ids)].tolist()
971
+ rank_items = ranks[selected_logprobs_idx:selected_logprobs_idx +
972
+ len(next_token_ids)].tolist()
973
+ for idx, (next_token_id, parent_id) in enumerate(
974
+ zip(next_token_ids, parent_seq_ids)):
975
+ # Get the logprob of a sampled token.
976
+ sampled_logprobs_dict = {
977
+ next_token_id:
978
+ (selected_logprob_items[idx], rank_items[idx])
979
+ }
980
+ if num_logprobs is not None and num_logprobs > 0:
981
+ # Get top K logprobs.
982
+ top_ids = top_token_ids[top_logprob_idx +
983
+ parent_id, :num_logprobs].tolist()
984
+ top_probs = top_logprobs[
985
+ top_logprob_idx + parent_id, :num_logprobs].tolist()
986
+ # Top K is already sorted by rank, so we can use 1 ~
987
+ # num_logprobs + 1 for rank.
988
+ top_ranks = range(1, num_logprobs + 1)
989
+ sampled_logprobs_dict.update({
990
+ top_id: (top_prob, rank)
991
+ for top_id, top_prob, rank in zip(
992
+ top_ids, top_probs, top_ranks)
993
+ })
994
+
995
+ sampled_logprobs.append({
996
+ token_id: Logprob(*logprob_and_rank)
997
+ for token_id, logprob_and_rank in
998
+ sampled_logprobs_dict.items()
999
})
1000
977
- sampled_logprobs.append({
978
- token_id: Logprob(*logprob_and_rank)
979
- for token_id, logprob_and_rank in
980
- sampled_logprobs_dict.items()
981
- })
982
-
1001
# NOTE: This part of code is not intuitive. `selected_logprobs` include
1002
# logprobs for the current step, which has len(next_token_ids) tokens
1003
# per sequence group. `logprobs` includes logprobs from the previous
vllm/outputs.py
+9
-8
@@ -29,7 +29,7 @@ class CompletionOutput:
29
index: int
30
text: str
31
token_ids: Tuple[int, ...]
32
- cumulative_logprob: float
32
+ cumulative_logprob: Optional[float]
33
logprobs: Optional[SampleLogprobs]
34
finish_reason: Optional[str] = None
35
stop_reason: Union[int, str, None] = None
@@ -124,13 +124,14 @@ class RequestOutput:
124
include_logprobs = seq_group.sampling_params.logprobs is not None
125
text_buffer_length = seq_group.sampling_params.output_text_buffer_length
126
outputs = [
127
- CompletionOutput(seqs.index(seq),
128
- seq.get_output_text_to_return(text_buffer_length),
129
- seq.get_output_token_ids(),
130
- seq.get_cumulative_logprob(),
131
- seq.output_logprobs if include_logprobs else None,
132
- SequenceStatus.get_finished_reason(seq.status),
133
- seq.stop_reason) for seq in top_n_seqs
127
+ CompletionOutput(
128
+ seqs.index(seq),
129
+ seq.get_output_text_to_return(text_buffer_length),
130
+ seq.get_output_token_ids(),
131
+ seq.get_cumulative_logprob() if include_logprobs else None,
132
+ seq.output_logprobs if include_logprobs else None,
133
+ SequenceStatus.get_finished_reason(seq.status),
134
+ seq.stop_reason) for seq in top_n_seqs
135
]
136
137
# Every sequence in the sequence group should have the same prompt.
vllm/sampling_params.py
+8
-7
@@ -92,11 +92,12 @@ class SamplingParams:
92
min_tokens: Minimum number of tokens to generate per output sequence
93
before EOS or stop_token_ids can be generated
94
logprobs: Number of log probabilities to return per output token.
95
- Note that the implementation follows the OpenAI API: The return
96
- result includes the log probabilities on the `logprobs` most likely
97
- tokens, as well the chosen tokens. The API will always return the
98
- log probability of the sampled token, so there may be up to
99
- `logprobs+1` elements in the response.
95
+ When set to None, no probability is returned. If set to a non-None
96
+ value, the result includes the log probabilities of the specified
97
+ number of most likely tokens, as well as the chosen tokens.
98
+ Note that the implementation follows the OpenAI API: The API will
99
+ always return the log probability of the sampled token, so there
100
+ may be up to `logprobs+1` elements in the response.
101
prompt_logprobs: Number of log probabilities to return per prompt token.
102
detokenize: Whether to detokenize the output. Defaults to True.
103
skip_special_tokens: Whether to skip special tokens in the output.
@@ -168,8 +169,8 @@ class SamplingParams:
169
self.ignore_eos = ignore_eos
170
self.max_tokens = max_tokens
171
self.min_tokens = min_tokens
171
- self.logprobs = logprobs
172
- self.prompt_logprobs = prompt_logprobs
172
+ self.logprobs = 1 if logprobs is True else logprobs
173
+ self.prompt_logprobs = 1 if prompt_logprobs is True else prompt_logprobs
174
# NOTE: This parameter is only exposed at the engine level for now.
175
# It is not exposed in the OpenAI API server, as the OpenAI API does
176
# not support returning only a list of token IDs.