A high-throughput and memory-efficient inference and serving engine for LLMs
Mirrored from https://github.com/tduncan-lum/Nvidia-Optimization-Stack---VLLM.git
Synced Nov 16, 22:00 UTC
[ BugFix ] Prompt Logprobs Detokenization (#6223)
Co-authored-by: Zifei Tong <zifeitong@gmail.com>
Robert Shaw committed
Jul 11, 2024 at 18:02 UTC
7ed6a4f0e1b39499675edf1dd6079d4bf21eb0fe
4 files changed
+117
-32
.buildkite/test-pipeline.yaml
+4
-1
@@ -87,7 +87,10 @@ steps:
87
88
- label: Engine Test
89
mirror_hardwares: [amd]
90
- command: pytest -v -s engine tokenization test_sequence.py test_config.py test_logger.py
90
+ commands:
91
+ - pytest -v -s engine test_sequence.py test_config.py test_logger.py
92
+ # OOM in the CI unless we run this separately
93
+ - pytest -v -s tokenization
94
95
- label: Entrypoints Test
96
mirror_hardwares: [amd]
tests/tokenization/test_detokenize.py
+87
-22
@@ -1,4 +1,4 @@
1
-from typing import Dict, List
1
+from typing import Any, Dict, List, Optional
2
3
import pytest
4
from transformers import AutoTokenizer
@@ -139,6 +139,15 @@ def create_dummy_logprobs(
139
} for token_id in complete_sequence_token_ids]
140
141
142
+def create_dummy_prompt_logprobs(
143
+ complete_sequence_token_ids: List[int]
144
+) -> List[Optional[Dict[int, Any]]]:
145
+ # logprob for the first prompt token is None.
146
+ logprobs: List[Optional[Dict[int, Any]]] = [None]
147
+ logprobs.extend(create_dummy_logprobs(complete_sequence_token_ids)[1:])
148
+ return logprobs
149
+
150
+
151
@pytest.mark.parametrize("complete_sequence", TRUTH)
152
@pytest.mark.parametrize("tokenizer_name", TOKENIZERS)
153
@pytest.mark.parametrize("skip_special_tokens", [True, False])
@@ -177,13 +186,10 @@ def test_decode_sequence_logprobs(complete_sequence: str,
186
187
@pytest.mark.parametrize("complete_sequence", TRUTH)
188
@pytest.mark.parametrize("tokenizer_name", TOKENIZERS)
180
-@pytest.mark.parametrize("skip_special_tokens", [True])
181
-def test_decode_prompt_logprobs(complete_sequence: str,
182
- complete_sequence_token_ids: List[int],
183
- detokenizer: Detokenizer,
184
- skip_special_tokens: bool):
189
+def test_decode_prompt_logprobs(complete_sequence_token_ids: List[int],
190
+ detokenizer: Detokenizer):
191
"""Verify Detokenizer decodes prompt logprobs correctly."""
186
- sampling_params = SamplingParams(skip_special_tokens=skip_special_tokens,
192
+ sampling_params = SamplingParams(skip_special_tokens=True,
193
prompt_logprobs=1)
194
195
# Run sequentially.
@@ -192,19 +198,78 @@ def test_decode_prompt_logprobs(complete_sequence: str,
198
seqs=[seq],
199
sampling_params=sampling_params,
200
arrival_time=0.0)
195
- dummy_logprobs = create_dummy_logprobs(complete_sequence_token_ids)
196
- detokenizer.decode_prompt_logprobs_inplace(seq_group, dummy_logprobs)
197
- decoded_prompt_logprobs = dummy_logprobs
201
+ dummy_logprobs = create_dummy_prompt_logprobs(complete_sequence_token_ids)
202
+ detokenizer.decode_prompt_logprobs_inplace(seq_group,
203
+ dummy_logprobs,
204
+ position_offset=0)
205
+ # First logprob is None.
206
+ decoded_prompt_logprobs: List[Dict[int, Any]] = dummy_logprobs[
207
+ 1:] # type: ignore
208
199
- if skip_special_tokens:
200
- # Text for logprobs for the chosen token should be the same as the
201
- # prompt text. Note that this will only be true if we skip
202
- # special tokens.
203
- assert complete_sequence == "".join([
204
- logprobs[token_id].decoded_token for token_id, logprobs in zip(
205
- complete_sequence_token_ids, decoded_prompt_logprobs)
206
- ])
207
- assert complete_sequence != "".join([
208
- logprobs[token_id + 1].decoded_token for token_id, logprobs in zip(
209
- complete_sequence_token_ids, decoded_prompt_logprobs)
210
- ])
209
+ # decoded_prompt_logprobs doesn't contain the first token.
210
+ token_ids = complete_sequence_token_ids
211
+ tokenzier = detokenizer.get_tokenizer_for_seq(seq)
212
+ text_full = tokenzier.decode(token_ids, skip_special_tokens=True)
213
+ text_first = tokenzier.decode(token_ids[0], skip_special_tokens=True)
214
+ text = text_full[len(text_first):]
215
+
216
+ # Text for logprobs for the chosen token should be the same as the
217
+ # prompt text. Note that the first logprob is None.
218
+ assert text == "".join([
219
+ logprobs[token_id].decoded_token
220
+ for token_id, logprobs in zip(token_ids[1:], decoded_prompt_logprobs)
221
+ ])
222
+ assert text != "".join([
223
+ logprobs[token_id + 1].decoded_token
224
+ for token_id, logprobs in zip(token_ids[1:], decoded_prompt_logprobs)
225
+ ])
226
+
227
+
228
+@pytest.mark.parametrize("model", ["facebook/opt-125m"])
229
+@pytest.mark.parametrize("chunked_prefill_token_size", [1, 4, 7, 16, -1])
230
+def test_decode_prompt_logprobs_chunked_prefill(
231
+ vllm_runner,
232
+ model,
233
+ chunked_prefill_token_size: int,
234
+ example_prompts,
235
+):
236
+ max_num_seqs = 256
237
+ enable_chunked_prefill = False
238
+ max_num_batched_tokens = None
239
+ if chunked_prefill_token_size != -1:
240
+ enable_chunked_prefill = True
241
+ max_num_seqs = min(chunked_prefill_token_size, max_num_seqs)
242
+ max_num_batched_tokens = chunked_prefill_token_size
243
+
244
+ with vllm_runner(model,
245
+ dtype="half",
246
+ max_logprobs=5,
247
+ gpu_memory_utilization=0.5,
248
+ enable_chunked_prefill=enable_chunked_prefill,
249
+ max_num_batched_tokens=max_num_batched_tokens,
250
+ max_num_seqs=max_num_seqs) as vllm_model:
251
+
252
+ vllm_sampling_params = SamplingParams(max_tokens=10,
253
+ logprobs=5,
254
+ prompt_logprobs=5,
255
+ temperature=0.0)
256
+ vllm_results = vllm_model.model.generate(
257
+ example_prompts, sampling_params=vllm_sampling_params)
258
+
259
+ for idx, result in enumerate(vllm_results):
260
+ assert result.prompt_logprobs is not None
261
+ assert result.prompt_logprobs[0] is None
262
+
263
+ # Compared detokenized prompts ids to original prompt.
264
+ generated_string = ""
265
+ for (prompt_token,
266
+ prompt_logprobs) in zip(result.prompt_token_ids[1:],
267
+ result.prompt_logprobs[1:]):
268
+ # prompt_logprobs is a dict of the token_id: logprob
269
+ # We select the token_id corresponding to the actual prompt
270
+ # Decoded token in the detokenized string corresponding to this
271
+ # prompt token.
272
+ generated_string += prompt_logprobs[prompt_token].decoded_token
273
+
274
+ assert generated_string == example_prompts[idx], (
275
+ "Detokenized prompt logprobs do not match original prompt")
vllm/engine/output_processor/single_step.py
+14
-5
@@ -60,14 +60,23 @@ class SingleStepOutputProcessor(SequenceGroupOutputProcessor):
60
assert len(outputs) == 1, ("Single step should only has 1 output.")
61
output = outputs[0]
62
prompt_logprobs = output.prompt_logprobs
63
+
64
+ # If this is the first (or only) "chunk" of the prefill, we need
65
+ # to prepend None to the list of prompt logprobs. The reason for this
66
+ # is that for N prompt tokens, the Sampler will generate N-1 total
67
+ # prompt logprobs during prefill since the token at idx 0 will not
68
+ # have a logprob associated with it.
69
if prompt_logprobs is not None:
70
+ if not seq_group.prompt_logprobs:
71
+ prompt_logprobs = [None] + prompt_logprobs
72
+ seq_group.prompt_logprobs = []
73
+
74
if seq_group.sampling_params.detokenize and self.detokenizer:
75
self.detokenizer.decode_prompt_logprobs_inplace(
66
- seq_group, prompt_logprobs)
67
- if not seq_group.prompt_logprobs:
68
- # The first prompt token's logprob is None because it doesn't
69
- # have tokens that are precedent.
70
- seq_group.prompt_logprobs = [None]
76
+ seq_group,
77
+ prompt_logprobs,
78
+ position_offset=len(seq_group.prompt_logprobs))
79
+
80
seq_group.prompt_logprobs.extend(prompt_logprobs)
81
82
def _process_sequence_group_outputs(self, seq_group: SequenceGroup,
vllm/transformers_utils/detokenizer.py
+12
-4
@@ -21,14 +21,17 @@ class Detokenizer:
21
"""Returns the HF tokenizer to use for a given sequence."""
22
return self.tokenizer_group.get_lora_tokenizer(sequence.lora_request)
23
24
- def decode_prompt_logprobs_inplace(
25
- self, seq_group: SequenceGroup,
26
- prompt_logprobs: List[Optional[Dict[int, Logprob]]]) -> None:
24
+ def decode_prompt_logprobs_inplace(self, seq_group: SequenceGroup,
25
+ prompt_logprobs: List[Optional[Dict[
26
+ int, Logprob]]],
27
+ position_offset: int) -> None:
28
"""Decodes the logprobs for the prompt of a sequence group.
29
30
Args:
31
seq_group: The sequence group to decode.
32
prompt_logprobs: The logprobs to decode.
33
+ position_offset: Offset of the first index of the logprobs
34
+ relative to the start of the sequence (for chunked prefill).
35
36
Returns:
37
The prompt logprobs with the decoded tokens.
@@ -47,8 +50,13 @@ class Detokenizer:
50
next_iter_tokens: List[str] = []
51
prev_tokens = None
52
50
- for token_position, prompt_logprobs_for_token in enumerate(
53
+ for token_position_in_logprob, prompt_logprobs_for_token in enumerate(
54
prompt_logprobs):
55
+
56
+ # Absolute token position equals the index in the logprobs
57
+ # list plus the offset of the entire logprobs list relative
58
+ # to the start of the sequence.
59
+ token_position = token_position_in_logprob + position_offset
60
if not prompt_logprobs_for_token:
61
continue
62
for token_id, sample_logprob in prompt_logprobs_for_token.items():