A high-throughput and memory-efficient inference and serving engine for LLMs
Mirrored from https://github.com/tduncan-lum/Nvidia-Optimization-Stack---VLLM.git
Synced Nov 17, 03:00 UTC
[Core] [Frontend] Make detokenization optional (#3749)
Co-authored-by: Nick Hill <nickhill@us.ibm.com>
Matthias Gerstgrasser committed
Apr 3, 2024 at 21:52 UTC
aabe8f40f2d82da49f5fd542e80cae03550dd441
3 files changed
+53
-9
tests/engine/test_detokenization.py
new
+32
@@ -0,0 +1,32 @@
1
+import pytest
2
+
3
+from vllm.entrypoints.llm import LLM
4
+from vllm.sampling_params import SamplingParams
5
+
6
+
7
+@pytest.mark.parametrize("model", ["facebook/opt-125m"])
8
+def test_computed_prefix_blocks(model: str):
9
+ # This test checks if the engine generates completions both with and
10
+ # without optional detokenization, that detokenization includes text
11
+ # and no-detokenization doesn't, and that both completions have the same
12
+ # token_ids.
13
+ prompt = (
14
+ "You are a helpful assistant. How do I build a car from cardboard and "
15
+ "paper clips? Is there an easy to follow video tutorial available "
16
+ "online for free?")
17
+
18
+ llm = LLM(model=model)
19
+ sampling_params = SamplingParams(max_tokens=10,
20
+ temperature=0.0,
21
+ detokenize=False)
22
+
23
+ outputs_no_detokenization = llm.generate(prompt,
24
+ sampling_params)[0].outputs[0]
25
+ sampling_params.detokenize = True
26
+ outputs_with_detokenization = llm.generate(prompt,
27
+ sampling_params)[0].outputs[0]
28
+
29
+ assert outputs_no_detokenization.text == ''
30
+ assert outputs_with_detokenization.text != ''
31
+ assert outputs_no_detokenization.token_ids == \
32
+ outputs_with_detokenization.token_ids
vllm/engine/llm_engine.py
+11
-9
@@ -432,7 +432,7 @@ class LLMEngine:
432
433
# Process prompt logprobs
434
prompt_logprobs = outputs.prompt_logprobs
435
- if prompt_logprobs is not None:
435
+ if prompt_logprobs is not None and seq_group.sampling_params.detokenize:
436
self.detokenizer.decode_prompt_logprobs_inplace(
437
seq_group, prompt_logprobs)
438
seq_group.prompt_logprobs = prompt_logprobs
@@ -478,8 +478,9 @@ class LLMEngine:
478
child_seqs.append((parent, parent))
479
480
for seq, _ in child_seqs:
481
- self.detokenizer.decode_sequence_inplace(seq,
482
- seq_group.sampling_params)
481
+ if seq_group.sampling_params.detokenize:
482
+ self.detokenizer.decode_sequence_inplace(
483
+ seq, seq_group.sampling_params)
484
self._check_stop(seq, seq_group.sampling_params)
485
486
# Non-beam search case
@@ -791,12 +792,13 @@ class LLMEngine:
792
if seq.get_output_len() < sampling_params.min_tokens:
793
return
794
794
- for stop_str in sampling_params.stop:
795
- if seq.output_text.endswith(stop_str):
796
- self._finalize_sequence(seq, sampling_params, stop_str)
797
- seq.status = SequenceStatus.FINISHED_STOPPED
798
- seq.stop_reason = stop_str
799
- return
795
+ if sampling_params.detokenize:
796
+ for stop_str in sampling_params.stop:
797
+ if seq.output_text.endswith(stop_str):
798
+ self._finalize_sequence(seq, sampling_params, stop_str)
799
+ seq.status = SequenceStatus.FINISHED_STOPPED
800
+ seq.stop_reason = stop_str
801
+ return
802
last_token_id = seq.get_last_token_id()
803
if last_token_id in sampling_params.stop_token_ids:
804
stop_str = self.get_tokenizer_for_seq(seq).convert_ids_to_tokens(
vllm/sampling_params.py
+10
@@ -88,6 +88,7 @@ class SamplingParams:
88
log probability of the sampled token, so there may be up to
89
`logprobs+1` elements in the response.
90
prompt_logprobs: Number of log probabilities to return per prompt token.
91
+ detokenize: Whether to detokenize the output. Defaults to True.
92
skip_special_tokens: Whether to skip special tokens in the output.
93
spaces_between_special_tokens: Whether to add spaces between special
94
tokens in the output. Defaults to True.
@@ -118,6 +119,7 @@ class SamplingParams:
119
min_tokens: int = 0,
120
logprobs: Optional[int] = None,
121
prompt_logprobs: Optional[int] = None,
122
+ detokenize: bool = True,
123
skip_special_tokens: bool = True,
124
spaces_between_special_tokens: bool = True,
125
logits_processors: Optional[List[LogitsProcessor]] = None,
@@ -150,6 +152,10 @@ class SamplingParams:
152
self.min_tokens = min_tokens
153
self.logprobs = logprobs
154
self.prompt_logprobs = prompt_logprobs
155
+ # NOTE: This parameter is only exposed at the engine level for now.
156
+ # It is not exposed in the OpenAI API server, as the OpenAI API does
157
+ # not support returning only a list of token IDs.
158
+ self.detokenize = detokenize
159
self.skip_special_tokens = skip_special_tokens
160
self.spaces_between_special_tokens = spaces_between_special_tokens
161
self.logits_processors = logits_processors
@@ -210,6 +216,10 @@ class SamplingParams:
216
if self.prompt_logprobs is not None and self.prompt_logprobs < 0:
217
raise ValueError(f"prompt_logprobs must be non-negative, got "
218
f"{self.prompt_logprobs}.")
219
+ if self.stop and not self.detokenize:
220
+ raise ValueError(
221
+ "stop strings are only supported when detokenize is True. "
222
+ "Set detokenize=True to use stop.")
223
224
def _verify_beam_search(self) -> None:
225
if self.best_of == 1: