@tduncan / Nvidia-Optimization-Stack-VLLM Mirror · read-only / commits / aabe8f40f2
Fork 0

A high-throughput and memory-efficient inference and serving engine for LLMs

Mirrored from https://github.com/tduncan-lum/Nvidia-Optimization-Stack---VLLM.git

Synced Nov 17, 03:00 UTC

[Core] [Frontend] Make detokenization optional (#3749)

Co-authored-by: Nick Hill <nickhill@us.ibm.com>

Matthias Gerstgrasser committed Apr 3, 2024 at 21:52 UTC aabe8f40f2d82da49f5fd542e80cae03550dd441
3 files changed +53 -9
tests/engine/test_detokenization.py new
+32
@@ -0,0 +1,32 @@
1 +import pytest
2 +
3 +from vllm.entrypoints.llm import LLM
4 +from vllm.sampling_params import SamplingParams
5 +
6 +
7 +@pytest.mark.parametrize("model", ["facebook/opt-125m"])
8 +def test_computed_prefix_blocks(model: str):
9 + # This test checks if the engine generates completions both with and
10 + # without optional detokenization, that detokenization includes text
11 + # and no-detokenization doesn't, and that both completions have the same
12 + # token_ids.
13 + prompt = (
14 + "You are a helpful assistant. How do I build a car from cardboard and "
15 + "paper clips? Is there an easy to follow video tutorial available "
16 + "online for free?")
17 +
18 + llm = LLM(model=model)
19 + sampling_params = SamplingParams(max_tokens=10,
20 + temperature=0.0,
21 + detokenize=False)
22 +
23 + outputs_no_detokenization = llm.generate(prompt,
24 + sampling_params)[0].outputs[0]
25 + sampling_params.detokenize = True
26 + outputs_with_detokenization = llm.generate(prompt,
27 + sampling_params)[0].outputs[0]
28 +
29 + assert outputs_no_detokenization.text == ''
30 + assert outputs_with_detokenization.text != ''
31 + assert outputs_no_detokenization.token_ids == \
32 + outputs_with_detokenization.token_ids
vllm/engine/llm_engine.py
+11 -9
@@ -432,7 +432,7 @@ class LLMEngine:
432
433 # Process prompt logprobs
434 prompt_logprobs = outputs.prompt_logprobs
435 - if prompt_logprobs is not None:
435 + if prompt_logprobs is not None and seq_group.sampling_params.detokenize:
436 self.detokenizer.decode_prompt_logprobs_inplace(
437 seq_group, prompt_logprobs)
438 seq_group.prompt_logprobs = prompt_logprobs
@@ -478,8 +478,9 @@ class LLMEngine:
478 child_seqs.append((parent, parent))
479
480 for seq, _ in child_seqs:
481 - self.detokenizer.decode_sequence_inplace(seq,
482 - seq_group.sampling_params)
481 + if seq_group.sampling_params.detokenize:
482 + self.detokenizer.decode_sequence_inplace(
483 + seq, seq_group.sampling_params)
484 self._check_stop(seq, seq_group.sampling_params)
485
486 # Non-beam search case
@@ -791,12 +792,13 @@ class LLMEngine:
792 if seq.get_output_len() < sampling_params.min_tokens:
793 return
794
794 - for stop_str in sampling_params.stop:
795 - if seq.output_text.endswith(stop_str):
796 - self._finalize_sequence(seq, sampling_params, stop_str)
797 - seq.status = SequenceStatus.FINISHED_STOPPED
798 - seq.stop_reason = stop_str
799 - return
795 + if sampling_params.detokenize:
796 + for stop_str in sampling_params.stop:
797 + if seq.output_text.endswith(stop_str):
798 + self._finalize_sequence(seq, sampling_params, stop_str)
799 + seq.status = SequenceStatus.FINISHED_STOPPED
800 + seq.stop_reason = stop_str
801 + return
802 last_token_id = seq.get_last_token_id()
803 if last_token_id in sampling_params.stop_token_ids:
804 stop_str = self.get_tokenizer_for_seq(seq).convert_ids_to_tokens(
vllm/sampling_params.py
+10
@@ -88,6 +88,7 @@ class SamplingParams:
88 log probability of the sampled token, so there may be up to
89 `logprobs+1` elements in the response.
90 prompt_logprobs: Number of log probabilities to return per prompt token.
91 + detokenize: Whether to detokenize the output. Defaults to True.
92 skip_special_tokens: Whether to skip special tokens in the output.
93 spaces_between_special_tokens: Whether to add spaces between special
94 tokens in the output. Defaults to True.
@@ -118,6 +119,7 @@ class SamplingParams:
119 min_tokens: int = 0,
120 logprobs: Optional[int] = None,
121 prompt_logprobs: Optional[int] = None,
122 + detokenize: bool = True,
123 skip_special_tokens: bool = True,
124 spaces_between_special_tokens: bool = True,
125 logits_processors: Optional[List[LogitsProcessor]] = None,
@@ -150,6 +152,10 @@ class SamplingParams:
152 self.min_tokens = min_tokens
153 self.logprobs = logprobs
154 self.prompt_logprobs = prompt_logprobs
155 + # NOTE: This parameter is only exposed at the engine level for now.
156 + # It is not exposed in the OpenAI API server, as the OpenAI API does
157 + # not support returning only a list of token IDs.
158 + self.detokenize = detokenize
159 self.skip_special_tokens = skip_special_tokens
160 self.spaces_between_special_tokens = spaces_between_special_tokens
161 self.logits_processors = logits_processors
@@ -210,6 +216,10 @@ class SamplingParams:
216 if self.prompt_logprobs is not None and self.prompt_logprobs < 0:
217 raise ValueError(f"prompt_logprobs must be non-negative, got "
218 f"{self.prompt_logprobs}.")
219 + if self.stop and not self.detokenize:
220 + raise ValueError(
221 + "stop strings are only supported when detokenize is True. "
222 + "Set detokenize=True to use stop.")
223
224 def _verify_beam_search(self) -> None:
225 if self.best_of == 1: